1 package fileIO; 2 3 import java.io.BufferedInputStream; 4 import java.io.BufferedOutputStream; 5 import java.io.BufferedReader; 6 import java.io.File; 7 import java.io.FileInputStream; 8 import java.io.FileNotFoundException; 9 import java.io.FileOutputStream; 10 import java.io.IOException; 11 import java.io.InputStream; 12 import java.io.InputStreamReader; 13 import java.io.ObjectInputStream; 14 import java.io.ObjectOutputStream; 15 import java.io.OutputStream; 16 import java.io.PrintWriter; 17 import java.io.Reader; 18 import java.lang.ProcessBuilder.Redirect; 19 import java.net.MalformedURLException; 20 import java.net.URL; 21 import java.util.ArrayList; 22 import java.util.Arrays; 23 import java.util.HashMap; 24 import java.util.HashSet; 25 import java.util.Locale; 26 import java.util.zip.GZIPInputStream; 27 import java.util.zip.GZIPOutputStream; 28 import java.util.zip.ZipEntry; 29 import java.util.zip.ZipInputStream; 30 import java.util.zip.ZipOutputStream; 31 32 import dna.Data; 33 import shared.KillSwitch; 34 import shared.Shared; 35 import shared.Tools; 36 import stream.ConcurrentReadOutputStream; 37 import stream.ConcurrentReadStreamInterface; 38 import stream.MultiCros; 39 import structures.ByteBuilder; 40 41 public class ReadWrite { 42 43 main(String[] args)44 public static void main(String[] args){ 45 File f=new File(args[1]); 46 assert(!f.exists()) : "Destination file already exists."; 47 copyFile(args[0], args[1]); 48 } 49 writeStringInThread(CharSequence x, String fname)50 public static void writeStringInThread(CharSequence x, String fname){ 51 writeStringInThread(x, fname, false); 52 } 53 writeStringInThread(CharSequence x, String fname, boolean append)54 public static void writeStringInThread(CharSequence x, String fname, boolean append){ 55 addThread(1); 56 new Thread(new WriteStringThread(x, fname, append)).start(); 57 } 58 writeObjectInThread(Object x, String fname, boolean allowSubprocess)59 public static void writeObjectInThread(Object x, String fname, boolean allowSubprocess){ 60 addThread(1); 61 new Thread(new WriteObjectThread(x, fname, allowSubprocess)).start(); 62 } 63 64 private static class WriteStringThread implements Runnable{ 65 66 private final CharSequence x; 67 private final String fname; 68 private final boolean append; WriteStringThread(CharSequence x_, String fname_, boolean append_)69 WriteStringThread(CharSequence x_, String fname_, boolean append_){ 70 x=x_; 71 fname=fname_; 72 append=append_; 73 } 74 75 @Override run()76 public void run() { 77 if(verbose){System.err.println("WriteStringThread.run() started for fname "+fname);} 78 addRunningThread(1); 79 writeStringAsync(x, fname, append); 80 addThread(-1); 81 if(verbose){System.err.println("WriteStringThread.run() finished for fname "+fname);} 82 } 83 84 } 85 86 private static class WriteObjectThread implements Runnable{ 87 88 private final Object x; 89 private final String fname; 90 private final boolean allowSubprocess; WriteObjectThread(Object x_, String fname_, boolean allowSubprocess_)91 WriteObjectThread(Object x_, String fname_, boolean allowSubprocess_){ 92 x=x_; 93 fname=fname_; 94 allowSubprocess=allowSubprocess_; 95 } 96 97 @Override run()98 public void run() { 99 if(verbose){System.err.println("WriteObjectThread.run() started for fname "+fname);} 100 addRunningThread(1); 101 // System.out.println(fname+" began writing."); 102 writeAsync(x, fname, allowSubprocess); 103 // System.out.println(fname+" finished writing."); 104 addThread(-1); 105 // System.out.println(fname+" reports "+countActiveThreads()+" active threads."); 106 if(verbose){System.err.println("WriteObjectThread.run() finished for fname "+fname);} 107 } 108 109 } 110 setPermissions(String fname, boolean read, boolean write, boolean execute, boolean ownerOnly)111 public static boolean setPermissions(String fname, boolean read, boolean write, boolean execute, boolean ownerOnly){ 112 File f=new File(fname); 113 if(!f.exists()){return false;} 114 try { 115 f.setReadable(read, ownerOnly); 116 f.setWritable(write, ownerOnly); 117 f.setExecutable(execute, ownerOnly); 118 } catch (Exception e) { 119 return false; 120 } 121 return true; 122 } 123 writeString(CharSequence x, String fname)124 public static void writeString(CharSequence x, String fname){writeString(x, fname, false);} writeString(CharSequence x, String fname, boolean append)125 public static void writeString(CharSequence x, String fname, boolean append){ 126 if(verbose){System.err.println("writeString(x, "+fname+", "+append+")");} 127 OutputStream os=getOutputStream(fname, append, true, false); 128 129 try { 130 131 synchronized(diskSync){ 132 PrintWriter out=new PrintWriter(os); 133 out.print(x); 134 out.flush(); 135 136 if(os.getClass()==ZipOutputStream.class){ 137 ZipOutputStream zos=(ZipOutputStream)os; 138 zos.closeEntry(); 139 zos.finish(); 140 } 141 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){ 142 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os; 143 // zos.finish(); 144 // } 145 out.close(); 146 } 147 // System.out.println("Wrote to "+fname); 148 149 // String read=readString(fname); 150 // assert(x.equals(read)) : x.length()+", "+read.length(); 151 152 } catch (FileNotFoundException e) { 153 throw new RuntimeException(e); 154 } catch (IOException e) { 155 throw new RuntimeException(e); 156 } catch (OutOfMemoryError e) { 157 KillSwitch.memKill(e); 158 } 159 } 160 writeStringAsync(CharSequence x, String fname)161 public static void writeStringAsync(CharSequence x, String fname){writeStringAsync(x, fname, false);} writeStringAsync(CharSequence x, String fname, boolean append)162 public static void writeStringAsync(CharSequence x, String fname, boolean append){ 163 if(verbose){System.err.println("writeStringAsync(x, "+fname+", "+append+")");} 164 165 OutputStream os=getOutputStream(fname, append, true, false); 166 167 try { 168 169 synchronized(diskSync){ 170 PrintWriter out=new PrintWriter(os); 171 out.print(x); 172 out.flush(); 173 174 if(os.getClass()==ZipOutputStream.class){ 175 ZipOutputStream zos=(ZipOutputStream)os; 176 zos.closeEntry(); 177 zos.finish(); 178 } 179 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){ 180 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os; 181 // zos.finish(); 182 // } 183 out.close(); 184 } 185 // System.out.println("Wrote to "+fname); 186 187 // String read=readString(fname); 188 // assert(x.equals(read)) : x.length()+", "+read.length(); 189 190 } catch (FileNotFoundException e) { 191 throw new RuntimeException(e); 192 } catch (IOException e) { 193 throw new RuntimeException(e); 194 } catch (OutOfMemoryError e) { 195 KillSwitch.memKill(e); 196 } 197 } 198 write(X x, String fname, boolean allowSubprocess)199 public static <X> void write(X x, String fname, boolean allowSubprocess){ 200 if(verbose){System.err.println("write(x, "+fname+", "+allowSubprocess+")");} 201 202 OutputStream os=getOutputStream(fname, false, true, allowSubprocess); 203 204 try { 205 206 synchronized(diskSync){ 207 ObjectOutputStream out=new ObjectOutputStream(os); 208 out.writeObject(x); 209 close(out); 210 } 211 212 } catch (FileNotFoundException e) { 213 throw new RuntimeException(e); 214 } catch (IOException e) { 215 throw new RuntimeException(e); 216 } catch (OutOfMemoryError e) { 217 KillSwitch.memKill(e); 218 } 219 } 220 writeAsync(X x, String fname, boolean allowSubprocess)221 public static <X> void writeAsync(X x, String fname, boolean allowSubprocess){ 222 if(verbose){System.err.println("writeAsync(x, "+fname+", "+allowSubprocess+")");} 223 224 OutputStream os=getOutputStream(fname, false, true, allowSubprocess); 225 226 try { 227 228 ObjectOutputStream out=new ObjectOutputStream(os); 229 out.writeObject(x); 230 close(out); 231 232 } catch (FileNotFoundException e) { 233 throw new RuntimeException(e); 234 } catch (IOException e) { 235 throw new RuntimeException(e); 236 } catch (OutOfMemoryError e) { 237 KillSwitch.memKill(e); 238 } 239 } 240 finishReading(InputStream is, String fname, boolean killProcess, Reader...ra)241 public static final boolean finishReading(InputStream is, String fname, boolean killProcess, Reader...ra){ 242 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+", "+ra.length+")");} 243 boolean error=false; 244 if(ra!=null){ 245 for(Reader r : ra){ 246 try { 247 r.close(); 248 } catch (IOException e) { 249 error=true; 250 e.printStackTrace(); 251 } 252 } 253 } 254 error|=finishReading(is, fname, killProcess); 255 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+", "+ra.length+") returned "+error);} 256 return error; 257 } 258 finishReading(InputStream is, String fname, boolean killProcess)259 public static final boolean finishReading(InputStream is, String fname, boolean killProcess){ 260 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+")");} 261 boolean error=false; 262 if(is!=System.in){ 263 try { 264 is.close(); 265 } catch (IOException e) { 266 error=true; 267 // TODO Auto-generated catch block 268 e.printStackTrace(); 269 } 270 } 271 if(killProcess && fname!=null && is!=System.in){error|=ReadWrite.killProcess(fname);} 272 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+") returned "+error);} 273 return error; 274 } 275 276 // public static final boolean finishWriting(PrintWriter writer, OutputStream outStream, String fname){ 277 // return finishWriting(writer, outStream, fname, fname!=null); 278 // } 279 finishWriting(PrintWriter writer, OutputStream outStream, String fname, boolean killProcess)280 public static final boolean finishWriting(PrintWriter writer, OutputStream outStream, String fname, boolean killProcess){ 281 if(verbose){System.err.println("finishWriting("+writer+", "+outStream+" , "+fname+", "+killProcess+")");} 282 boolean error=false; 283 if(writer!=null){writer.flush();} 284 close(outStream); 285 if(writer!=null && outStream!=System.out && outStream!=System.err){writer.close();} 286 if(killProcess && fname!=null && outStream!=System.err && outStream!=System.out){error|=ReadWrite.killProcess(fname);} 287 if(verbose){System.err.println("finishWriting("+writer+", "+outStream+" , "+fname+", "+killProcess+") returned "+error);} 288 return error; 289 } 290 close(OutputStream os, String fname)291 public static final boolean close(OutputStream os, String fname){ 292 if(verbose){System.err.println("close("+os+", "+fname+")");} 293 boolean error=false; 294 if(os!=null){error|=close(os);} 295 if(fname!=null && os!=System.err && os!=System.out){error|=killProcess(fname);} 296 if(verbose){System.err.println("close("+os+", "+fname+") returned "+error);} 297 return error; 298 } 299 close(OutputStream os)300 public static final boolean close(OutputStream os){ 301 if(verbose){System.err.println("close("+os+")");} 302 boolean error=false; 303 try { 304 os.flush(); 305 } catch (IOException e1) { 306 // TODO Auto-generated catch block 307 e1.printStackTrace(); 308 error=true; 309 } 310 if(os.getClass()==ZipOutputStream.class){ 311 ZipOutputStream zos=(ZipOutputStream)os; 312 try { 313 zos.closeEntry(); 314 zos.finish(); 315 } catch (IOException e) { 316 // TODO Auto-generated catch block 317 e.printStackTrace(); 318 error=true; 319 } 320 } 321 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){ 322 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os; 323 // try { 324 // zos.finish(); 325 // } catch (IOException e) { 326 // // TODO Auto-generated catch block 327 // e.printStackTrace(); 328 // } 329 // } 330 if(os!=System.out && os!=System.err){ 331 try { 332 os.close(); 333 } catch (IOException e) { 334 // TODO Auto-generated catch block 335 e.printStackTrace(); 336 error=true; 337 } 338 } 339 if(verbose){System.err.println("close("+os+") returned "+error);} 340 return error; 341 } 342 getOutputStream(FileFormat ff, boolean buffered)343 public static OutputStream getOutputStream(FileFormat ff, boolean buffered){ 344 return getOutputStream(ff.name(), ff.append(), buffered, ff.allowSubprocess()); 345 } 346 getOutputStream(String fname, boolean append, boolean buffered, boolean allowSubprocess)347 public static OutputStream getOutputStream(String fname, boolean append, boolean buffered, boolean allowSubprocess){ 348 349 if(verbose){ 350 System.err.println("getOutputStream("+fname+", "+append+", "+buffered+", "+allowSubprocess+")"); 351 new Exception().printStackTrace(System.err); 352 } 353 354 // assert(false) : fname; //TODO: for testing 355 // fname=fname.replaceAll("\\\\", "/"); 356 fname=fname.replace('\\', '/'); 357 assert(fname.indexOf('\\')<0); 358 // assert(!fname.contains("//")); 359 360 {//Create directories if needed. 361 final int index=fname.lastIndexOf('/'); 362 if(index>0){ 363 File f=new File(fname.substring(0, index+1)); 364 if(!f.exists()){f.mkdirs();} 365 } 366 } 367 368 boolean gzipped=fname.endsWith(".gz") || fname.endsWith(".gzip"); 369 boolean zipped=fname.endsWith(".zip"); 370 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2"); 371 boolean xz=PROCESS_XZ && fname.endsWith(".xz"); 372 boolean dsrced=fname.endsWith(".dsrc"); 373 boolean fqz=USE_FQZ && fname.endsWith(".fqz"); 374 boolean alapy=USE_ALAPY && fname.endsWith(".ac"); 375 376 // assert(false) : fname; 377 378 allowSubprocess=(allowSubprocess && Shared.threads()>1); 379 380 if(gzipped){ 381 // assert(!append); 382 return getGZipOutputStream(fname, append, allowSubprocess); 383 }else if(zipped){ 384 assert(!append) : "Append is not allowed for zip archives."; 385 return getZipOutputStream(fname, buffered, allowSubprocess); 386 }else if(bzipped){ 387 assert(!append) : "Append is not allowed for bz2 archives.";//TODO: This might be OK; try it. 388 return getBZipOutputStream(fname, buffered, append, allowSubprocess); 389 }else if(xz){ 390 assert(!append) : "Append is not allowed for xz archives."; 391 return getXZOutputStream(fname, buffered, allowSubprocess); 392 }else if(dsrced){ 393 assert(!append) : "Append is not allowed for dsrc archives."; 394 return getDsrcOutputStream(fname, buffered, allowSubprocess); 395 }else if(fqz){ 396 assert(!append) : "Append is not allowed for fqz archives."; 397 return getFqzStream(fname); 398 }else if(alapy){ 399 assert(!append) : "Append is not allowed for alapy archives."; 400 return getAlapyStream(fname); 401 } 402 return getRawOutputStream(fname, append, buffered); 403 } 404 getRawOutputStream(String fname, boolean append, boolean buffered)405 public static OutputStream getRawOutputStream(String fname, boolean append, boolean buffered){ 406 407 if(verbose){System.err.println("getRawOutputStream("+fname+", "+append+", "+buffered+")");} 408 409 if(fname.equals("stdout") || fname.startsWith("stdout.")){ 410 return System.out; 411 }else if(fname.equals("stderr") || fname.startsWith("stderr.")){ 412 return System.err; 413 }else if(fname.startsWith("/dev/null/")){ 414 fname="/dev/null/"; 415 } 416 417 if(fname.indexOf('|')>=0){fname=fname.replace('|', '_');} 418 419 FileOutputStream fos=null; 420 try { 421 fos = new FileOutputStream(fname, append); 422 } catch (FileNotFoundException e) { 423 synchronized(ReadWrite.class){ 424 try { 425 File f=new File(fname); 426 String parent=f.getParent(); 427 428 if(parent!=null){ 429 f=new File(parent); 430 if(!f.exists()){ 431 boolean b=f.mkdirs(); 432 if(!b){System.err.println("Warning - could not create directory "+f.getAbsolutePath());} 433 } 434 } 435 fos = new FileOutputStream(fname, append); 436 } catch (Exception e2) { 437 throw new RuntimeException(e2); 438 } 439 } 440 } 441 assert(fos!=null); 442 if(buffered){return new BufferedOutputStream(fos);} 443 return fos; 444 } 445 getXZOutputStream(String fname, boolean buffered, boolean allowSubprocess)446 public static OutputStream getXZOutputStream(String fname, boolean buffered, boolean allowSubprocess){ 447 final OutputStream raw=getRawOutputStream(fname, false, buffered); 448 if(RAWMODE){return raw;} 449 throw new RuntimeException("Unsupported format: XZ"); 450 // try { 451 // org.tukaani.xz.LZMA2Options options = new org.tukaani.xz.LZMA2Options(); 452 // options.setPreset(ZIPLEVEL); 453 // org.tukaani.xz.XZOutputStream out=new org.tukaani.xz.XZOutputStream(raw, options); 454 // return out; 455 // } catch (IOException e) { 456 // // TODO Auto-generated catch block 457 // e.printStackTrace(); 458 // } 459 // assert(false); 460 // return null; 461 } 462 getBZipOutputStream(String fname, boolean buffered, boolean append, boolean allowSubprocess)463 public static OutputStream getBZipOutputStream(String fname, boolean buffered, boolean append, boolean allowSubprocess){ 464 if(verbose){System.err.println("getBZipOutputStream("+fname+", "+buffered+", "+append+", "+allowSubprocess+")");} 465 // assert(false) : ReadWrite.ZIPLEVEL+", "+Shared.threads()+", "+MAX_ZIP_THREADS+", "+ZIP_THREAD_MULT+", "+allowSubprocess+", "+USE_PIGZ+", "+Data.PIGZ(); 466 467 if(RAWMODE){ 468 final OutputStream raw=getRawOutputStream(fname, false, buffered); 469 return raw; 470 } 471 472 if(USE_LBZIP2 && Data.LBZIP2()){return getLbzip2Stream(fname, append);} 473 if(USE_PBZIP2 && Data.PBZIP2()){return getPbzip2Stream(fname, append);} 474 if(USE_BZIP2 && Data.BZIP2()){return getBzip2Stream(fname, append);} 475 476 throw new RuntimeException("bz2 compression not supported in this version, unless lbzip2, pbzip2 or bzip2 is installed."); 477 478 479 // getBzip2Stream 480 481 // {//comment to disable BZip2 482 // try { 483 // raw.write('B'); 484 // raw.write('Z'); 485 // CBZip2OutputStream out=new CBZip2OutputStream(raw, 8192); 486 // return out; 487 // } catch (IOException e) { 488 // // TODO Auto-generated catch block 489 // e.printStackTrace(); 490 // } 491 // assert(false); 492 // return null; 493 // } 494 } 495 getDsrcOutputStream(String fname, boolean buffered, boolean append)496 public static OutputStream getDsrcOutputStream(String fname, boolean buffered, boolean append){ 497 if(verbose){System.err.println("getDsrcOutputStream("+fname+", "+buffered+", "+append+")");} 498 if(RAWMODE){ 499 final OutputStream raw=getRawOutputStream(fname, false, buffered); 500 return raw; 501 } 502 503 if(USE_DSRC && Data.DSRC() /*&& (Data.SH() || fname.equals("stdout") || fname.startsWith("stdout."))*/){return getDsrcOutputStream2(fname, append);} 504 505 throw new RuntimeException("dsrc compression requires dsrc in the path."); 506 } 507 getZipOutputStream(String fname, boolean buffered, boolean allowSubprocess)508 public static OutputStream getZipOutputStream(String fname, boolean buffered, boolean allowSubprocess){ 509 if(verbose){System.err.println("getZipOutputStream("+fname+", "+buffered+", "+allowSubprocess+")");} 510 final OutputStream raw=getRawOutputStream(fname, false, buffered); 511 if(RAWMODE){return raw;} 512 try { 513 ZipOutputStream out=new ZipOutputStream(raw); 514 out.setLevel(Tools.min(ZIPLEVEL, 9)); 515 final String basename=basename(fname); 516 out.putNextEntry(new ZipEntry(basename)); 517 return out; 518 } catch (IOException e) { 519 // TODO Auto-generated catch block 520 e.printStackTrace(); 521 } 522 assert(false); 523 return null; 524 } 525 getGZipOutputStream(String fname, boolean append, boolean allowSubprocess)526 public static OutputStream getGZipOutputStream(String fname, boolean append, boolean allowSubprocess){ 527 if(verbose){System.err.println("getGZipOutputStream("+fname+", "+append+", "+allowSubprocess+"); "+USE_BGZIP+", "+USE_PIGZ+", "+USE_GZIP+", "+RAWMODE);} 528 // assert(false) : ReadWrite.ZIPLEVEL+", "+Shared.threads()+", "+MAX_ZIP_THREADS+", "+ZIP_THREAD_MULT+", "+allowSubprocess+", "+USE_PIGZ+", "+Data.PIGZ(); 529 530 if(FORCE_BGZIP && USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);} 531 532 if(FORCE_PIGZ || (allowSubprocess && Shared.threads()>=2)){ 533 if((fname.endsWith(".vcf.gz") || fname.endsWith(".sam.gz") || (PREFER_BGZIP && ZIPLEVEL<10)) && USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);} 534 if(USE_PIGZ && Data.PIGZ()){return getPigzStream(fname, append);} 535 if(USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);} 536 if(USE_GZIP && Data.GZIP()/* && (Data.SH() /*|| fname.equals("stdout") || fname.startsWith("stdout."))*/){return getGzipStream(fname, append);} 537 } 538 539 final OutputStream raw=getRawOutputStream(fname, append, false); 540 if(RAWMODE){return raw;} 541 try { 542 final GZIPOutputStream out=new GZIPOutputStream(raw, 8192){ 543 { 544 // def.setLevel(Deflater.DEFAULT_COMPRESSION); 545 def.setLevel(Tools.min(ZIPLEVEL, 9)); 546 } 547 }; 548 return out; 549 } catch (IOException e) { 550 // TODO Auto-generated catch block 551 e.printStackTrace(); 552 } 553 assert(false); 554 return null; 555 } 556 getPigzStream(String fname, boolean append)557 public static OutputStream getPigzStream(String fname, boolean append){ 558 if(verbose){System.err.println("getPigzStream("+fname+")");} 559 // System.err.println(MAX_ZIP_THREADS); //123 560 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1)); 561 // System.err.println(threads); //123 562 threads=Tools.max(1, Tools.min(Shared.threads(), threads)); 563 // System.err.println(threads); //123 564 int zl=(ZIPLEVEL<10 ? ZIPLEVEL : Data.PIGZ_VERSION_23plus ? 11 : 9); 565 if(ALLOW_ZIPLEVEL_CHANGE && threads>=4 && zl>0 && zl<4){zl=4;} 566 if(zl<3){threads=Tools.min(threads, 12);} 567 else if(zl<5){threads=Tools.min(threads, 24);} 568 else if(zl<7){threads=Tools.min(threads, 40);} 569 // System.err.println(threads); //123 570 OutputStream out; 571 String command="pigz -c -p "+threads+" -"+zl; 572 if(PIGZ_BLOCKSIZE!=128){ 573 command=command+" -b "+PIGZ_BLOCKSIZE; 574 } 575 if(PIGZ_ITERATIONS>0 && Data.PIGZ_VERSION_231plus){ 576 command=command+" -I "+PIGZ_ITERATIONS; 577 } 578 579 // System.err.println("*** "+command); 580 581 //Sample command on command line, without piping: pigz -11 -k -f -b 256 -I 25000 file.fa 582 // assert(false) : MAX_ZIP_THREADS+", "+Shared.threads()+", "+ZIP_THREAD_MULT+", "+ZIPLEVEL+", "+command; 583 out=getOutputStreamFromProcess(fname, command, true, append, true, true); 584 585 return out; 586 } 587 getFqzStream(String fname)588 public static OutputStream getFqzStream(String fname){ 589 if(verbose){System.err.println("getFqzStream("+fname+")");} 590 String command="fqz_comp -s"+Tools.mid(1, ZIPLEVEL, 8)+"+"; //9 gives bad compression 591 if(ZIPLEVEL>5){command=command+" -q3";} 592 //if(ZIPLEVEL>5){command=command+" -b -q3";} //b does not seem to work 593 OutputStream out=getOutputStreamFromProcess(fname, command, true, false, true, true); 594 return out; 595 } 596 getAlapyStream(String fname)597 public static OutputStream getAlapyStream(String fname){ 598 if(verbose){System.err.println("getAlapyStream("+fname+")");} 599 String compression=(ZIPLEVEL>6 ? "-l best" : ZIPLEVEL<4 ? "-l fast" : "-l medium"); 600 // String compression=""; 601 String command="alapy_arc "+compression+" -n "+fname+" -q -c - "; 602 OutputStream out=getOutputStreamFromProcess(fname, command, true, false, true, false); 603 return out; 604 } 605 getGzipStream(String fname, boolean append)606 public static OutputStream getGzipStream(String fname, boolean append){ 607 if(verbose){System.err.println("getGzipStream("+fname+")");} 608 OutputStream out=getOutputStreamFromProcess(fname, "gzip -c -"+Tools.min(ZIPLEVEL, 9), true, append, true, true); 609 return out; 610 } 611 getBgzipStream(String fname, boolean append)612 public static OutputStream getBgzipStream(String fname, boolean append){ 613 if(verbose){System.err.println("getBgzipStream("+fname+")");} 614 615 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1)); 616 // System.err.println(threads); //123 617 threads=Tools.max(1, Tools.min(Shared.threads(), threads)); 618 // System.err.println(threads); //123 619 int zl=Tools.mid(ZIPLEVEL, 1, 9); 620 if(ALLOW_ZIPLEVEL_CHANGE && threads>=4 && zl>0 && zl<4){zl=4;} 621 if(zl<3){threads=Tools.min(threads, 12);} 622 else if(zl<5){threads=Tools.min(threads, 16);} 623 else if(zl<7){threads=Tools.min(threads, 40);} 624 625 // assert(false) : Data.BGZIP()+", "+Data.PIGZ(); 626 String command="bgzip -c "+(append ? "" : "-f ")+(Data.BGZIP_VERSION_levelFlag ? "-l "+zl+" " : "")+(Data.BGZIP_VERSION_threadsFlag ? "-@ "+threads+" " : ""); 627 if(verbose){System.err.println(command);} 628 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true); 629 if(verbose){System.err.println("fetched bgzip stream.");} 630 return out; 631 } 632 getBzip2Stream(String fname, boolean append)633 public static OutputStream getBzip2Stream(String fname, boolean append){ 634 if(verbose){System.err.println("getBzip2Stream("+fname+")");} 635 String command="bzip2 -c -"+Tools.min(BZIPLEVEL, 9); 636 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true); 637 return out; 638 } 639 getPbzip2Stream(String fname, boolean append)640 public static OutputStream getPbzip2Stream(String fname, boolean append){ 641 if(verbose){System.err.println("getPbzip2Stream("+fname+")");} 642 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1)); 643 threads=Tools.max(1, Tools.min(Shared.threads(), threads)); 644 String command="pbzip2 -c -p"+threads+" -"+Tools.min(BZIPLEVEL, 9); 645 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true); 646 return out; 647 } 648 getLbzip2Stream(String fname, boolean append)649 public static OutputStream getLbzip2Stream(String fname, boolean append){ 650 if(verbose){System.err.println("getLbzip2Stream("+fname+")");} 651 String command="lbzip2 -"+Tools.min(BZIPLEVEL, 9); 652 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true); 653 return out; 654 } 655 getDsrcOutputStream2(String fname, boolean append)656 public static OutputStream getDsrcOutputStream2(String fname, boolean append){ 657 if(verbose){System.err.println("getDsrcOutpustream2("+fname+")");} 658 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1)); 659 threads=Tools.max(1, Tools.min(Shared.threads()-1, threads)); 660 String params=null; 661 if(ZIPLEVEL<=2){ 662 params="-d0 -q0 -b8"; 663 }else if(ZIPLEVEL<=4){ 664 params="-d1 -q1 -b16"; 665 }else if(ZIPLEVEL<=8){ 666 params="-d2 -q2 -b32"; 667 }else{ 668 params="-d3 -q2 -b64"; 669 } 670 String command="dsrc c -t"+threads+" "+params+" -s"; 671 if(fname.equals("stdout") || fname.startsWith("stdout.")){ 672 //??? 673 assert(false) : "Undefined dsrc option."; 674 }else{ 675 command+=" "+fname; 676 } 677 System.err.println(command);//123 678 // OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true); 679 OutputStream out=getOutputStreamFromProcess(fname, command+" "+fname, true, append, true, false); 680 return out; 681 } 682 getOutputStreamFromProcess(final String fname, final String command, boolean sh, boolean append, boolean useProcessBuilder, boolean useFname)683 public static OutputStream getOutputStreamFromProcess(final String fname, final String command, boolean sh, boolean append, boolean useProcessBuilder, boolean useFname){ 684 if(verbose){System.err.println("getOutputStreamFromProcess("+fname+", "+command+", "+sh+", "+useProcessBuilder+")");} 685 686 OutputStream out=null; 687 Process p=null; 688 if(useProcessBuilder){ 689 ProcessBuilder pb=new ProcessBuilder(); 690 pb.redirectError(Redirect.INHERIT); 691 692 if(fname.equals("stdout") || fname.startsWith("stdout.")){ 693 pb.redirectOutput(Redirect.INHERIT); 694 pb.command(command.split(" ")); 695 }else{ 696 697 if(useFname){ 698 if(append){ 699 pb.redirectOutput(ProcessBuilder.Redirect.appendTo(new File(fname))); 700 }else{ 701 pb.redirectOutput(new File(fname)); 702 } 703 } 704 705 pb.command(command.split(" ")); 706 } 707 try { 708 p=pb.start(); 709 } catch (IOException e) { 710 // TODO Auto-generated catch block 711 e.printStackTrace(); 712 } 713 assert(p!=null) : "Could not execute "+command; 714 addProcess(fname, p); 715 out=p.getOutputStream(); 716 { 717 out=p.getOutputStream(); 718 InputStream es=p.getErrorStream(); 719 assert(es!=null); 720 PipeThread et=new PipeThread(es, System.err); 721 addPipeThread(fname, et); 722 et.start(); 723 } 724 return out; 725 } 726 727 if(fname.equals("stdout") || fname.startsWith("stdout.")){ 728 try { 729 p = Runtime.getRuntime().exec(command); 730 } catch (IOException e) { 731 // TODO Auto-generated catch block 732 e.printStackTrace(); 733 } 734 assert(p!=null) : "Could not execute "+command; 735 InputStream is=p.getInputStream(); 736 PipeThread it=new PipeThread(is, System.out); 737 addPipeThread(fname, it); 738 it.start(); 739 // }else if(fname.equals("stderr") || fname.startsWith("stderr.")){ 740 // try { 741 // p = Runtime.getRuntime().exec(command); 742 // } catch (IOException e) { 743 // // TODO Auto-generated catch block 744 // e.printStackTrace(); 745 // } 746 // InputStream is=p.getErrorStream(); 747 // PipeThread it=new PipeThread(is, System.err); 748 // it.start(); 749 }else{ 750 try { 751 if(sh){ 752 String[] cmd = { 753 "sh", 754 "-c", 755 command+(useFname ? " 1"+(append ? ">>" : ">")+fname : "") 756 }; 757 p=Runtime.getRuntime().exec(cmd); 758 }else{ 759 //TODO: append won't work here... 760 assert(false) : command; 761 p=Runtime.getRuntime().exec(command); 762 } 763 } catch (IOException e) { 764 // TODO Auto-generated catch block 765 e.printStackTrace(); 766 } 767 } 768 assert(p!=null) : "Could not execute "+command; 769 addProcess(fname, p); 770 out=p.getOutputStream(); 771 InputStream es=p.getErrorStream(); 772 assert(es!=null); 773 PipeThread et=new PipeThread(es, System.err); 774 addPipeThread(fname, et); 775 et.start(); 776 777 return out; 778 } 779 readString(String fname)780 public static String readString(String fname){ 781 if(verbose){System.err.println("readString("+fname+")");} 782 String x=null; 783 InputStream is=getInputStream(fname, false, false); 784 785 try { 786 787 StringBuilder sb=new StringBuilder(); 788 789 // synchronized(diskSync){ 790 BufferedReader in=new BufferedReader(new InputStreamReader(is), INBUF); 791 String temp=in.readLine(); 792 while(temp!=null){ 793 sb.append(temp).append('\n'); 794 temp=in.readLine(); 795 } 796 in.close(); 797 // } 798 799 x=sb.toString(); 800 } catch (FileNotFoundException e) { 801 throw new RuntimeException(e); 802 } catch (IOException e) { 803 throw new RuntimeException(e); 804 } catch (OutOfMemoryError e) { 805 KillSwitch.memKill(e); 806 } 807 808 return x; 809 } 810 readObject(String fname, boolean allowSubprocess)811 public static Object readObject(String fname, boolean allowSubprocess){ 812 if(verbose){System.err.println("readObject("+fname+")");} 813 Object x=null; 814 InputStream is=getInputStream(fname, true, allowSubprocess); 815 816 try { 817 // synchronized(diskSync){ 818 ObjectInputStream in=new ObjectInputStream(is); 819 x=in.readObject(); 820 in.close(); 821 // } 822 } catch (IOException e) { 823 throw new RuntimeException(e); 824 } catch (ClassNotFoundException e) { 825 throw new RuntimeException(e); 826 } catch (OutOfMemoryError e) { 827 KillSwitch.memKill(e); 828 } 829 830 return x; 831 } 832 getInputStream(String fname, boolean buffer, boolean allowSubprocess)833 public static InputStream getInputStream(String fname, boolean buffer, boolean allowSubprocess){ 834 if(verbose){System.err.println("getInputStream("+fname+", "+buffer+", "+allowSubprocess+")");} 835 boolean xz=fname.endsWith(".xz"); 836 boolean gzipped=fname.endsWith(".gz") || fname.endsWith(".gzip"); 837 boolean zipped=fname.endsWith(".zip"); 838 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2"); 839 boolean dsrced=fname.endsWith(".dsrc"); 840 boolean bam=fname.endsWith(".bam") && (SAMBAMBA() || Data.SAMTOOLS()); 841 boolean fqz=fname.endsWith(".fqz"); 842 boolean alapy=fname.endsWith(".ac"); 843 844 allowSubprocess=(allowSubprocess && Shared.threads()>1); 845 846 if(!RAWMODE){ 847 if(zipped){return getZipInputStream(fname);} 848 if(gzipped){return getGZipInputStream(fname, allowSubprocess, false);} 849 if(bzipped){return getBZipInputStream(fname, allowSubprocess);} 850 if(dsrced){return getDsrcInputStream(fname);} 851 if(bam){ 852 if(SAMBAMBA()){ 853 String command="sambamba -q view -h"; 854 // new Exception().printStackTrace(); //123 855 if(SAMTOOLS_IGNORE_FLAG!=0){ 856 command=command+" --num-filter=0/"+SAMTOOLS_IGNORE_FLAG; 857 } 858 return getInputStreamFromProcess(fname, command, false, true, true); 859 }else{ 860 String command="samtools view -h"; 861 // new Exception().printStackTrace(); //123 862 if(SAMTOOLS_IGNORE_FLAG!=0){ 863 // command=command+" -F 4"; 864 command=command+" -F 0x"+Integer.toHexString(SAMTOOLS_IGNORE_FLAG); 865 } 866 String version=Data.SAMTOOLS_VERSION; 867 if(Shared.threads()>1 && version!=null && version.startsWith("1.") && version.length()>2){ 868 try { 869 String[] split=version.split("\\."); 870 int number=-1; 871 try { 872 number=Integer.parseInt(split[1]); 873 } catch (Exception e) {} 874 if(number<0){ 875 try { 876 number=Integer.parseInt(split[1].substring(0, 1)); 877 } catch (Exception e1) {} 878 } 879 if(number>3){ 880 command=command+" -@ 2"; 881 } 882 } catch (NumberFormatException e) { 883 // TODO Auto-generated catch block 884 e.printStackTrace(); 885 } 886 } 887 // System.err.println(command); 888 return getInputStreamFromProcess(fname, command, false, true, true); 889 } 890 } 891 892 if(fqz){return getInputStreamFromProcess(fname, "fqz_comp -d ", false, true, true);} 893 if(alapy){ 894 return getInputStreamFromProcess(fname, "alapy_arc -q -d "+fname+" -", false, false, true); 895 } 896 } 897 898 return getRawInputStream(fname, buffer); 899 } 900 getRawInputStream(String fname, boolean buffer)901 public static InputStream getRawInputStream(String fname, boolean buffer){ 902 if(verbose){System.err.println("getRawInputStream("+fname+", "+buffer+")");} 903 904 assert(fname!=null); 905 fname=fname.replace('\\', '/'); 906 assert(fname.indexOf('\\')<0); 907 assert(!fname.contains("\\\\")); 908 // assert(!fname.contains("//")) : fname; 909 910 final boolean jar=fname.startsWith("jar:"); 911 912 if(!jar){ 913 boolean failed=false; 914 File f=new File(fname); 915 if(!f.exists()){ 916 String f2=fname.toLowerCase(); 917 if(f2.equals("stdin") || f2.startsWith("stdin.")){ 918 // System.err.println("Returning stdin: A"); 919 return System.in; 920 } 921 922 if(fname.indexOf('/')<0){ 923 f2=Data.ROOT_CURRENT+"/"+fname; 924 if(!new File(f2).exists()){ 925 failed=true; 926 }else{ 927 fname=f2; 928 } 929 }else{ 930 failed=true; 931 } 932 } 933 if(failed){throw new RuntimeException("Can't find file "+fname);} 934 } 935 936 // System.err.println("Getting input stream for "+fname); 937 // assert(!fname.contains("\\")); 938 // assert(!loadedFiles.contains(fname)) : "Already loaded "+fname; 939 // loadedFiles.add(fname); 940 941 InputStream in=null; 942 if(jar){ 943 try { 944 945 URL url=new URL(fname); 946 947 InputStream is=url.openStream(); 948 949 if(buffer){ 950 BufferedInputStream bis=new BufferedInputStream(is, INBUF); 951 in=bis; 952 }else{ 953 in=is; 954 } 955 956 } catch (FileNotFoundException e) { 957 System.err.println("Error when attempting to read "+fname); 958 throw new RuntimeException(e); 959 } catch (MalformedURLException e) { 960 System.err.println("Error when attempting to read "+fname); 961 throw new RuntimeException(e); 962 } catch (IOException e) { 963 System.err.println("Error when attempting to read "+fname); 964 throw new RuntimeException(e); 965 } 966 }else{ 967 try { 968 969 FileInputStream fis=new FileInputStream(fname); 970 971 if(buffer){ 972 BufferedInputStream bis=new BufferedInputStream(fis, INBUF); 973 in=bis; 974 }else{ 975 in=fis; 976 } 977 978 } catch (FileNotFoundException e) { 979 throw new RuntimeException(e); 980 } 981 } 982 983 return in; 984 } 985 986 public static InputStream getZipInputStream(String fname){return getZipInputStream(fname, true);} 987 public static InputStream getZipInputStream(String fname, boolean buffer){ 988 if(verbose){System.err.println("getZipInputStream("+fname+", "+buffer+")");} 989 InputStream raw=getRawInputStream(fname, buffer); 990 InputStream in=null; 991 992 final String basename=basename(fname); 993 994 try { 995 996 ZipInputStream zis=new ZipInputStream(raw); 997 ZipEntry ze=zis.getNextEntry(); 998 assert(ze!=null); 999 assert(basename.equals(ze.getName())) : basename+" != "+ze.getName(); 1000 in=zis; 1001 1002 } catch (FileNotFoundException e) { 1003 System.err.println("Error when attempting to read "+fname); 1004 throw new RuntimeException(e); 1005 } catch (IOException e) { 1006 System.err.println("Error when attempting to read "+fname); 1007 throw new RuntimeException(e); 1008 } 1009 1010 return in; 1011 } 1012 1013 public static InputStream getGZipInputStream(String fname, boolean allowSubprocess, boolean buffer){ 1014 if(verbose){ 1015 System.err.println("getGZipInputStream("+fname+", "+allowSubprocess+")"); 1016 // new Exception().printStackTrace(System.err); 1017 } 1018 // assert(!fname.contains("temp")) : fname+", "+USE_UNBGZIP+", "+allowSubprocess; 1019 if(allowSubprocess && Shared.threads()>2){ 1020 if(!fname.startsWith("jar:")){ 1021 if(verbose){ 1022 System.err.println("Fetching gzip input stream: "+fname+", allowSubprocess="+allowSubprocess+", USE_UNPIGZ="+USE_UNPIGZ+", Data.PIGZ()="+Data.PIGZ()); 1023 } 1024 if((PREFER_UNBGZIP || fname.endsWith(".vcf.gz")) && USE_UNBGZIP && Data.BGZIP()){ 1025 if(!fname.contains("stdin") && new File(fname).exists()){ 1026 int magicNumber=getMagicNumber(fname); 1027 if(magicNumber==529205252){return getUnbgzipStream(fname);} 1028 // System.err.println(magicNumber); 1029 } 1030 } 1031 if(USE_UNPIGZ && Data.PIGZ()){return getUnpigzStream(fname);} 1032 // if(USE_UNBGZIP && Data.BGZIP()){return getUnbgzipStream(fname);} 1033 if(USE_GUNZIP && Data.GUNZIP()){return getGunzipStream(fname);} 1034 } 1035 } 1036 // assert(false) : "allowSubprocess="+allowSubprocess+", Shared.threads()="+Shared.threads()+", fname="+fname+"\n" 1037 // +"PREFER_UNBGZIP="+PREFER_UNBGZIP+", "+"USE_UNBGZIP="+USE_UNBGZIP+", "+"Data.BGZIP()="+Data.BGZIP()+"\n" 1038 // +"USE_UNPIGZ="+USE_UNPIGZ+", "+"Data.PIGZ()="+Data.PIGZ()+"\n"; 1039 InputStream raw=getRawInputStream(fname, buffer);//123 1040 InputStream in=null; 1041 try { 1042 in=new GZIPInputStream(raw, INBUF); 1043 } catch (FileNotFoundException e) { 1044 System.err.println("Error when attempting to read "+fname); 1045 throw new RuntimeException(e); 1046 } catch (IOException e) { 1047 System.err.println("Error when attempting to read "+fname); 1048 throw new RuntimeException(e); 1049 } 1050 1051 return in; 1052 } 1053 1054 public static InputStream getGunzipStream(String fname){ 1055 if(verbose){System.err.println("getGunzipStream("+fname+")");} 1056 return getInputStreamFromProcess(fname, "gzip -c -d", false, true, true); 1057 } 1058 1059 public static InputStream getUnpigzStream(String fname){ 1060 if(verbose){System.err.println("getUnpigzStream("+fname+")");} 1061 return getInputStreamFromProcess(fname, "pigz -c -d", false, true, true); 1062 } 1063 1064 public static InputStream getUnbgzipStream(String fname){ 1065 if(verbose){System.err.println("getUnbgzipStream("+fname+")");} 1066 int threads=Tools.mid(4, 1, Shared.threads()); 1067 return getInputStreamFromProcess(fname, "bgzip -c -d"+(Data.BGZIP_VERSION_threadsFlag ? " -@ "+threads : ""), false, true, true); 1068 } 1069 1070 public static InputStream getUnpbzip2Stream(String fname){ 1071 if(verbose){System.err.println("getUnpbzip2Stream("+fname+")");} 1072 return getInputStreamFromProcess(fname, "pbzip2 -c -d", false, true, true); 1073 } 1074 1075 public static InputStream getUnlbzip2Stream(String fname){ 1076 if(verbose){System.err.println("getUnlbzip2Stream("+fname+")");} 1077 return getInputStreamFromProcess(fname, "lbzip2 -c -d", false, true, true); 1078 } 1079 1080 public static InputStream getUnbzip2Stream(String fname){ 1081 if(verbose){System.err.println("getUnbzip2Stream("+fname+")");} 1082 return getInputStreamFromProcess(fname, "bzip2 -c -d", false, true, true); 1083 } 1084 1085 public static InputStream getUnDsrcStream(String fname){ 1086 if(verbose){System.err.println("getUnDsrcStream("+fname+")");} 1087 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1)); 1088 threads=Tools.max(1, Tools.min(Shared.threads()-1, threads)); 1089 return getInputStreamFromProcess(fname, "dsrc d -s -t"+threads, false, true, true); 1090 } 1091 1092 1093 public static InputStream getInputStreamFromProcess(final String fname, String command, boolean cat, final boolean appendFname, final boolean useProcessBuilder){ 1094 if(verbose){System.err.println("getInputStreamFromProcess("+fname+", "+command+", "+cat+")");} 1095 1096 //InputStream raw=getRawInputStream(fname, false); 1097 InputStream in=null; 1098 1099 Process p=null; 1100 1101 if(useProcessBuilder){ 1102 ProcessBuilder pb=new ProcessBuilder(); 1103 pb.redirectError(Redirect.INHERIT); 1104 1105 if(fname.equals("stdin") || fname.startsWith("stdin.")){ 1106 pb.redirectInput(Redirect.INHERIT); 1107 pb.command(command.split(" ")); 1108 }else{ 1109 if(appendFname){ 1110 command=command+" "+fname; 1111 }else{ 1112 pb.redirectInput(new File(fname)); 1113 } 1114 // System.err.println(command+", "+appendFname); 1115 pb.command(command.split(" ")); 1116 } 1117 try { 1118 p=pb.start(); 1119 } catch (IOException e) { 1120 // TODO Auto-generated catch block 1121 e.printStackTrace(); 1122 } 1123 assert(p!=null) : "Could not execute "+command; 1124 addProcess(fname, p); 1125 in=p.getInputStream(); 1126 // { 1127 // out=p.getOutputStream(); 1128 // InputStream es=p.getErrorStream(); 1129 // assert(es!=null); 1130 // PipeThread et=new PipeThread(es, System.err); 1131 // addPipeThread(fname, et); 1132 // et.start(); 1133 // } 1134 return in; 1135 } 1136 1137 if(!appendFname){ 1138 try { 1139 p=Runtime.getRuntime().exec(command); 1140 } catch (IOException e) { 1141 // TODO Auto-generated catch block 1142 e.printStackTrace(); 1143 } 1144 }else if(fname.equals("stdin") || fname.startsWith("stdin.")){ 1145 try { 1146 if(cat){ 1147 throw new RuntimeException(); 1148 }else{ 1149 p=Runtime.getRuntime().exec(command); 1150 } 1151 } catch (IOException e) { 1152 // TODO Auto-generated catch block 1153 e.printStackTrace(); 1154 } 1155 assert(p!=null) : "Could not execute "+command; 1156 OutputStream os=p.getOutputStream(); 1157 PipeThread it=new PipeThread(System.in, os); 1158 addPipeThread(fname, it); 1159 it.start(); 1160 }else{ 1161 try { 1162 if(cat){ 1163 assert(false) : "This mode is untested."; 1164 String[] cmd = { 1165 "sh","cat "+fname, 1166 " | "+command 1167 }; 1168 p=Runtime.getRuntime().exec(cmd); 1169 }else{ 1170 p = Runtime.getRuntime().exec(command+" "+fname); 1171 } 1172 } catch (IOException e) { 1173 // TODO Auto-generated catch block 1174 e.printStackTrace(); 1175 } 1176 } 1177 assert(p!=null) : "Could not execute "+command; 1178 1179 addProcess(fname, p); 1180 in=p.getInputStream(); 1181 InputStream es=p.getErrorStream(); 1182 assert(es!=null); 1183 PipeThread et=new PipeThread(es, System.err); 1184 addPipeThread(fname, et); 1185 et.start(); 1186 1187 return in; 1188 } 1189 1190 1191 public static InputStream getBZipInputStream(String fname, boolean allowSubprocess){ 1192 if(verbose){System.err.println("getBZipInputStream("+fname+")");} 1193 InputStream in=null; 1194 1195 try {in=getBZipInputStream2(fname, allowSubprocess);} 1196 catch (IOException e) { 1197 System.err.println("Error when attempting to read "+fname); 1198 throw new RuntimeException(e); 1199 }catch (NullPointerException e) { 1200 System.err.println("Error when attempting to read "+fname); 1201 throw new RuntimeException(e); 1202 } 1203 1204 assert(in!=null); 1205 return in; 1206 } 1207 1208 private static InputStream getBZipInputStream2(String fname, boolean allowSubprocess) throws IOException{ 1209 if(verbose){ 1210 if(verbose){System.err.println("getBZipInputStream("+fname+")");} 1211 } 1212 1213 if(!fname.startsWith("jar:")){ 1214 if(verbose){System.err.println("Fetching bz2 input stream: "+fname+", "+USE_PBZIP2+", "+USE_BZIP2+", "+Data.PBZIP2()+Data.BZIP2());} 1215 if(USE_LBZIP2 && Data.LBZIP2()){return getUnlbzip2Stream(fname);} 1216 if(USE_PBZIP2 && Data.PBZIP2()){return getUnpbzip2Stream(fname);} 1217 if(USE_BZIP2 && Data.BZIP2()){return getUnbzip2Stream(fname);} 1218 } 1219 1220 throw new IOException("\nlbzip2, pbzip2, or bzip2 must be in the path to read bz2 files:\n"+fname+"\n"); 1221 } 1222 1223 public static InputStream getDsrcInputStream(String fname){ 1224 if(verbose){System.err.println("getDsrcInputStream("+fname+")");} 1225 InputStream in=null; 1226 1227 try {in=getDsrcInputStream2(fname);} 1228 catch (IOException e) { 1229 System.err.println("Error when attempting to read "+fname); 1230 throw new RuntimeException(e); 1231 }catch (NullPointerException e) { 1232 System.err.println("Error when attempting to read "+fname); 1233 throw new RuntimeException(e); 1234 } 1235 1236 assert(in!=null); 1237 return in; 1238 } 1239 1240 private static InputStream getDsrcInputStream2(String fname) throws IOException{ 1241 if(verbose){ 1242 if(verbose){System.err.println("getDsrcInputStream2("+fname+")");} 1243 } 1244 1245 if(USE_DSRC && Data.DSRC()){return getUnDsrcStream(fname);} 1246 1247 throw new IOException("\nDsrc must be in the path to read Dsrc files:\n"+fname+"\n"); 1248 } 1249 1250 public static InputStream getXZInputStream(String fname){ 1251 1252 InputStream in=null; 1253 1254 // if(PROCESS_XZ){ 1255 // InputStream raw=getRawInputStream(fname, true); 1256 // try { 1257 // in=new org.tukaani.xz.XZInputStream(raw); 1258 // } catch (FileNotFoundException e) { 1259 // throw new RuntimeException(e); 1260 // } catch (IOException e) { 1261 // throw new RuntimeException(e); 1262 // } 1263 // } 1264 1265 return in; 1266 } 1267 1268 public static byte[] readRaw(String fname) throws IOException{ 1269 InputStream ris=getRawInputStream(fname, false); 1270 ByteBuilder bb=new ByteBuilder(); 1271 byte[] buffer=new byte[16384]; 1272 int x=ris.read(buffer); 1273 while(x>0){ 1274 bb.append(buffer, x); 1275 x=ris.read(buffer); 1276 } 1277 ris.close(); 1278 return bb.toBytes(); 1279 } 1280 read(Class<X> cx, String fname, boolean allowSubprocess)1281 public static <X> X read(Class<X> cx, String fname, boolean allowSubprocess){ 1282 X x=(X)readObject(fname, allowSubprocess); 1283 return x; 1284 } 1285 readArray(Class<X> cx, String fname, boolean allowSubprocess)1286 public static <X> X[] readArray(Class<X> cx, String fname, boolean allowSubprocess){ 1287 X[] x=(X[])readObject(fname, allowSubprocess); 1288 return x; 1289 } 1290 readArray2(Class<X> cx, String fname, boolean allowSubprocess)1291 public static <X> X[][] readArray2(Class<X> cx, String fname, boolean allowSubprocess){ 1292 X[][] x=(X[][])readObject(fname, allowSubprocess); 1293 return x; 1294 } 1295 readArray3(Class<X> cx, String fname, boolean allowSubprocess)1296 public static <X> X[][][] readArray3(Class<X> cx, String fname, boolean allowSubprocess){ 1297 X[][][] x=(X[][][])readObject(fname, allowSubprocess); 1298 return x; 1299 } 1300 1301 basename(String fname)1302 public static String basename(String fname){ 1303 fname=fname.replace('\\', '/'); 1304 boolean xz=fname.endsWith(".xz"); 1305 boolean gzipped=fname.endsWith(".gz"); 1306 boolean zipped=fname.endsWith(".zip"); 1307 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2"); 1308 boolean dsrced=fname.endsWith(".dsrc"); 1309 String basename=fname; 1310 // if(basename.contains("\\")){basename=basename.substring(basename.lastIndexOf("\\")+1);} 1311 if(basename.contains("/")){basename=basename.substring(basename.lastIndexOf('/')+1);} 1312 if(zipped || bzipped){basename=basename.substring(0, basename.length()-4);} 1313 else if(gzipped){basename=basename.substring(0, basename.length()-3);} 1314 else if(dsrced){basename=basename.substring(0, basename.length()-5);} 1315 return basename; 1316 } 1317 rawName(String fname)1318 public static String rawName(String fname){ 1319 for(String s : compressedExtensions){ 1320 while(fname.endsWith(s)){fname=fname.substring(0, fname.length()-s.length());} 1321 } 1322 return fname; 1323 } 1324 1325 /** 1326 * Returns the path without the file extension. 1327 * Only strips known extensions. */ stripExtension(String fname)1328 public static String stripExtension(String fname){ 1329 if(fname==null){return null;} 1330 for(String ext : FileFormat.EXTENSION_LIST){ 1331 String s="."+ext; 1332 if(fname.endsWith(s)){return stripExtension(fname.substring(0, fname.length()-s.length()));} 1333 } 1334 return fname; 1335 } 1336 1337 /** Returns the whole extension, include compression and raw type */ getExtension(String fname)1338 public static String getExtension(String fname){ 1339 if(fname==null){return null;} 1340 String stripped=stripExtension(fname); 1341 if(stripped==null){return fname;} 1342 if(stripped.length()==fname.length()){return "";} 1343 return fname.substring(stripped.length()); 1344 } 1345 stripToCore(String fname)1346 public static String stripToCore(String fname){ 1347 fname=stripPath(fname); 1348 return stripExtension(fname); 1349 } 1350 1351 /** 1352 * Strips the directories, leaving only a filename 1353 * @param fname 1354 * @return File name without directories 1355 */ stripPath(String fname)1356 public static String stripPath(String fname){ 1357 if(fname==null){return null;} 1358 fname=fname.replace('\\', '/'); 1359 int idx=fname.lastIndexOf('/'); 1360 if(idx>=0){fname=fname.substring(idx+1);} 1361 return fname; 1362 } 1363 getPath(String fname)1364 public static String getPath(String fname){ 1365 if(fname==null){return null;} 1366 fname=fname.replace('\\', '/'); 1367 int idx=fname.lastIndexOf('/'); 1368 if(idx>=0){return fname.substring(0, idx+1);} 1369 return ""; 1370 } 1371 compressionType(String fname)1372 public static String compressionType(String fname){ 1373 fname=fname.toLowerCase(Locale.ENGLISH); 1374 for(int i=0; i<compressedExtensions.length; i++){ 1375 if(fname.endsWith(compressedExtensions[i])){return compressedExtensionMap[i];} 1376 } 1377 return null; 1378 } 1379 isCompressed(String fname)1380 public static boolean isCompressed(String fname){ 1381 return compressionType(fname)!=null; 1382 } 1383 isSam(String fname)1384 public static boolean isSam(String fname){ 1385 fname=fname.toLowerCase(Locale.ENGLISH); 1386 if(fname.endsWith(".sam")){return true;} 1387 String s=compressionType(fname); 1388 if(s==null){return false;} 1389 return fname.substring(0, fname.lastIndexOf('.')).endsWith(".sam"); 1390 } 1391 1392 /** Returns extension, lower-case, without a period */ rawExtension(String fname)1393 public static String rawExtension(String fname){ 1394 fname=rawName(fname); 1395 int x=fname.lastIndexOf('.'); 1396 //if(x<0){return "";} 1397 return fname.substring(x+1).toLowerCase(Locale.ENGLISH); 1398 } 1399 parseRoot(String path)1400 public static String parseRoot(String path){ 1401 File f=new File(path); 1402 if(f.isDirectory()){ 1403 if(!path.endsWith(FILESEP)){ 1404 path=path+FILESEP; 1405 } 1406 return path; 1407 }else if(f.isFile()){ 1408 int slash=path.lastIndexOf(FILESEP); 1409 if(slash<0){ 1410 return ""; 1411 }else{ 1412 return path.substring(0, slash+1); 1413 } 1414 }else{ 1415 throw new RuntimeException("Can't find "+path); //Try using parseRoot2 instead. 1416 } 1417 } 1418 1419 /** This one does not throw an exception for non-existing paths */ parseRoot2(String path)1420 public static String parseRoot2(String path){ 1421 File f=new File(path); 1422 1423 if(!f.exists()){ 1424 if(path.endsWith(FILESEP)){return path;} 1425 int slash=path.lastIndexOf(FILESEP); 1426 if(slash<0){ 1427 return ""; 1428 }else{ 1429 return path.substring(0, slash+1); 1430 } 1431 } 1432 1433 if(f.isDirectory()){ 1434 if(!path.endsWith(FILESEP)){ 1435 path=path+FILESEP; 1436 } 1437 return path; 1438 }else if(f.isFile()){ 1439 int slash=path.lastIndexOf(FILESEP); 1440 if(slash<0){ 1441 return ""; 1442 }else{ 1443 return path.substring(0, slash+1); 1444 } 1445 }else{ 1446 throw new RuntimeException("Can't find "+path); 1447 } 1448 } 1449 findFileExtension(final String fname)1450 public static String findFileExtension(final String fname){ 1451 1452 File file=new File(fname); 1453 if(file.exists()){return fname;} 1454 1455 String basename=fname, temp; 1456 if(fname.endsWith(".zip") || fname.endsWith(".gz") || (PROCESS_BZ2 && fname.endsWith(".bz2")) || (PROCESS_XZ && fname.endsWith(".xz"))){ 1457 basename=fname.substring(0, fname.lastIndexOf('.')); 1458 } 1459 temp=basename; 1460 file=new File(temp); 1461 if(!file.exists()){ 1462 temp=basename+".gz"; 1463 file=new File(temp); 1464 } 1465 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist")); 1466 if(!file.exists()){ 1467 temp=basename+".zip"; 1468 file=new File(temp); 1469 } 1470 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist")); 1471 if(!file.exists() && PROCESS_BZ2){ 1472 temp=basename+".bz2"; 1473 file=new File(temp); 1474 } 1475 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist")); 1476 if(!file.exists() && PROCESS_XZ){ 1477 temp=basename+".xz"; 1478 file=new File(temp); 1479 } 1480 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist")); 1481 if(!file.exists()){temp=fname;} 1482 1483 return temp; 1484 } 1485 1486 /** 1487 * Delete a file. 1488 */ delete(String path, boolean verbose)1489 public static boolean delete(String path, boolean verbose){ 1490 if(path==null){return false;} 1491 if(verbose){System.err.println("Trying to delete "+path);} 1492 File f=new File(path); 1493 if(f.exists()){ 1494 try { 1495 f.delete(); 1496 return true; 1497 } catch (Exception e) { 1498 // TODO Auto-generated catch block 1499 e.printStackTrace(); 1500 } 1501 } 1502 return false; 1503 } 1504 copyFile(String source, String dest)1505 public static synchronized void copyFile(String source, String dest){copyFile(source, dest, false);} copyFile(String source, String dest, boolean createPathIfNeeded)1506 public static synchronized void copyFile(String source, String dest, boolean createPathIfNeeded){ 1507 1508 assert(!new File(dest).exists()) : "Destination file already exists: "+dest; 1509 if(createPathIfNeeded){ 1510 File parent=new File(dest).getParentFile(); 1511 if(parent!=null && !parent.exists()){ 1512 parent.mkdirs(); 1513 } 1514 } 1515 1516 final boolean oldRawmode=RAWMODE; 1517 if((source.endsWith(".zip") && dest.endsWith(".zip")) 1518 || (source.endsWith(".gz") && dest.endsWith(".gz") 1519 || (source.endsWith(".bz2") && dest.endsWith(".bz2")) 1520 || (source.endsWith(".xz") && dest.endsWith(".xz")))){ 1521 RAWMODE=true; 1522 } 1523 1524 try{ 1525 InputStream in=getInputStream(source, false, false); 1526 OutputStream out=getOutputStream(dest, false, false, true); 1527 1528 byte[] buffer=new byte[INBUF]; 1529 int len; 1530 1531 while((len = in.read(buffer)) > 0){ 1532 out.write(buffer, 0, len); 1533 } 1534 1535 in.close(); 1536 out.flush(); 1537 if(out.getClass()==ZipOutputStream.class){ 1538 ZipOutputStream zos=(ZipOutputStream)out; 1539 zos.closeEntry(); 1540 zos.finish(); 1541 } 1542 // else if(PROCESS_XZ && out.getClass()==org.tukaani.xz.XZOutputStream.class){ 1543 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)out; 1544 // zos.finish(); 1545 // } 1546 out.close(); 1547 1548 }catch(FileNotFoundException e){ 1549 RAWMODE=oldRawmode; 1550 throw new RuntimeException(e); 1551 }catch(IOException e){ 1552 RAWMODE=oldRawmode; 1553 throw new RuntimeException(e); 1554 } 1555 1556 RAWMODE=oldRawmode; 1557 } 1558 copyDirectoryContents(String from, String to)1559 public static void copyDirectoryContents(String from, String to){ 1560 assert(!from.equalsIgnoreCase(to)); 1561 1562 if(to.indexOf('\\')>0){to=to.replace('\\', '/');} 1563 1564 File d1=new File(from); 1565 assert(d1.exists()); 1566 assert(d1.isDirectory()); 1567 1568 File d2=new File(to); 1569 assert(!d1.equals(d2)); 1570 if(d2.exists()){ 1571 assert(d2.isDirectory()); 1572 }else{ 1573 d2.mkdirs(); 1574 } 1575 if(!to.endsWith("/")){to=to+"/";} 1576 1577 File[] array=d1.listFiles(); 1578 1579 for(File f : array){ 1580 String name=f.getName(); 1581 String dest=to+name; 1582 if(f.isFile()){ 1583 copyFile(f.getAbsolutePath(), dest); 1584 }else{ 1585 assert(f.isDirectory()); 1586 File f2=new File(dest); 1587 if(!f2.exists()){ 1588 f2.mkdir(); 1589 }else{ 1590 assert(f2.isDirectory()); 1591 } 1592 copyDirectoryContents(f.getAbsolutePath(), f2.getAbsolutePath()); 1593 } 1594 } 1595 1596 } 1597 1598 addThread(int x)1599 static final int addThread(int x){ 1600 if(verbose){System.err.println("addThread("+x+")");} 1601 synchronized(activeThreads){ 1602 assert(x!=0); 1603 if(x>0){ 1604 activeThreads[0]+=x; 1605 activeThreads[1]+=x; 1606 }else{ 1607 addRunningThread(x); 1608 } 1609 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 && 1610 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads); 1611 1612 return activeThreads[0]; 1613 } 1614 } 1615 addRunningThread(int x)1616 static final int addRunningThread(int x){ 1617 if(verbose){System.err.println("addRunningThread("+x+")");} 1618 final int max=(Shared.LOW_MEMORY ? 1 : maxWriteThreads); 1619 synchronized(activeThreads){ 1620 assert(x!=0); 1621 if(x>0){ 1622 assert(activeThreads[1]>=x); 1623 while(activeThreads[2]>=max){ 1624 try { 1625 activeThreads.wait(); 1626 } catch (InterruptedException e) { 1627 // TODO Auto-generated catch block 1628 e.printStackTrace(); 1629 } 1630 } 1631 activeThreads[1]-=x; //Remove from waiting 1632 }else{ 1633 activeThreads[0]+=x; //Remove from active 1634 } 1635 activeThreads[2]+=x; //Change number running 1636 1637 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 && 1638 activeThreads[2]>=0 && activeThreads[2]<=max) : Arrays.toString(activeThreads); 1639 1640 if(activeThreads[2]==0 || (activeThreads[2]<max && activeThreads[1]>0)){activeThreads.notify();} 1641 return activeThreads[2]; 1642 } 1643 } 1644 countActiveThreads()1645 public static final int countActiveThreads(){ 1646 if(verbose){System.err.println("countActiveThreads()");} 1647 synchronized(activeThreads){ 1648 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 && 1649 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads); 1650 return activeThreads[0]; 1651 } 1652 } 1653 waitForWritingToFinish()1654 public static final void waitForWritingToFinish(){ 1655 if(verbose){System.err.println("waitForWritingToFinish()");} 1656 synchronized(activeThreads){ 1657 while(activeThreads[0]>0){ 1658 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 && 1659 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads); 1660 try { 1661 activeThreads.wait(8000); 1662 } catch (InterruptedException e) { 1663 // TODO Auto-generated catch block 1664 e.printStackTrace(); 1665 } 1666 if(activeThreads[2]==0 || (activeThreads[2]<maxWriteThreads && activeThreads[1]>0)){activeThreads.notify();} 1667 } 1668 } 1669 } 1670 1671 closeStream(ConcurrentReadStreamInterface cris)1672 public static final boolean closeStream(ConcurrentReadStreamInterface cris){return closeStreams(cris, (ConcurrentReadOutputStream[])null);} closeStream(ConcurrentReadOutputStream ross)1673 public static final boolean closeStream(ConcurrentReadOutputStream ross){return closeStreams((ConcurrentReadStreamInterface)null, ross);} closeOutputStreams(ConcurrentReadOutputStream...ross)1674 public static final boolean closeOutputStreams(ConcurrentReadOutputStream...ross){return closeStreams(null, ross);} 1675 closeStreams(MultiCros mc)1676 public static final boolean closeStreams(MultiCros mc){ 1677 if(mc==null){return false;} 1678 return closeStreams(null, mc.streamList.toArray(new ConcurrentReadOutputStream[0])); 1679 } 1680 1681 /** 1682 * Close these streams and wait for them to finish. 1683 * @param cris An input stream. May be null. 1684 * @param ross Zero or more output streams. 1685 * @return True if an error was encountered. 1686 */ closeStreams(ConcurrentReadStreamInterface cris, ConcurrentReadOutputStream...ross)1687 public static final boolean closeStreams(ConcurrentReadStreamInterface cris, ConcurrentReadOutputStream...ross){ 1688 if(verbose){ 1689 System.err.println("closeStreams("+cris+", "+(ross==null ? "null" : ross.length)+")"); 1690 new Exception().printStackTrace(System.err); 1691 } 1692 boolean errorState=false; 1693 if(cris!=null){ 1694 if(verbose){System.err.println("Closing cris; error="+errorState);} 1695 cris.close(); 1696 errorState|=cris.errorState(); 1697 // Object[] prods=cris.producers(); 1698 // for(Object o : prods){ 1699 // if(o!=null && o.getClass()==ReadInputStream.class){ 1700 // ReadInputStream ris=(ReadInputStream)o; 1701 // ris. 1702 // } 1703 // } 1704 if(verbose){System.err.println("Closed cris; error="+errorState);} 1705 } 1706 if(ross!=null){ 1707 for(ConcurrentReadOutputStream ros : ross){ 1708 if(ros!=null){ 1709 if(verbose){System.err.println("Closing ros "+ros+"; error="+errorState);} 1710 ros.close(); 1711 ros.join(); 1712 errorState|=(ros.errorState() || !ros.finishedSuccessfully()); 1713 if(verbose){System.err.println("Closed ros; error="+errorState);} 1714 } 1715 } 1716 } 1717 return errorState; 1718 } 1719 killProcess(String fname)1720 public static boolean killProcess(String fname){ 1721 if(verbose){ 1722 System.err.println("killProcess("+fname+")"); 1723 new Exception().printStackTrace(System.err); 1724 System.err.println("processMap before: "+processMap.keySet()); 1725 } 1726 if(fname==null || (!isCompressed(fname) && !fname.endsWith(".bam") && !FORCE_KILL)){return false;} 1727 1728 boolean error=false; 1729 synchronized(processMap){ 1730 Process p=processMap.remove(fname); 1731 if(p!=null){ 1732 if(verbose){System.err.println("Found Process for "+fname);} 1733 int x=-1, tries=0; 1734 for(; tries<20; tries++){ 1735 if(verbose){System.err.println("Trying p.waitFor()");} 1736 try { 1737 // long t=System.nanoTime(); 1738 // Thread.sleep(4000); 1739 if(verbose){System.err.println("p.isAlive()="+p.isAlive());} 1740 x=p.waitFor(); 1741 // if(verbose){System.err.println(System.nanoTime()-t+" ns");} 1742 if(verbose){System.err.println("success; return="+x);} 1743 break; 1744 } catch (InterruptedException e) { 1745 if(verbose){System.err.println("Failed.");} 1746 e.printStackTrace(); 1747 } 1748 } 1749 error|=(tries>=20 || (x!=0 && x!=141));//141 is sigpipe and appears to be OK when forcibly closing a pipe. 1750 if(verbose){System.err.println("killProcess("+fname+") returned "+error+"; tries="+tries+", code="+x);} 1751 if(tries>=20){ 1752 if(verbose){System.err.println("Calling p.destroy because tries=="+tries+"; error="+error);} 1753 p.destroy(); 1754 if(verbose){System.err.println("destroyed");} 1755 } 1756 }else{ 1757 if(verbose){System.err.println("WARNING: Could not find process for "+fname);} 1758 } 1759 if(verbose){ 1760 System.err.println("processMap after: "+processMap.keySet()); 1761 } 1762 } 1763 synchronized(pipeThreadMap){ 1764 if(verbose){System.err.println("pipeMap before: "+processMap.keySet());} 1765 ArrayList<PipeThread> atp=pipeThreadMap.remove(fname); 1766 if(atp!=null){ 1767 for(PipeThread p : atp){ 1768 if(p!=null){ 1769 if(verbose){System.err.println("Found PipeThread for "+fname);} 1770 p.terminate(); 1771 if(verbose){System.err.println("Terminated PipeThread");} 1772 }else{ 1773 if(verbose){System.err.println("WARNING: Could not find process for "+fname);} 1774 } 1775 } 1776 } 1777 if(verbose){System.err.println("pipeMap after: "+processMap.keySet());} 1778 } 1779 if(verbose){System.err.println("killProcess("+fname+") returned "+error);} 1780 return error; 1781 } 1782 addProcess(String fname, Process p)1783 private static void addProcess(String fname, Process p){ 1784 if(verbose){ 1785 System.err.println("addProcess("+fname+", "+p+")"); 1786 new Exception().printStackTrace(); 1787 } 1788 synchronized(processMap){ 1789 Process old=processMap.put(fname, p); 1790 if(old!=null){ 1791 old.destroy(); 1792 // throw new RuntimeException("Duplicate process for file "+fname); 1793 KillSwitch.kill("Duplicate process for file "+fname); 1794 } 1795 } 1796 } 1797 addPipeThread(String fname, PipeThread pt)1798 private static void addPipeThread(String fname, PipeThread pt){ 1799 if(verbose){System.err.println("addPipeThread("+fname+", "+pt+")");} 1800 synchronized(pipeThreadMap){ 1801 // System.err.println("Adding PipeThread for "+fname); 1802 ArrayList<PipeThread> atp=pipeThreadMap.get(fname); 1803 if(atp==null){ 1804 atp=new ArrayList<PipeThread>(2); 1805 pipeThreadMap.put(fname, atp); 1806 } 1807 atp.add(pt); 1808 } 1809 } 1810 1811 /** 1812 * Note: 1813 * Magic number of bgzip files is (first 4 bytes): 1814 * 1f 8b 08 04 1815 * 31 139 8 4 1816 * = 529205252 1817 * 1818 * gzip/pigz: 1819 * 1f 8b 08 00 1820 * 31 139 8 0 1821 * = 529205248 1822 * 1823 * od --format=x1 --read-bytes=16 names.txt_gzip.gz 1824 */ getMagicNumber(String fname)1825 public static int getMagicNumber(String fname) { 1826 InputStream is=null; 1827 try { 1828 FileInputStream fis=new FileInputStream(fname); 1829 is=new BufferedInputStream(fis); 1830 } catch (FileNotFoundException e) { 1831 // TODO Auto-generated catch block 1832 e.printStackTrace(); 1833 } 1834 1835 //This is fine but uses Java11 methods 1836 // byte[] array=new byte[4]; 1837 // int read=0; 1838 // try { 1839 //// read=is.readNBytes(array, 0, 4); 1840 // read=is.readNBytes(array, 0, 4); 1841 // } catch (IOException e) { 1842 // // TODO Auto-generated catch block 1843 // e.printStackTrace(); 1844 // } 1845 // assert(read==4) : read; 1846 // int x=0; 1847 // for(int i=0; i<read; i++){ 1848 // int b=((int)array[i])&255; 1849 // x=(x<<8)|b; 1850 // } 1851 1852 int x=0; 1853 for(int i=0; i<4; i++){ 1854 try { 1855 x=(x<<8)|(is.read()&255); 1856 } catch (IOException e) { 1857 // TODO Auto-generated catch block 1858 e.printStackTrace(); 1859 } 1860 } 1861 1862 return x; 1863 } 1864 1865 /** {active, waiting, running} <br> 1866 * Active means running or waiting. 1867 */ 1868 public static int[] activeThreads={0, 0, 0}; 1869 public static int maxWriteThreads=Shared.threads(); 1870 1871 public static boolean verbose=false; 1872 1873 public static boolean RAWMODE=false; //Does not automatically compress and decompress when true 1874 1875 //For killing subprocesses that are neither compression nor samtools 1876 public static boolean FORCE_KILL=false; 1877 1878 public static boolean USE_GZIP=false; 1879 public static boolean USE_BGZIP=true; 1880 public static boolean USE_PIGZ=true; 1881 public static boolean USE_GUNZIP=false; 1882 public static boolean USE_UNBGZIP=true; 1883 public static boolean USE_UNPIGZ=true; 1884 1885 public static boolean FORCE_PIGZ=false; 1886 public static boolean FORCE_BGZIP=false; 1887 1888 public static boolean PREFER_BGZIP=true; 1889 public static boolean PREFER_UNBGZIP=true; 1890 1891 public static boolean USE_BZIP2=true; 1892 public static boolean USE_PBZIP2=true; 1893 public static boolean USE_LBZIP2=true; 1894 public static boolean USE_DSRC=true; 1895 public static boolean USE_FQZ=true; 1896 public static boolean USE_ALAPY=true; 1897 public static boolean USE_SAMBAMBA=true; SAMBAMBA()1898 public static boolean SAMBAMBA(){return USE_SAMBAMBA && Data.SAMBAMBA();} 1899 1900 // public static boolean SAMTOOLS_IGNORE_UNMAPPED_INPUT=false; 1901 public static int SAMTOOLS_IGNORE_FLAG=0; 1902 public static final int SAM_UNMAPPED=0x4; 1903 public static final int SAM_DUPLICATE=0x400; 1904 public static final int SAM_SUPPLIMENTARY=0x800; 1905 public static final int SAM_SECONDARY=0x100; 1906 public static final int SAM_QFAIL=0x200; 1907 1908 public static boolean PROCESS_BZ2=true; 1909 public static final boolean PROCESS_XZ=false; 1910 1911 public static final int INBUF=16384; 1912 public static final int OUTBUF=16384; 1913 1914 /** Gzip compression level */ 1915 public static int ZIPLEVEL=4; 1916 /** Bzip2 compression level */ 1917 public static int BZIPLEVEL=9; 1918 public static int MAX_ZIP_THREADS=96; 1919 public static int MAX_SAMTOOLS_THREADS=64; 1920 public static int PIGZ_BLOCKSIZE=128; 1921 public static int PIGZ_ITERATIONS=-1; 1922 setZipThreadMult(float x)1923 public static void setZipThreadMult(float x){ 1924 ZIP_THREAD_MULT=Tools.min(1, Tools.max(0.125f, x)); 1925 } 1926 public static float ZIP_THREAD_MULT=1f; 1927 public static boolean ALLOW_ZIPLEVEL_CHANGE=true; 1928 1929 public static final String FILESEP=System.getProperty("file.separator"); 1930 1931 private static final String diskSync=new String("DISKSYNC"); 1932 1933 public static final HashSet<String> loadedFiles=new HashSet<String>(); 1934 1935 private static final String[] compressedExtensions=new String[] {".gz", ".gzip", ".zip", ".bz2", ".xz", ".dsrc", ".fqz", ".ac"}; 1936 private static final String[] compressedExtensionMap=new String[] {"gz", "gz", "zip", "bz2", "xz", "dsrc", "fqz", "ac"}; 1937 1938 // private static HashMap<String, Process> inputProcesses=new HashMap<String, Process>(8); 1939 // private static HashMap<String, Process> outputProcesses=new HashMap<String, Process>(8); 1940 private static HashMap<String, Process> processMap=new HashMap<String, Process>(8); 1941 private static HashMap<String, ArrayList<PipeThread>> pipeThreadMap=new HashMap<String, ArrayList<PipeThread>>(8); 1942 1943 } 1944