1 package org.coolreader.crengine; 2 3 import android.annotation.SuppressLint; 4 5 import org.coolreader.CoolReader; 6 import org.coolreader.crengine.Engine.DelayedProgress; 7 import org.xml.sax.Attributes; 8 import org.xml.sax.SAXException; 9 import org.xml.sax.helpers.DefaultHandler; 10 11 import java.io.ByteArrayInputStream; 12 import java.io.File; 13 import java.io.FileOutputStream; 14 import java.io.IOException; 15 import java.io.InputStream; 16 import java.net.Authenticator; 17 import java.net.HttpURLConnection; 18 import java.net.InetSocketAddress; 19 import java.net.PasswordAuthentication; 20 import java.net.Proxy; 21 import java.net.URL; 22 import java.net.URLConnection; 23 import java.security.cert.X509Certificate; 24 import java.text.ParseException; 25 import java.text.SimpleDateFormat; 26 import java.util.ArrayList; 27 import java.util.Collection; 28 import java.util.HashSet; 29 import java.util.Stack; 30 import java.util.concurrent.Callable; 31 32 import javax.net.ssl.HttpsURLConnection; 33 import javax.net.ssl.SSLContext; 34 import javax.net.ssl.TrustManager; 35 import javax.net.ssl.X509TrustManager; 36 import javax.xml.parsers.SAXParser; 37 import javax.xml.parsers.SAXParserFactory; 38 39 @SuppressLint("SimpleDateFormat") 40 public class OPDSUtil { 41 42 public static final boolean EXTENDED_LOG = false; // set to false for production 43 public static final int CONNECT_TIMEOUT = 60000; 44 public static final int READ_TIMEOUT = 60000; 45 /* 46 <?xml version="1.0" encoding="utf-8"?> 47 <feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:relevance="http://a9.com/-/opensearch/extensions/relevance/1.0/" 48 xmlns="http://www.w3.org/2005/Atom" 49 xml:base="http://lib.ololo.cc/opds/"> 50 <id>http://lib.ololo.cc/opds/</id> 51 <updated>2011-05-31T10:28:22+04:00</updated> 52 <title>OPDS: lib.ololo.cc</title> 53 <subtitle>Librusec mirror.</subtitle> 54 <author> 55 <name>ololo team</name> 56 <uri>http://lib.ololo.cc</uri><email>libololo@gmail.com</email> 57 </author> 58 <icon>http://lib.ololo.cc/book.png</icon> 59 <link rel="self" title="This Page" type="application/atom+xml" href="/opds/"/> 60 <link rel="alternate" type="text/html" title="HTML Page" href="/"/> 61 <entry> 62 <updated>2011-05-31T10:28:22+04:00</updated> 63 <id>http://lib.ololo.cc/opds/asearch/</id> 64 <title>sample type</title> 65 <content type="text">sample content</content> 66 <link type="application/atom+xml" href="http://lib.ololo.cc/opds/asearch/"/> 67 </entry> 68 </feed> 69 */ 70 /** 71 * Callback interface for OPDS. 72 */ 73 public interface DownloadCallback { 74 /** 75 * Some entries are downloaded. 76 * @param doc is document 77 * @param entries is list of entries to add 78 */ onEntries( DocInfo doc, Collection<EntryInfo> entries )79 public boolean onEntries( DocInfo doc, Collection<EntryInfo> entries ); 80 /** 81 * All entries are downloaded. 82 * @param doc is document 83 * @param entries is list of entries to add 84 */ onFinish( DocInfo doc, Collection<EntryInfo> entries )85 public boolean onFinish( DocInfo doc, Collection<EntryInfo> entries ); 86 /** 87 * Before download: request filename to save as. 88 */ onDownloadStart( String type, String url )89 public File onDownloadStart( String type, String url ); 90 /** 91 * Download progress 92 */ onDownloadProgress( String type, String url, int percent )93 public void onDownloadProgress( String type, String url, int percent ); 94 /** 95 * Book is downloaded. 96 */ onDownloadEnd( String type, String url, File file )97 public void onDownloadEnd( String type, String url, File file ); 98 /** 99 * Error occured 100 */ onError( String message )101 public void onError( String message ); 102 } 103 104 public static class DocInfo { 105 public String id; 106 public long updated; 107 public String title; 108 public String subtitle; 109 public String icon; 110 public String language; 111 public LinkInfo selfLink; 112 public LinkInfo alternateLink; 113 public LinkInfo nextLink; 114 } 115 dirPath(String filePath)116 public static String dirPath(String filePath) { 117 int pos = filePath.lastIndexOf("/"); 118 if (pos < 0) 119 return filePath; 120 return filePath.substring(0, pos+1); 121 } 122 123 public static class LinkInfo { 124 public String href; 125 public String rel; 126 public String title; 127 public String type; LinkInfo( URL baseURL, Attributes attributes )128 public LinkInfo( URL baseURL, Attributes attributes ) { 129 rel = attributes.getValue("rel"); 130 type = attributes.getValue("type"); 131 title = attributes.getValue("title"); 132 href = convertHref( baseURL, attributes.getValue("href") ); 133 } convertHref( URL baseURL, String href )134 public static String convertHref( URL baseURL, String href ) { 135 if ( href==null ) 136 return href; 137 String port = ""; 138 if (baseURL.getPort() != 80 && baseURL.getPort() > 0) 139 port = ":" + baseURL.getPort(); 140 String hostPort = baseURL.getHost() + port; 141 if ( href.startsWith("//") ) 142 return baseURL.getProtocol() + ":" + href; 143 if ( href.startsWith("/") ) 144 return baseURL.getProtocol() + "://" + hostPort + href; 145 if ( !href.startsWith("http://") && !href.startsWith("https://") ) { 146 return baseURL.getProtocol() + "://" + hostPort + dirPath(baseURL.getPath()) + "/" + href; 147 } 148 return href; 149 } isValid()150 public boolean isValid() { 151 return href!=null && href.length()!=0; 152 } getPriority()153 public int getPriority() { 154 if ( type==null ) 155 return 0; 156 DocumentFormat df = DocumentFormat.byMimeType(type); 157 if ( rel!=null && rel.indexOf("acquisition")<0 && df!=DocumentFormat.FB2 && df!=DocumentFormat.EPUB 158 && df!=DocumentFormat.RTF && df!=DocumentFormat.DOC) 159 return 0; 160 return df!=null ? df.getPriority() : 0; 161 } 162 @Override toString()163 public String toString() { 164 return "[ rel=" + rel + ", type=" + type 165 + ", title=" + title + ", href=" + href + "]"; 166 } 167 168 } 169 170 public static class AuthorInfo { 171 public String name; 172 public String uri; 173 } 174 175 public static class EntryInfo { 176 public String id; 177 public long updated; 178 public String title=""; 179 public String content=""; 180 public String summary=""; 181 public LinkInfo link; 182 public ArrayList<LinkInfo> links = new ArrayList<LinkInfo>(); 183 public String icon; 184 public ArrayList<String> categories = new ArrayList<String>(); 185 public ArrayList<AuthorInfo> authors = new ArrayList<AuthorInfo>(); getBestAcquisitionLink()186 public LinkInfo getBestAcquisitionLink() { 187 LinkInfo best = null; 188 int bestPriority = 0; 189 for ( LinkInfo link : links ) { 190 //boolean isAcquisition = link.rel!=null && link.rel.indexOf("acquisition")>=0; 191 int priority = link.getPriority(); 192 if (priority>0 && priority>bestPriority) { 193 if ( link.getPriority()>0 && (best==null || best.getPriority()<link.getPriority()) ) { 194 best = link; 195 bestPriority = priority; 196 } 197 } 198 } 199 return best; 200 } getAuthors()201 public String getAuthors() { 202 if ( authors.size()==0 ) 203 return null; 204 StringBuilder buf = new StringBuilder(100); 205 for ( AuthorInfo a : authors ) { 206 if ( buf.length()>0 ) 207 buf.append(", "); 208 buf.append(a.name); 209 } 210 return buf.toString(); 211 } 212 } 213 214 public static class OPDSHandler extends DefaultHandler { 215 private URL url; 216 private DocInfo docInfo = new DocInfo(); 217 private EntryInfo entryInfo = new EntryInfo(); 218 private ArrayList<EntryInfo> entries = new ArrayList<EntryInfo>(); 219 private Stack<String> elements = new Stack<String>(); 220 //private Attributes currentAttributes; 221 private AuthorInfo authorInfo; 222 private boolean insideFeed; 223 private boolean insideEntry; 224 private boolean insideEntryTitle; 225 //private boolean singleEntry; 226 private int level = 0; 227 //2011-05-31T10:28:22+04:00 228 private static SimpleDateFormat tsFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); 229 private static SimpleDateFormat tsFormat2 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); OPDSHandler( URL url )230 public OPDSHandler( URL url ) { 231 this.url = url; 232 } setUrl(URL url)233 public void setUrl(URL url) { 234 this.url = url; 235 } parseTimestamp( String ts )236 private long parseTimestamp( String ts ) { 237 if ( ts==null ) 238 return 0; 239 ts = ts.trim(); 240 try { 241 if ( ts.length()=="2010-01-10T10:01:10Z".length() ) 242 return tsFormat2.parse(ts).getTime(); 243 if ( ts.length()=="2011-11-11T11:11:11+67:87".length()&& ts.lastIndexOf(":")==ts.length()-3 ) { 244 ts = ts.substring(0, ts.length()-3) + ts.substring(0, ts.length()-2); 245 return tsFormat.parse(ts).getTime(); 246 } 247 if ( ts.length()=="2011-11-11T11:11:11+6787".length()) { 248 return tsFormat.parse(ts).getTime(); 249 } 250 } catch (ParseException e) { 251 } 252 L.e("cannot parse timestamp " + ts); 253 return 0; 254 } 255 256 @Override characters(char[] ch, int start, int length)257 public void characters(char[] ch, int start, int length) 258 throws SAXException { 259 super.characters(ch, start, length); 260 261 String s = new String( ch, start, length); 262 s = s.trim(); 263 if (s.length()==0 || (s.length()==1 && s.charAt(0) == '\n') ) 264 return; // ignore empty line 265 //L.d(tab() + " {" + s + "}"); 266 String currentElement = elements.peek(); 267 if ( currentElement==null ) 268 return; 269 if ( insideFeed ) { 270 if ( "id".equals(currentElement) ) { 271 if ( insideEntry ) 272 entryInfo.id = s; 273 else 274 docInfo.id = s; 275 } else if ( "updated".equals(currentElement) ) { 276 long ts = parseTimestamp(s); 277 if ( insideEntry ) 278 entryInfo.updated = ts; 279 else 280 docInfo.updated = ts; 281 } else if ( "title".equals(currentElement) ) { 282 if ( !insideEntry ) { 283 docInfo.title = s; 284 } else { 285 entryInfo.title = entryInfo.title + s; 286 } 287 } else if ( "summary".equals(currentElement) ) { 288 if ( insideEntry ) 289 entryInfo.summary = entryInfo.summary + s; 290 } else if ( "name".equals(currentElement) ) { 291 if ( authorInfo != null ) 292 authorInfo.name = s; 293 } else if ( "uri".equals(currentElement) ) { 294 if ( authorInfo!=null ) 295 authorInfo.uri = s; 296 } else if ( "icon".equals(currentElement) ) { 297 if ( !insideEntry ) 298 docInfo.icon = s; 299 else 300 entryInfo.icon = s; 301 } else if ( "link".equals(currentElement) ) { 302 // rel, type, title, href 303 // if ( !insideEntry ) 304 // docInfo.icon = s; 305 // else 306 // entryInfo.icon = s; 307 } else if ( "content".equals(currentElement) ) { 308 if ( insideEntry ) 309 entryInfo.content = entryInfo.content + s; 310 } else if ( "subtitle".equals(currentElement) ) { 311 if ( !insideEntry ) 312 docInfo.subtitle = s; 313 } else if ( "language".equals(currentElement) ) { 314 if ( !insideEntry ) 315 docInfo.language = s; 316 } else if ( insideEntryTitle ) { 317 if (entryInfo.title.length() > 0) 318 entryInfo.title = entryInfo.title + " "; 319 entryInfo.title = entryInfo.title + s; 320 } 321 } 322 } 323 324 @Override endDocument()325 public void endDocument() throws SAXException { 326 super.endDocument(); 327 if (EXTENDED_LOG) L.d("endDocument: " + entries.size() + " entries parsed"); 328 if (EXTENDED_LOG) 329 for ( EntryInfo entry : entries ) { 330 L.d(" " + entry.title + " : " + entry.link.toString()); 331 } 332 } 333 tab()334 private String tab() { 335 if ( level<=1 ) 336 return ""; 337 StringBuffer buf = new StringBuffer(level*2); 338 for ( int i=1; i<level; i++ ) 339 buf.append(" "); 340 return buf.toString(); 341 } 342 343 @Override startElement(String uri, String localName, String qName, Attributes attributes)344 public void startElement(String uri, String localName, 345 String qName, Attributes attributes) 346 throws SAXException { 347 super.startElement(uri, localName, qName, attributes); 348 if ( qName!=null && qName.length()>0 ) 349 localName = qName; 350 level++; 351 //L.d(tab() + "<" + localName + ">"); 352 //currentAttributes = attributes; 353 elements.push(localName); 354 //String currentElement = elements.peek(); 355 if ( !insideFeed && "feed".equals(localName) ) { 356 insideFeed = true; 357 } else if ( "entry".equals(localName) ) { 358 if ( !insideFeed ) { 359 insideFeed = true; 360 //singleEntry = true; 361 } 362 insideEntry = true; 363 entryInfo = new EntryInfo(); 364 } else if ( "category".equals(localName) ) { 365 if ( insideEntry ) { 366 String category = attributes.getValue("label"); 367 if ( category!=null ) 368 entryInfo.categories.add(category); 369 } 370 } else if ( "id".equals(localName) ) { 371 372 } else if ( "updated".equals(localName) ) { 373 374 } else if ( "title".equals(localName) ) { 375 insideEntryTitle = insideEntry; 376 } else if ( "name".equals(localName) ) { 377 378 } else if ( "link".equals(localName) ) { 379 LinkInfo link = new LinkInfo(url, attributes); 380 if ( link.isValid() && insideFeed ) { 381 if (EXTENDED_LOG) L.d(tab()+link.toString()); 382 if ( insideEntry ) { 383 if ( link.type!=null ) { 384 entryInfo.links.add(link); 385 int priority = link.getPriority(); 386 if ( link.type.startsWith("application/atom+xml") ) { 387 if (entryInfo.link == null || !entryInfo.link.type.startsWith("application/atom+xml")) 388 entryInfo.link = link; 389 } else if ( "http://opds-spec.org/cover".equals(link.rel) && "image/jpeg".equals(link.type)) { 390 entryInfo.icon = link.href; 391 } else if ( "http://opds-spec.org/thumbnail".equals(link.rel) && "image/jpeg".equals(link.type)) { 392 if (entryInfo.icon == null) 393 entryInfo.icon = link.href; 394 } else if (priority>0 && (entryInfo.link==null || entryInfo.link.getPriority()<priority)) { 395 entryInfo.link = link; 396 } 397 } 398 } else { 399 if ( "self".equals(link.rel) ) 400 docInfo.selfLink = link; 401 else if ( "alternate".equals(link.rel) ) 402 docInfo.alternateLink = link; 403 else if ( "next".equals(link.rel) ) 404 docInfo.nextLink = link; 405 } 406 } 407 } else if ( "author".equals(localName) ) { 408 authorInfo = new AuthorInfo(); 409 } 410 } 411 412 @Override endElement(String uri, String localName, String qName)413 public void endElement(String uri, String localName, 414 String qName) throws SAXException { 415 super.endElement(uri, localName, qName); 416 if ( qName!=null && qName.length()>0 ) 417 localName = qName; 418 //L.d(tab() + "</" + localName + ">"); 419 //String currentElement = elements.peek(); 420 if ( insideFeed && "feed".equals(localName) ) { 421 insideFeed = false; 422 } else if ( "title".equals(localName) ) { 423 insideEntryTitle = false; 424 } else if ( "entry".equals(localName) ) { 425 if ( !insideFeed || !insideEntry ) 426 throw new SAXException("unexpected element " + localName); 427 if ( entryInfo.link!=null || entryInfo.getBestAcquisitionLink()!=null ) { 428 entries.add(entryInfo); 429 } 430 insideEntry = false; 431 entryInfo = null; 432 } else if ( "author".equals(localName) ) { 433 if (insideEntry) { 434 if ( authorInfo!=null && authorInfo.name!=null ) 435 entryInfo.authors.add(authorInfo); 436 } 437 authorInfo = null; 438 } 439 //currentAttributes = null; 440 if ( level>0 ) 441 level--; 442 } 443 444 @Override startDocument()445 public void startDocument() throws SAXException { 446 // TODO Auto-generated method stub 447 super.startDocument(); 448 } 449 450 } 451 452 public static class DownloadTask { 453 final private CoolReader coolReader; 454 private URL url; 455 private String username; 456 private String password; 457 final private String expectedType; 458 final private String referer; 459 final private String defaultFileName; 460 final private DownloadCallback callback; 461 private String progressMessage = "Dowloading..."; 462 private HttpURLConnection connection; 463 private DelayedProgress delayedProgress; 464 OPDSHandler handler; DownloadTask(CoolReader coolReader, URL url, String defaultFileName, String expectedType, String referer, DownloadCallback callback, String username, String password)465 public DownloadTask(CoolReader coolReader, URL url, String defaultFileName, String expectedType, String referer, DownloadCallback callback, String username, String password) { 466 this.url = url; 467 this.coolReader = coolReader; 468 this.callback = callback; 469 this.referer = referer; 470 this.expectedType = expectedType; 471 this.defaultFileName = defaultFileName; 472 this.username = username; 473 this.password = password; 474 L.d("Created DownloadTask for " + url); 475 } setProgressMessage( String url, int totalSize )476 private void setProgressMessage( String url, int totalSize ) { 477 progressMessage = coolReader.getString(org.coolreader.R.string.progress_downloading) + " " + url; 478 if ( totalSize>0 ) 479 progressMessage = progressMessage + " (" + totalSize + ")"; 480 } 481 // call in GUI thread only! hideProgress()482 private void hideProgress() { 483 if ( progressShown && Services.getEngine() != null) 484 Services.getEngine().hideProgress(); 485 if ( delayedProgress != null ) { 486 delayedProgress.cancel(); 487 delayedProgress.hide(); 488 } 489 } onError(final String msg)490 private void onError(final String msg) { 491 BackgroundThread.instance().executeGUI(() -> { 492 hideProgress(); 493 callback.onError(msg); 494 }); 495 } parseFeed( InputStream is )496 private void parseFeed( InputStream is ) throws Exception { 497 try { 498 if (handler==null) 499 handler = new OPDSHandler(url); 500 else 501 handler.setUrl(url); // download next part 502 String[] namespaces = new String[] { 503 "access", "http://www.bloglines.com/about/specs/fac-1.0", 504 "admin", "http://webns.net/mvcb/", 505 "ag", "http://purl.org/rss/1.0/modules/aggregation/", 506 "annotate", "http://purl.org/rss/1.0/modules/annotate/", 507 "app", "http://www.w3.org/2007/app", 508 "atom", "http://www.w3.org/2005/Atom", 509 "audio", "http://media.tangent.org/rss/1.0/", 510 "blogChannel", "http://backend.userland.com/blogChannelModule", 511 "cc", "http://web.resource.org/cc/", 512 "cf", "http://www.microsoft.com/schemas/rss/core/2005", 513 "company", "http://purl.org/rss/1.0/modules/company", 514 "content", "http://purl.org/rss/1.0/modules/content/", 515 "conversationsNetwork", "http://conversationsnetwork.org/rssNamespace-1.0/", 516 "cp", "http://my.theinfo.org/changed/1.0/rss/", 517 "creativeCommons", "http://backend.userland.com/creativeCommonsRssModule", 518 "dc", "http://purl.org/dc/elements/1.1/", 519 "dcterms", "http://purl.org/dc/terms/", 520 "email", "http://purl.org/rss/1.0/modules/email/", 521 "ev", "http://purl.org/rss/1.0/modules/event/", 522 "feedburner", "http://rssnamespace.org/feedburner/ext/1.0", 523 "fh", "http://purl.org/syndication/history/1.0", 524 "foaf", "http://xmlns.com/foaf/0.1/", 525 "foaf", "http://xmlns.com/foaf/0.1", 526 "geo", "http://www.w3.org/2003/01/geo/wgs84_pos#", 527 "georss", "http://www.georss.org/georss", 528 "geourl", "http://geourl.org/rss/module/", 529 "g", "http://base.google.com/ns/1.0", 530 "gml", "http://www.opengis.net/gml", 531 "icbm", "http://postneo.com/icbm", 532 "image", "http://purl.org/rss/1.0/modules/image/", 533 "indexing", "urn:atom-extension:indexing", 534 "itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd", 535 "kml20", "http://earth.google.com/kml/2.0", 536 "kml21", "http://earth.google.com/kml/2.1", 537 "kml22", "http://www.opengis.net/kml/2.2", 538 "l", "http://purl.org/rss/1.0/modules/link/", 539 "mathml", "http://www.w3.org/1998/Math/MathML", 540 "media", "http://search.yahoo.com/mrss/", 541 "openid", "http://openid.net/xmlns/1.0", 542 "opensearch10", "http://a9.com/-/spec/opensearchrss/1.0/", 543 "opensearch", "http://a9.com/-/spec/opensearch/1.1/", 544 "opml", "http://www.opml.org/spec2", 545 "rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 546 "rdfs", "http://www.w3.org/2000/01/rdf-schema#", 547 "ref", "http://purl.org/rss/1.0/modules/reference/", 548 "reqv", "http://purl.org/rss/1.0/modules/richequiv/", 549 "rss090", "http://my.netscape.com/rdf/simple/0.9/", 550 "rss091", "http://purl.org/rss/1.0/modules/rss091#", 551 "rss1", "http://purl.org/rss/1.0/", 552 "rss11", "http://purl.org/net/rss1.1#", 553 "search", "http://purl.org/rss/1.0/modules/search/", 554 "slash", "http://purl.org/rss/1.0/modules/slash/", 555 "ss", "http://purl.org/rss/1.0/modules/servicestatus/", 556 "str", "http://hacks.benhammersley.com/rss/streaming/", 557 "sub", "http://purl.org/rss/1.0/modules/subscription/", 558 "svg", "http://www.w3.org/2000/svg", 559 "sx", "http://feedsync.org/2007/feedsync", 560 "sy", "http://purl.org/rss/1.0/modules/syndication/", 561 "taxo", "http://purl.org/rss/1.0/modules/taxonomy/", 562 "thr", "http://purl.org/rss/1.0/modules/threading/", 563 "thr", "http://purl.org/syndication/thread/1.0", 564 "trackback", "http://madskills.com/public/xml/rss/module/trackback/", 565 "wfw", "http://wellformedweb.org/CommentAPI/", 566 "wiki", "http://purl.org/rss/1.0/modules/wiki/", 567 "xhtml", "http://www.w3.org/1999/xhtml", 568 "xlink", "http://www.w3.org/1999/xlink", 569 "xrd", "xri://$xrd*($v*2.0)", 570 "xrds", "xri://$xrds" 571 }; 572 for ( int i=0; i<namespaces.length-1; i+=2 ) 573 handler.startPrefixMapping(namespaces[i], namespaces[i+1]); 574 SAXParserFactory spf = SAXParserFactory.newInstance(); 575 spf.setValidating(false); 576 // spf.setNamespaceAware(true); 577 // spf.setFeature("http://xml.org/sax/features/namespaces", false); 578 SAXParser sp = spf.newSAXParser(); 579 //XMLReader xr = sp.getXMLReader(); 580 sp.parse(is, handler); 581 } catch (SAXException se) { 582 L.e("sax error", se); 583 throw se; 584 } catch (IOException ioe) { 585 L.e("sax parse io error", ioe); 586 throw ioe; 587 } 588 } 589 generateFileName( File outDir, String fileName, String type, boolean isZip )590 private File generateFileName( File outDir, String fileName, String type, boolean isZip ) { 591 DocumentFormat fmt = type!=null ? DocumentFormat.byMimeType(type) : null; 592 //DocumentFormat fmtext = fileName!=null ? DocumentFormat.byExtension(fileName) : null; 593 if ( fileName==null ) 594 fileName = "noname"; 595 String ext = null; 596 if ( fileName.lastIndexOf(".")>0 ) { 597 ext = fileName.substring(fileName.lastIndexOf(".")+1); 598 fileName = fileName.substring(0, fileName.lastIndexOf(".")); 599 } 600 fileName = Utils.transcribeFileName( fileName ); 601 if ( fmt!=null ) { 602 if ( fmt==DocumentFormat.FB2 && isZip ) 603 ext = ".fb2.zip"; 604 else 605 ext = fmt.getExtensions()[0].substring(1); 606 } 607 for (int i=0; i<1000; i++ ) { 608 String fn = fileName + (i==0 ? "" : "(" + i + ")") + "." + ext; 609 File f = new File(outDir, fn); 610 if ( !f.exists() && !f.isDirectory() ) 611 return f; 612 } 613 return null; 614 } downloadBook( final String type, final String url, InputStream is, int contentLength, final String fileName, final boolean isZip )615 private void downloadBook( final String type, final String url, InputStream is, int contentLength, final String fileName, final boolean isZip ) throws Exception { 616 L.d("Download requested: " + type + " " + url + " " + contentLength); 617 DocumentFormat fmt = DocumentFormat.byMimeType(type); 618 if ( fmt==null ) { 619 L.d("Download: unknown type " + type); 620 throw new Exception("Unknown file type " + type); 621 } 622 final File outDir = BackgroundThread.instance().callGUI(new Callable<File>() { 623 @Override 624 public File call() throws Exception { 625 return callback.onDownloadStart(type, url); 626 } 627 }); 628 if ( outDir==null ) { 629 L.d("Cannot find writable location for downloaded file " + url); 630 throw new Exception("Cannot save file " + url); 631 } 632 final File outFile = generateFileName( outDir, fileName, type, isZip ); 633 if ( outFile==null ) { 634 L.d("Cannot generate file name"); 635 throw new Exception("Cannot generate file name"); 636 } 637 L.d("Creating file: " + outFile.getAbsolutePath()); 638 if ( outFile.exists() || !outFile.createNewFile() ) { 639 L.d("Cannot create file " + outFile.getAbsolutePath()); 640 throw new Exception("Cannot create file"); 641 } 642 643 L.d("Download started: " + outFile.getAbsolutePath()); 644 // long lastTs = System.currentTimeMillis(); 645 // int lastPercent = -1; 646 boolean success = false; 647 try (FileOutputStream os = new FileOutputStream(outFile)) { 648 byte[] buf = new byte[16384]; 649 int totalWritten = 0; 650 while (totalWritten<contentLength || contentLength==-1) { 651 int bytesRead = is.read(buf); 652 if ( bytesRead<=0 ) 653 break; 654 os.write(buf, 0, bytesRead); 655 totalWritten += bytesRead; 656 // final int percent = totalWritten * 100 / contentLength; 657 // long ts = System.currentTimeMillis(); 658 // if ( percent!=lastPercent && ts - lastTs > 1500 ) { 659 // L.d("Download progress: " + percent + "%"); 660 // BackgroundThread.instance().postGUI(new Runnable() { 661 // @Override 662 // public void run() { 663 // callback.onDownloadProgress(type, url, percent); 664 // } 665 // }); 666 // } 667 } 668 success = true; 669 } finally { 670 if ( !success ) { 671 if ( outFile.exists() && outFile.isFile() ) { 672 L.w("deleting unsuccessully downloaded file " + outFile); 673 outFile.delete(); 674 } 675 } 676 } 677 L.d("Download finished"); 678 BackgroundThread.instance().executeGUI(() -> callback.onDownloadEnd(type, url, outFile)); 679 } findSubstring( byte[]buf, String str )680 public static int findSubstring( byte[]buf, String str ) { 681 for ( int i=0; i<buf.length-str.length(); i++ ) { 682 boolean found = true; 683 for ( int j=0; j<str.length(); j++ ) 684 if ( str.charAt(j)!=buf[i+j] ) { 685 found = false; 686 break; 687 } 688 if ( found ) 689 return i; 690 } 691 return -1; // not found 692 } 693 encodePassword(String username, String password)694 public static String encodePassword(String username, String password) { 695 return Base64.encodeToString((username + ":" + password).getBytes(), Base64.NO_WRAP); 696 } 697 runInternal()698 public void runInternal() { 699 connection = null; 700 701 boolean itemsLoadedPartially = false; 702 boolean loadNext = false; 703 HashSet<String> visited = new HashSet<String>(); 704 705 do { 706 try { 707 setProgressMessage( url.toString(), -1 ); 708 visited.add(url.toString()); 709 long startTimeStamp = System.currentTimeMillis(); 710 if (!partialDownloadCompleted) { 711 if (delayedProgress != null) 712 delayedProgress.cancel(); 713 delayedProgress = Services.getEngine().showProgressDelayed(0, progressMessage, PROGRESS_DELAY_MILLIS); 714 } 715 URL newURL = url; 716 boolean useOrobotProxy = false; 717 String host = url.getHost(); 718 if (host.endsWith(".onion")) 719 useOrobotProxy = true; 720 String oldAddress = url.toString(); 721 if (oldAddress.startsWith("orobot://")) { 722 newURL = new URL("http://" + oldAddress.substring(9)); // skip orobot:// 723 useOrobotProxy = true; 724 L.d("Converting url - " + oldAddress + " to " + newURL + " for using ORobot proxy"); 725 } else if (oldAddress.startsWith("orobots://")) { 726 newURL = new URL("https://" + oldAddress.substring(10)); // skip orobots:// 727 useOrobotProxy = true; 728 L.d("Converting url - " + oldAddress + " to " + newURL + " for using ORobot proxy"); 729 } 730 Proxy proxy = null; 731 System.setProperty("http.keepAlive", "false"); 732 if (useOrobotProxy) { 733 // Set-up proxy 734 //System.setProperty("http.proxyHost", "127.0.0.1"); 735 //System.setProperty("http.proxyPort", "8118"); 736 //L.d("Using ORobot proxy: " + proxy); 737 proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8118)); // ORobot proxy running on this device 738 L.d("Using ORobot proxy: " + proxy); 739 } else { 740 //System.clearProperty("http.proxyHost"); 741 //System.clearProperty("http.proxyPort"); 742 } 743 744 URLConnection conn = proxy == null ? newURL.openConnection() : newURL.openConnection(proxy); 745 if ( conn instanceof HttpsURLConnection ) { 746 HttpsURLConnection https = (HttpsURLConnection)conn; 747 748 // Create a trust manager that does not validate certificate chains 749 TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { 750 public java.security.cert.X509Certificate[] getAcceptedIssuers() { 751 return null; 752 } 753 public void checkClientTrusted(X509Certificate[] certs, String authType) { 754 } 755 public void checkServerTrusted(X509Certificate[] certs, String authType) { 756 } 757 } }; 758 // Install the all-trusting trust manager 759 final SSLContext sc = SSLContext.getInstance("SSL"); 760 sc.init(null, trustAllCerts, new java.security.SecureRandom()); 761 HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); 762 763 https.setHostnameVerifier((arg0, arg1) -> true); 764 } 765 if ( !(conn instanceof HttpURLConnection) ) { 766 onError("Only HTTP supported"); 767 return; 768 } 769 connection = (HttpURLConnection)conn; 770 connection.setRequestProperty("User-Agent", "CoolReader/3(Android)"); 771 if ( referer!=null ) 772 connection.setRequestProperty("Referer", referer); 773 connection.setInstanceFollowRedirects(true); 774 connection.setUseCaches(false); 775 776 if (username != null && username.length() > 0 && password != null && password.length() > 0) { 777 connection.setRequestProperty("Authorization", encodePassword(username, password)); 778 Authenticator.setDefault(new Authenticator() { 779 protected PasswordAuthentication getPasswordAuthentication() { 780 return new PasswordAuthentication(username, password.toCharArray()); 781 }}); 782 } 783 784 connection.setAllowUserInteraction(false); 785 connection.setConnectTimeout(CONNECT_TIMEOUT); 786 connection.setReadTimeout(READ_TIMEOUT); 787 connection.setDoInput(true); 788 String fileName = null; 789 String disp = connection.getHeaderField("Content-Disposition"); 790 if ( disp!=null ) { 791 int p = disp.indexOf("filename="); 792 if ( p>0 ) { 793 fileName = disp.substring(p + 9); 794 } 795 } 796 //connection.setDoOutput(true); 797 //connection.set 798 799 int response = -1; 800 801 response = connection.getResponseCode(); 802 if (EXTENDED_LOG) L.d("Response: " + response); 803 if ( response == 301 || response == 302 || response == 307 || response == 303 ) { 804 // redirects 805 String redirect = connection.getHeaderField("Location"); 806 if (null == redirect) { 807 onError("Invalid redirect " + response); 808 return; 809 } 810 L.d("continue with next part: " + url); 811 url = new URL(redirect); 812 if (visited.contains(url.toString())) { 813 onError("Duplicate redirect " + url); 814 return; 815 } 816 loadNext = true; 817 L.d("Response " + response + ": redirect to " + url); 818 continue; 819 } 820 if ( response != 200 ) { 821 onError("Error " + response); 822 return; 823 } 824 825 if (cancelled) 826 break; 827 828 String contentType = connection.getContentType(); 829 String contentEncoding = connection.getContentEncoding(); 830 int contentLen = connection.getContentLength(); 831 //connection.getC 832 if (EXTENDED_LOG) L.d("Entity content length: " + contentLen); 833 if (EXTENDED_LOG) L.d("Entity content type: " + contentType); 834 if (EXTENDED_LOG) L.d("Entity content encoding: " + contentEncoding); 835 setProgressMessage( url.toString(), contentLen ); 836 InputStream is = connection.getInputStream(); 837 if (delayedProgress != null) 838 delayedProgress.cancel(); 839 is = new ProgressInputStream(is, startTimeStamp, progressMessage, contentLen, 80); 840 final int MAX_CONTENT_LEN_TO_BUFFER = 256*1024; 841 boolean isZip = contentType!=null && contentType.equals("application/zip"); 842 if ( expectedType!=null ) 843 contentType = expectedType; 844 else if ( contentLen>0 && contentLen<MAX_CONTENT_LEN_TO_BUFFER) { // autodetect type 845 byte[] buf = new byte[contentLen]; 846 if ( is.read(buf)!=contentLen ) { 847 onError("Wrong content length"); 848 return; 849 } 850 is.close(); 851 is = null; 852 is = new ByteArrayInputStream(buf); 853 if ( findSubstring(buf, "<?xml version=")>=0 && findSubstring(buf, "<feed")>=0 ) 854 contentType = "application/atom+xml"; // override type 855 } 856 if ( contentType.startsWith("application/atom+xml") ) { 857 if (EXTENDED_LOG) L.d("Parsing feed"); 858 parseFeed( is ); 859 itemsLoadedPartially = true; 860 if (handler.docInfo.nextLink!=null && handler.docInfo.nextLink.type.startsWith("application/atom+xml;profile=opds-catalog")) { 861 if (handler.entries.size() < MAX_OPDS_ITEMS) { 862 url = new URL(handler.docInfo.nextLink.href); 863 loadNext = !visited.contains(url.toString()); 864 L.d("continue with next part: " + url); 865 } else { 866 L.d("max item count reached: " + handler.entries.size()); 867 loadNext = false; 868 } 869 } else { 870 loadNext = false; 871 } 872 873 } else { 874 if ( fileName==null ) 875 fileName = defaultFileName; 876 L.d("Downloading book: " + contentEncoding); 877 downloadBook( contentType, url.toString(), is, contentLen, fileName, isZip ); 878 hideProgress(); 879 loadNext = false; 880 itemsLoadedPartially = false; 881 } 882 } catch (Exception e) { 883 L.e("Exception while trying to open URI " + url.toString(), e); 884 if ( progressShown ) 885 Services.getEngine().hideProgress(); 886 onError("Error occured while reading OPDS catalog"); 887 break; 888 } finally { 889 if ( connection!=null ) 890 try { 891 connection.disconnect(); 892 } catch ( Exception e ) { 893 // ignore 894 } 895 } 896 897 partialDownloadCompleted = true; // don't show progress 898 899 if (loadNext && !cancelled) { 900 // partially loaded 901 if ( progressShown ) 902 Services.getEngine().hideProgress(); 903 final ArrayList<EntryInfo> entries = new ArrayList<>(handler.entries); 904 BackgroundThread.instance().executeGUI(() -> { 905 L.d("Parsing is partially. " + handler.entries.size() + " entries found -- updating view"); 906 if (!callback.onEntries(handler.docInfo, entries)) 907 cancel(); 908 }); 909 } 910 } while (loadNext && !cancelled); 911 if (delayedProgress != null) 912 delayedProgress.cancel(); 913 hideProgress(); 914 if (itemsLoadedPartially && !cancelled) { 915 BackgroundThread.instance().executeGUI(() -> { 916 L.d("Parsing is finished successfully. " + handler.entries.size() + " entries found"); 917 hideProgress(); 918 if (!callback.onFinish(handler.docInfo, handler.entries)) 919 cancel(); 920 }); 921 } 922 } 923 run()924 public void run() { 925 BackgroundThread.instance().postBackground(() -> { 926 try { 927 runInternal(); 928 } catch ( Exception e ) { 929 L.e("exception while opening OPDS", e); 930 } 931 }); 932 } 933 cancel()934 public void cancel() { 935 if (!cancelled) { 936 L.d("cancelling current download task"); 937 cancelled = true; 938 } 939 } 940 941 volatile private boolean cancelled = false; 942 943 private boolean progressShown = false; 944 945 private boolean partialDownloadCompleted = false; 946 947 public class ProgressInputStream extends InputStream { 948 949 private static final int TIMEOUT = 1500; 950 951 private final InputStream sourceStream; 952 private final int totalSize; 953 private final String progressMessage; 954 private long lastUpdate; 955 private int lastPercent; 956 private int maxPercentToStartShowingProgress; 957 private int bytesRead; 958 ProgressInputStream( InputStream sourceStream, long startTimeStamp, String progressMessage, int totalSize, int maxPercentToStartShowingProgress )959 public ProgressInputStream( InputStream sourceStream, long startTimeStamp, String progressMessage, int totalSize, int maxPercentToStartShowingProgress ) { 960 this.sourceStream = sourceStream; 961 this.totalSize = totalSize; 962 this.maxPercentToStartShowingProgress = maxPercentToStartShowingProgress * 100; 963 this.progressMessage = progressMessage; 964 this.lastUpdate = startTimeStamp; 965 this.bytesRead = 0; 966 } 967 updateProgress()968 private void updateProgress() { 969 long ts = System.currentTimeMillis(); 970 long delay = ts - lastUpdate; 971 if ( delay > TIMEOUT ) { 972 lastUpdate = ts; 973 int percent = 0; 974 if ( totalSize>0 ) { 975 percent = bytesRead * 100 / totalSize * 100; 976 } 977 if ( !partialDownloadCompleted && (!progressShown || percent!=lastPercent) && (progressShown || percent<maxPercentToStartShowingProgress || delay > TIMEOUT*2 ) ) { 978 Services.getEngine().showProgress(percent, progressMessage); 979 lastPercent = percent; 980 progressShown = true; 981 } 982 } 983 984 } 985 986 @Override read()987 public int read() throws IOException { 988 bytesRead++; 989 updateProgress(); 990 return sourceStream.read(); 991 } 992 993 @Override close()994 public void close() throws IOException { 995 super.close(); 996 } 997 } 998 999 } 1000 private static DownloadTask currentTask; create(CoolReader coolReader, URL uri, String defaultFileName, String expectedType, String referer, DownloadCallback callback, String username, String password)1001 public static DownloadTask create(CoolReader coolReader, URL uri, String defaultFileName, String expectedType, String referer, DownloadCallback callback, String username, String password) { 1002 if (currentTask != null) 1003 currentTask.cancel(); 1004 final DownloadTask task = new DownloadTask(coolReader, uri, defaultFileName, expectedType, referer, callback, username, password); 1005 currentTask = task; 1006 return task; 1007 } 1008 1009 static class SubstTable { 1010 private final int startChar; 1011 private final String[] replacements; SubstTable( int startChar, String[] replacements )1012 public SubstTable( int startChar, String[] replacements ) { 1013 this.startChar = startChar; 1014 this.replacements = replacements; 1015 } isInRange( char ch )1016 boolean isInRange( char ch ) { 1017 return ch>=startChar && ch<startChar + replacements.length; 1018 } get( char ch )1019 String get( char ch ) { 1020 return (ch>=startChar && ch<startChar + replacements.length) ? replacements[ch - startChar] : ""; 1021 } 1022 } 1023 1024 public static final int PROGRESS_DELAY_MILLIS = 2000; 1025 public static final int MAX_OPDS_ITEMS = 1000; 1026 } 1027
