1 /* 2 * Created on Jul 8, 2009 3 * Created by Paul Gardner 4 * 5 * Copyright (C) Azureus Software, Inc, All Rights Reserved. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 2 10 * of the License, or (at your option) any later version. 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 */ 19 20 21 package com.aelitis.azureus.core.content; 22 23 import java.io.BufferedInputStream; 24 import java.io.ByteArrayInputStream; 25 import java.io.ByteArrayOutputStream; 26 import java.lang.ref.WeakReference; 27 import java.net.URL; 28 import java.util.*; 29 import java.util.concurrent.atomic.AtomicInteger; 30 import java.util.zip.GZIPInputStream; 31 import java.util.zip.GZIPOutputStream; 32 33 import org.gudy.azureus2.core3.config.COConfigurationManager; 34 import org.gudy.azureus2.core3.config.ParameterListener; 35 import org.gudy.azureus2.core3.download.DownloadManagerState; 36 import org.gudy.azureus2.core3.torrent.TOTorrent; 37 import org.gudy.azureus2.core3.torrent.TOTorrentAnnounceURLGroup; 38 import org.gudy.azureus2.core3.torrent.TOTorrentAnnounceURLSet; 39 import org.gudy.azureus2.core3.util.AENetworkClassifier; 40 import org.gudy.azureus2.core3.util.AERunnable; 41 import org.gudy.azureus2.core3.util.AESemaphore; 42 import org.gudy.azureus2.core3.util.AsyncDispatcher; 43 import org.gudy.azureus2.core3.util.BDecoder; 44 import org.gudy.azureus2.core3.util.BEncoder; 45 import org.gudy.azureus2.core3.util.Base32; 46 import org.gudy.azureus2.core3.util.ByteArrayHashMap; 47 import org.gudy.azureus2.core3.util.ByteFormatter; 48 import org.gudy.azureus2.core3.util.Debug; 49 import org.gudy.azureus2.core3.util.FileUtil; 50 import org.gudy.azureus2.core3.util.RandomUtils; 51 import org.gudy.azureus2.core3.util.SHA1Simple; 52 import org.gudy.azureus2.core3.util.SimpleTimer; 53 import org.gudy.azureus2.core3.util.StringInterner; 54 import org.gudy.azureus2.core3.util.SystemTime; 55 import org.gudy.azureus2.core3.util.TimerEvent; 56 import org.gudy.azureus2.core3.util.TimerEventPerformer; 57 import org.gudy.azureus2.core3.util.TorrentUtils; 58 import org.gudy.azureus2.plugins.PluginInterface; 59 import org.gudy.azureus2.plugins.PluginListener; 60 import org.gudy.azureus2.plugins.ddb.DistributedDatabase; 61 import org.gudy.azureus2.plugins.ddb.DistributedDatabaseTransferType; 62 import org.gudy.azureus2.plugins.disk.DiskManagerFileInfo; 63 import org.gudy.azureus2.plugins.download.Download; 64 import org.gudy.azureus2.plugins.download.DownloadManager; 65 import org.gudy.azureus2.plugins.download.DownloadManagerListener; 66 import org.gudy.azureus2.plugins.torrent.Torrent; 67 import org.gudy.azureus2.plugins.torrent.TorrentAttribute; 68 import org.gudy.azureus2.plugins.utils.search.SearchException; 69 import org.gudy.azureus2.plugins.utils.search.SearchInstance; 70 import org.gudy.azureus2.plugins.utils.search.SearchObserver; 71 import org.gudy.azureus2.plugins.utils.search.SearchProvider; 72 import org.gudy.azureus2.pluginsimpl.local.PluginCoreUtils; 73 import org.gudy.azureus2.pluginsimpl.local.ddb.DDBaseImpl; 74 75 import com.aelitis.azureus.core.AzureusCore; 76 import com.aelitis.azureus.core.cnetwork.ContentNetwork; 77 import com.aelitis.azureus.core.proxy.impl.AEPluginProxyHandler; 78 import com.aelitis.azureus.core.security.CryptoManagerFactory; 79 import com.aelitis.azureus.core.tag.Tag; 80 import com.aelitis.azureus.core.tag.TagManager; 81 import com.aelitis.azureus.core.tag.TagManagerFactory; 82 import com.aelitis.azureus.core.tag.TagType; 83 import com.aelitis.azureus.core.torrent.PlatformTorrentUtils; 84 import com.aelitis.azureus.core.util.CopyOnWriteList; 85 import com.aelitis.azureus.core.util.FeatureAvailability; 86 import com.aelitis.azureus.core.util.bloom.BloomFilter; 87 import com.aelitis.azureus.core.util.bloom.BloomFilterFactory; 88 import com.aelitis.azureus.plugins.dht.DHTPlugin; 89 import com.aelitis.azureus.plugins.dht.DHTPluginContact; 90 import com.aelitis.azureus.plugins.dht.DHTPluginInterface; 91 import com.aelitis.azureus.plugins.dht.DHTPluginOperationListener; 92 import com.aelitis.azureus.plugins.dht.DHTPluginValue; 93 import com.aelitis.azureus.util.ImportExportUtils; 94 95 public class 96 RelatedContentManager 97 { 98 public static final long FILE_ASSOC_MIN_SIZE = 50*1024*1024; 99 100 public static final int RCM_SEARCH_PROPERTY_CONTENT_NETWORK = 50000; // don't change these, used in plugin 101 public static final int RCM_SEARCH_PROPERTY_TRACKER_KEYS = 50001; 102 public static final int RCM_SEARCH_PROPERTY_WEB_SEED_KEYS = 50002; 103 public static final int RCM_SEARCH_PROPERTY_TAGS = 50003; 104 public static final int RCM_SEARCH_PROPERTY_NETWORKS = 50004; 105 106 private static final boolean TRACE = false; 107 108 private static final int MAX_HISTORY = 16; 109 private static final int MAX_TITLE_LENGTH = 80; 110 private static final int MAX_CONCURRENT_PUBLISH; 111 private static final boolean DISABLE_PUBLISHING; 112 113 static{ 114 int max_conc_pub = 2; 115 116 DISABLE_PUBLISHING = System.getProperty( "azureus.rcm.publish.disable", "0").equals( "1" ); 117 118 try{ 119 120 max_conc_pub = Integer.parseInt( System.getProperty( "azureus.rcm.max.concurrent.publish", ""+max_conc_pub)); 121 122 }catch( Throwable e ){ 123 Debug.out( e ); 124 } 125 126 MAX_CONCURRENT_PUBLISH = max_conc_pub; 127 } 128 129 private static final int TEMPORARY_SPACE_DELTA = 50; 130 131 private static final int MAX_RANK = 100; 132 133 private static final String CONFIG_FILE = "rcm.config"; 134 private static final String PERSIST_DEL_FILE = "rcmx.config"; 135 136 private static final String CONFIG_TOTAL_UNREAD = "rcm.numunread.cache"; 137 138 private static RelatedContentManager singleton; 139 private static AzureusCore core; 140 141 protected static final int TIMER_PERIOD = 30*1000; 142 143 private static final int CONFIG_SAVE_CHECK_PERIOD = 60*1000; 144 private static final int CONFIG_SAVE_PERIOD = 5*60*1000; 145 private static final int CONFIG_SAVE_CHECK_TICKS = CONFIG_SAVE_CHECK_PERIOD/TIMER_PERIOD; 146 private static final int CONFIG_SAVE_TICKS = CONFIG_SAVE_PERIOD/TIMER_PERIOD; 147 private static final int PUBLISH_CHECK_PERIOD = 30*1000; 148 private static final int PUBLISH_CHECK_TICKS = PUBLISH_CHECK_PERIOD/TIMER_PERIOD; 149 private static final int PUBLISH_SLEEPING_CHECK_PERIOD = 5*60*1000; 150 private static final int PUBLISH_SLEEPING_CHECK_TICKS = PUBLISH_SLEEPING_CHECK_PERIOD/TIMER_PERIOD; 151 152 private static final int SECONDARY_LOOKUP_PERIOD = 15*60*1000; 153 private static final int SECONDARY_LOOKUP_TICKS = SECONDARY_LOOKUP_PERIOD/TIMER_PERIOD; 154 private static final int REPUBLISH_PERIOD = 8*60*60*1000; 155 private static final int REPUBLISH_TICKS = REPUBLISH_PERIOD/TIMER_PERIOD; 156 157 private static final int I2P_SEARCHER_CHECK_PERIOD = 10*60*1000; 158 private static final int I2P_SEARCHER_CHECK_TICKS = I2P_SEARCHER_CHECK_PERIOD/TIMER_PERIOD; 159 160 161 162 private static final int INITIAL_PUBLISH_DELAY = 3*60*1000; 163 private static final int INITIAL_PUBLISH_TICKS = INITIAL_PUBLISH_DELAY/TIMER_PERIOD; 164 165 166 private static final int CONFIG_DISCARD_MILLIS = 60*1000; 167 168 protected static final byte NET_NONE = 0x00; 169 protected static final byte NET_PUBLIC = 0x01; 170 protected static final byte NET_I2P = 0x02; 171 protected static final byte NET_TOR = 0x04; 172 173 private static final String[] NET_PUBLIC_ARRAY = { AENetworkClassifier.AT_PUBLIC }; 174 private static final String[] NET_I2P_ARRAY = { AENetworkClassifier.AT_I2P }; 175 private static final String[] NET_TOR_ARRAY = { AENetworkClassifier.AT_TOR }; 176 private static final String[] NET_PUBLIC_AND_I2P_ARRAY = { AENetworkClassifier.AT_PUBLIC, AENetworkClassifier.AT_I2P }; 177 178 public static synchronized void preInitialise( AzureusCore _core )179 preInitialise( 180 AzureusCore _core ) 181 { 182 core = _core; 183 } 184 185 public static synchronized RelatedContentManager getSingleton()186 getSingleton() 187 188 throws ContentException 189 { 190 if ( singleton == null ){ 191 192 singleton = new RelatedContentManager(); 193 } 194 195 return( singleton ); 196 } 197 198 protected final Object rcm_lock = new Object(); 199 200 private PluginInterface plugin_interface; 201 private TorrentAttribute ta_networks; 202 private TorrentAttribute ta_category; 203 private DHTPluginInterface public_dht_plugin; 204 205 private volatile Map<Byte,DHTPluginInterface> i2p_dht_plugin_map = new HashMap<Byte, DHTPluginInterface>(); 206 207 private TagManager tag_manager; 208 209 private long global_random_id = -1; 210 211 private LinkedList<DownloadInfo> pub_download_infos1 = new LinkedList<DownloadInfo>(); 212 private LinkedList<DownloadInfo> pub_download_infos2 = new LinkedList<DownloadInfo>(); 213 214 private LinkedList<DownloadInfo> non_pub_download_infos1 = new LinkedList<DownloadInfo>(); 215 private LinkedList<DownloadInfo> non_pub_download_infos2 = new LinkedList<DownloadInfo>(); 216 217 private ByteArrayHashMapEx<DownloadInfo> download_info_map = new ByteArrayHashMapEx<DownloadInfo>(); 218 private Set<String> download_priv_set = new HashSet<String>(); 219 220 221 private final boolean enabled; 222 223 private int max_search_level; 224 private int max_results; 225 226 private AtomicInteger temporary_space = new AtomicInteger(); 227 228 private int publishing_count = 0; 229 230 private CopyOnWriteList<RelatedContentManagerListener> listeners = new CopyOnWriteList<RelatedContentManagerListener>(); 231 232 private AESemaphore initialisation_complete_sem = new AESemaphore( "RCM:init" ); 233 234 private ContentCache content_cache_ref; 235 private WeakReference<ContentCache> content_cache; 236 237 private boolean content_dirty; 238 private long last_config_access; 239 private int content_discard_ticks; 240 241 private AtomicInteger total_unread = new AtomicInteger( COConfigurationManager.getIntParameter( CONFIG_TOTAL_UNREAD, 0 )); 242 243 private AsyncDispatcher content_change_dispatcher = new AsyncDispatcher(); 244 245 private static final int SECONDARY_LOOKUP_CACHE_MAX = 10; 246 247 private LinkedList<SecondaryLookup> secondary_lookups = new LinkedList<SecondaryLookup>(); 248 249 private boolean secondary_lookup_in_progress; 250 private long secondary_lookup_complete_time; 251 252 private RCMSearchXFer transfer_type = new RCMSearchXFer(); 253 254 private final CopyOnWriteList<RelatedContentSearcher> searchers = new CopyOnWriteList<RelatedContentSearcher>(); 255 private boolean added_i2p_searcher; 256 257 private static final int MAX_TRANSIENT_CACHE = 256; 258 259 protected static Map<String,DownloadInfo> transient_info_cache = 260 new LinkedHashMap<String,DownloadInfo>(MAX_TRANSIENT_CACHE,0.75f,true) 261 { 262 protected boolean 263 removeEldestEntry( 264 Map.Entry<String,DownloadInfo> eldest) 265 { 266 return size() > MAX_TRANSIENT_CACHE; 267 } 268 }; 269 270 271 private boolean persist; 272 273 { 274 COConfigurationManager.addAndFireParameterListener( 275 "rcm.persist", 276 new ParameterListener() 277 { 278 public void 279 parameterChanged( 280 String parameterName ) 281 { 282 persist = COConfigurationManager.getBooleanParameter( "rcm.persist" ) || true; 283 } 284 }); 285 286 // remove one day 287 288 COConfigurationManager.removeParameter( "rcm.dlinfo.history" ); 289 } 290 291 protected RelatedContentManager()292 RelatedContentManager() 293 294 throws ContentException 295 { 296 COConfigurationManager.addAndFireParameterListeners( 297 new String[]{ 298 "rcm.ui.enabled", 299 "rcm.max_search_level", 300 "rcm.max_results", 301 }, 302 new ParameterListener() 303 { 304 public void 305 parameterChanged( 306 String name ) 307 { 308 max_search_level = COConfigurationManager.getIntParameter( "rcm.max_search_level", 3 ); 309 max_results = COConfigurationManager.getIntParameter( "rcm.max_results", 500 ); 310 } 311 }); 312 313 if ( !FeatureAvailability.isRCMEnabled() || 314 !COConfigurationManager.getBooleanParameter( "rcm.overall.enabled", true )){ 315 316 enabled = false; 317 318 deleteRelatedContent(); 319 320 initialisation_complete_sem.releaseForever(); 321 322 return; 323 } 324 325 enabled = true; 326 327 try{ 328 if ( core == null ){ 329 330 throw( new ContentException( "getSingleton called before pre-initialisation" )); 331 } 332 333 while( global_random_id == -1 ){ 334 335 global_random_id = COConfigurationManager.getLongParameter( "rcm.random.id", -1 ); 336 337 if ( global_random_id == -1 ){ 338 339 global_random_id = RandomUtils.nextLong(); 340 341 COConfigurationManager.setParameter( "rcm.random.id", global_random_id ); 342 } 343 } 344 345 plugin_interface = core.getPluginManager().getDefaultPluginInterface(); 346 347 ta_networks = plugin_interface.getTorrentManager().getAttribute( TorrentAttribute.TA_NETWORKS ); 348 ta_category = plugin_interface.getTorrentManager().getAttribute( TorrentAttribute.TA_CATEGORY ); 349 350 tag_manager = TagManagerFactory.getTagManager(); 351 352 plugin_interface.getUtilities().createDelayedTask(new AERunnable() { 353 public void runSupport() { 354 SimpleTimer.addEvent( 355 "rcm.delay.init", 356 SystemTime.getOffsetTime( 15*1000 ), 357 new TimerEventPerformer() 358 { 359 public void 360 perform( 361 TimerEvent event ) 362 { 363 delayedInit(); 364 } 365 }); 366 } 367 }).queue(); 368 369 }catch( Throwable e ){ 370 371 initialisation_complete_sem.releaseForever(); 372 373 if ( e instanceof ContentException ){ 374 375 throw((ContentException)e); 376 } 377 378 throw( new ContentException( "Initialisation failed", e )); 379 } 380 } 381 382 protected PluginInterface getPluginInterface()383 getPluginInterface() 384 { 385 return( plugin_interface ); 386 } 387 delayedInit()388 private void delayedInit() { 389 390 plugin_interface.addListener( 391 new PluginListener() 392 { 393 public void 394 initializationComplete() 395 { 396 if ( !persist ){ 397 398 deleteRelatedContent(); 399 } 400 401 try{ 402 PluginInterface dht_pi = 403 plugin_interface.getPluginManager().getPluginInterfaceByClass( 404 DHTPlugin.class ); 405 406 if ( dht_pi != null ){ 407 408 DHTPlugin dp = (DHTPlugin)dht_pi.getPlugin(); 409 410 public_dht_plugin = dp; 411 412 RelatedContentSearcher public_searcher = new RelatedContentSearcher( RelatedContentManager.this, transfer_type, dp, true ); 413 414 searchers.add( public_searcher ); 415 416 DownloadManager dm = plugin_interface.getDownloadManager(); 417 418 Download[] downloads = dm.getDownloads(); 419 420 addDownloads( downloads, true ); 421 422 dm.addListener( 423 new DownloadManagerListener() 424 { 425 public void 426 downloadAdded( 427 Download download ) 428 { 429 addDownloads( new Download[]{ download }, false ); 430 } 431 432 public void 433 downloadRemoved( 434 Download download ) 435 { 436 } 437 }, 438 false ); 439 440 SimpleTimer.addPeriodicEvent( 441 "RCM:publisher", 442 TIMER_PERIOD, 443 new TimerEventPerformer() 444 { 445 private int tick_count; 446 447 public void 448 perform( 449 TimerEvent event ) 450 { 451 tick_count++; 452 453 if ( tick_count == 1 || tick_count % I2P_SEARCHER_CHECK_TICKS == 0 ){ 454 455 checkI2PSearcher( false ); 456 } 457 458 if ( enabled ){ 459 460 if ( tick_count >= INITIAL_PUBLISH_TICKS ){ 461 462 if ( tick_count % ( public_dht_plugin.isSleeping()?PUBLISH_SLEEPING_CHECK_TICKS:PUBLISH_CHECK_TICKS) == 0 ){ 463 464 publish(); 465 } 466 467 if ( tick_count % SECONDARY_LOOKUP_TICKS == 0 ){ 468 469 secondaryLookup(); 470 } 471 472 if ( tick_count % REPUBLISH_TICKS == 0 ){ 473 474 republish(); 475 } 476 477 if ( tick_count % CONFIG_SAVE_CHECK_TICKS == 0 ){ 478 479 saveRelatedContent( tick_count ); 480 } 481 } 482 } 483 484 for ( RelatedContentSearcher searcher: searchers ){ 485 486 searcher.timerTick( enabled, tick_count ); 487 } 488 } 489 }); 490 } 491 }finally{ 492 493 initialisation_complete_sem.releaseForever(); 494 } 495 } 496 497 public void 498 closedownInitiated() 499 { 500 saveRelatedContent( 0 ); 501 } 502 503 public void 504 closedownComplete() 505 { 506 } 507 }); 508 } 509 510 private void checkI2PSearcher( boolean force )511 checkI2PSearcher( 512 boolean force ) 513 { 514 // wanna defer adding the I2P one so we don't activate the pure destination just for rcm i2p search 515 516 synchronized( searchers ){ 517 518 if ( added_i2p_searcher ){ 519 520 return; 521 } 522 523 if ( !force ){ 524 525 DownloadManager dm = plugin_interface.getDownloadManager(); 526 527 Download[] downloads = dm.getDownloads(); 528 529 boolean found = false; 530 531 for ( Download download: downloads ){ 532 533 String[] nets = PluginCoreUtils.unwrap( download ).getDownloadState().getNetworks(); 534 535 if ( nets.length == 1 && nets[0] == AENetworkClassifier.AT_I2P ){ 536 537 found = true; 538 539 break; 540 } 541 } 542 543 if ( !found ){ 544 545 return; 546 } 547 } 548 549 List<DistributedDatabase> ddbs = DDBaseImpl.getDDBs( new String[]{ AENetworkClassifier.AT_I2P }); 550 551 for ( DistributedDatabase ddb: ddbs ){ 552 553 if ( ddb.getNetwork() == AENetworkClassifier.AT_I2P ){ 554 555 DHTPluginInterface i2p_dht = ddb.getDHTPlugin(); 556 557 RelatedContentSearcher i2p_searcher = new RelatedContentSearcher( RelatedContentManager.this, transfer_type, i2p_dht, false ); 558 559 searchers.add( i2p_searcher ); 560 561 added_i2p_searcher = true; 562 } 563 } 564 } 565 } 566 567 public boolean isEnabled()568 isEnabled() 569 { 570 return( enabled ); 571 } 572 573 public int getMaxSearchLevel()574 getMaxSearchLevel() 575 { 576 return( max_search_level ); 577 } 578 579 public void setMaxSearchLevel( int _level )580 setMaxSearchLevel( 581 int _level ) 582 { 583 COConfigurationManager.setParameter( "rcm.max_search_level", _level ); 584 } 585 586 public int getMaxResults()587 getMaxResults() 588 { 589 return( max_results ); 590 } 591 592 public void setMaxResults( int _max )593 setMaxResults( 594 int _max ) 595 { 596 COConfigurationManager.setParameter( "rcm.max_results", _max ); 597 598 enforceMaxResults( false ); 599 } 600 601 private int[] getAggregateSeedsLeechers( DownloadManagerState state )602 getAggregateSeedsLeechers( 603 DownloadManagerState state ) 604 { 605 String cache = state.getAttribute( DownloadManagerState.AT_AGGREGATE_SCRAPE_CACHE ); 606 607 int [] result = null; 608 609 if ( cache != null ){ 610 611 String[] bits = cache.split(","); 612 613 if ( bits.length == 3 ){ 614 615 try{ 616 long updated_mins = Long.parseLong( bits[0] ); 617 618 long mins = SystemTime.getCurrentTime()/(1000*60); 619 620 long age_mins = mins - updated_mins; 621 622 long WEEK_MINS = 7*24*60; 623 624 if ( age_mins <= WEEK_MINS ){ 625 626 int seeds = Integer.parseInt( bits[1] ); 627 int peers = Integer.parseInt( bits[2] ); 628 629 if ( seeds >= 0 && peers >= 0 ){ 630 631 result = new int[]{ seeds, peers }; 632 } 633 } 634 }catch( Throwable e ){ 635 636 } 637 } 638 } 639 640 return( result ); 641 } 642 643 private DHTPluginInterface selectDHT( byte networks )644 selectDHT( 645 byte networks ) 646 { 647 DHTPluginInterface result = null; 648 649 if ((networks & NET_PUBLIC ) != 0 ){ 650 651 result = public_dht_plugin; 652 653 }else if ((networks & NET_I2P ) != 0 ){ 654 655 synchronized( i2p_dht_plugin_map ){ 656 657 result = i2p_dht_plugin_map.get( networks ); 658 659 if ( result == null && !i2p_dht_plugin_map.containsKey( networks )){ 660 661 try{ 662 663 List<DistributedDatabase> ddbs = DDBaseImpl.getDDBs( convertNetworks( networks )); 664 665 for ( DistributedDatabase ddb: ddbs ){ 666 667 if ( ddb.getNetwork() == AENetworkClassifier.AT_I2P ){ 668 669 result = ddb.getDHTPlugin(); 670 } 671 } 672 }finally{ 673 674 i2p_dht_plugin_map.put( networks, result ); 675 } 676 } 677 } 678 } 679 680 if ( result != null ){ 681 682 if ( !result.isEnabled()){ 683 684 result = null; 685 } 686 } 687 688 return( result ); 689 } 690 691 protected void addDownloads( Download[] downloads, boolean initialising )692 addDownloads( 693 Download[] downloads, 694 boolean initialising ) 695 { 696 synchronized( rcm_lock ){ 697 698 List<DownloadInfo> new_info = new ArrayList<DownloadInfo>( downloads.length ); 699 700 for ( Download download: downloads ){ 701 702 try{ 703 if ( !download.isPersistent()){ 704 705 continue; 706 } 707 708 Torrent torrent = download.getTorrent(); 709 710 if ( torrent == null ){ 711 712 continue; 713 } 714 715 byte[] hash = torrent.getHash(); 716 717 if ( download_info_map.containsKey( hash )){ 718 719 continue; 720 } 721 722 byte nets = getNetworks( download ); 723 724 if ( nets == NET_NONE ){ 725 726 continue; 727 } 728 729 TOTorrent to_torrent = PluginCoreUtils.unwrap( torrent ); 730 731 if ( !TorrentUtils.isReallyPrivate( to_torrent )){ 732 733 DownloadManagerState state = PluginCoreUtils.unwrap( download ).getDownloadState(); 734 735 if ( state.getFlag(DownloadManagerState.FLAG_LOW_NOISE ) || state.getFlag(DownloadManagerState.FLAG_METADATA_DOWNLOAD )){ 736 737 continue; 738 } 739 740 LinkedList<DownloadInfo> download_infos1; 741 LinkedList<DownloadInfo> download_infos2; 742 743 if (( nets & NET_PUBLIC ) != 0 ){ 744 745 download_infos1 = pub_download_infos1; 746 download_infos2 = pub_download_infos2; 747 748 }else{ 749 750 download_infos1 = non_pub_download_infos1; 751 download_infos2 = non_pub_download_infos2; 752 } 753 754 int version = RelatedContent.VERSION_INITIAL; 755 756 long rand = global_random_id ^ state.getLongParameter( DownloadManagerState.PARAM_RANDOM_SEED ); 757 758 int seeds_leechers; 759 760 int[] aggregate_seeds_leechers = getAggregateSeedsLeechers( state ); 761 762 if ( aggregate_seeds_leechers == null ){ 763 764 long cache = state.getLongAttribute( DownloadManagerState.AT_SCRAPE_CACHE ); 765 766 if ( cache == -1 ){ 767 768 seeds_leechers = -1; 769 770 }else{ 771 772 int seeds = (int)((cache>>32)&0x00ffffff); 773 int leechers = (int)(cache&0x00ffffff); 774 775 seeds_leechers = (int)((seeds<<16)|(leechers&0xffff)); 776 } 777 }else{ 778 779 version = RelatedContent.VERSION_BETTER_SCRAPE; 780 781 int seeds = aggregate_seeds_leechers[0]; 782 int leechers = aggregate_seeds_leechers[1]; 783 784 seeds_leechers = (int)((seeds<<16)|(leechers&0xffff)); 785 } 786 787 byte[][] keys = getKeys( download ); 788 789 DownloadInfo info = 790 new DownloadInfo( 791 version, 792 hash, 793 hash, 794 download.getName(), 795 (int)rand, 796 torrent.isPrivate()?StringInterner.intern(torrent.getAnnounceURL().getHost()):null, 797 keys[0], 798 keys[1], 799 getTags( download ), 800 nets, 801 0, 802 false, 803 torrent.getSize(), 804 (int)( to_torrent.getCreationDate()/(60*60)), 805 seeds_leechers, 806 (byte)PlatformTorrentUtils.getContentNetworkID( to_torrent )); 807 808 new_info.add( info ); 809 810 if ( initialising || download_infos1.size() == 0 ){ 811 812 download_infos1.add( info ); 813 814 }else{ 815 816 download_infos1.add( RandomUtils.nextInt( download_infos1.size()), info ); 817 } 818 819 download_infos2.add( info ); 820 821 download_info_map.put( hash, info ); 822 823 if ( info.getTracker() != null ){ 824 825 download_priv_set.add( getPrivateInfoKey( info )); 826 } 827 } 828 }catch( Throwable e ){ 829 830 Debug.out( e ); 831 } 832 } 833 834 List<Map<String,Object>> history = (List<Map<String,Object>>)COConfigurationManager.getListParameter( "rcm.dlinfo.history.privx", new ArrayList<Map<String,Object>>()); 835 836 if ( initialising ){ 837 838 int padd = MAX_HISTORY - download_info_map.size(); 839 840 for ( int i=0;i<history.size() && padd > 0;i++ ){ 841 842 try{ 843 DownloadInfo info = deserialiseDI((Map<String,Object>)history.get(i), null); 844 845 if ( info != null && !download_info_map.containsKey( info.getHash())){ 846 847 download_info_map.put( info.getHash(), info ); 848 849 if ( info.getTracker() != null ){ 850 851 download_priv_set.add( getPrivateInfoKey( info )); 852 } 853 854 byte nets = info.getNetworksInternal(); 855 856 if ( nets != NET_NONE ){ 857 858 if (( nets & NET_PUBLIC ) != 0 ){ 859 860 pub_download_infos1.add( info ); 861 pub_download_infos2.add( info ); 862 863 }else{ 864 865 non_pub_download_infos1.add( info ); 866 non_pub_download_infos2.add( info ); 867 } 868 869 padd--; 870 } 871 } 872 }catch( Throwable e ){ 873 } 874 } 875 876 Collections.shuffle( pub_download_infos1 ); 877 Collections.shuffle( non_pub_download_infos1 ); 878 879 }else{ 880 881 if ( new_info.size() > 0 ){ 882 883 final List<String> base32_hashes = new ArrayList<String>(); 884 885 for ( DownloadInfo info: new_info ){ 886 887 byte[] hash = info.getHash(); 888 889 if ( hash != null ){ 890 891 base32_hashes.add( Base32.encode( hash )); 892 } 893 894 Map<String,Object> map = serialiseDI( info, null ); 895 896 if ( map != null ){ 897 898 history.add( map ); 899 } 900 } 901 902 while( history.size() > MAX_HISTORY ){ 903 904 history.remove(0); 905 } 906 907 COConfigurationManager.setParameter( "rcm.dlinfo.history.privx", history ); 908 909 if ( base32_hashes.size() > 0 ){ 910 911 content_change_dispatcher.dispatch( 912 new AERunnable() 913 { 914 public void 915 runSupport() 916 { 917 List<RelatedContent> to_remove = new ArrayList<RelatedContent>(); 918 919 synchronized( rcm_lock ){ 920 921 ContentCache content_cache = loadRelatedContent(); 922 923 for ( String h: base32_hashes ){ 924 925 DownloadInfo di = content_cache.related_content.get( h ); 926 927 if ( di != null ){ 928 929 to_remove.add( di ); 930 } 931 } 932 } 933 934 if ( to_remove.size() > 0 ){ 935 936 delete( to_remove.toArray( new RelatedContent[ to_remove.size()] )); 937 } 938 } 939 }); 940 } 941 } 942 } 943 } 944 } 945 946 private void republish()947 republish() 948 { 949 if ( DISABLE_PUBLISHING ){ 950 951 return; 952 } 953 954 synchronized( rcm_lock ){ 955 956 if ( publishing_count > 0 ){ 957 958 return; 959 } 960 961 if ( pub_download_infos1.isEmpty() || 962 ( pub_download_infos1.size() == 1 && 963 pub_download_infos1.getFirst() == pub_download_infos2.getFirst())){ 964 965 pub_download_infos1.clear(); 966 pub_download_infos2.clear(); 967 968 List<DownloadInfo> list = download_info_map.values(); 969 970 for ( DownloadInfo info: list ){ 971 972 if (( info.getNetworksInternal() & NET_PUBLIC ) != 0 ){ 973 974 pub_download_infos1.add( info ); 975 pub_download_infos2.add( info ); 976 } 977 } 978 979 Collections.shuffle( pub_download_infos1 ); 980 } 981 982 if ( non_pub_download_infos1.isEmpty() || 983 ( non_pub_download_infos1.size() == 1 && 984 non_pub_download_infos1.getFirst() == non_pub_download_infos2.getFirst())){ 985 986 non_pub_download_infos1.clear(); 987 non_pub_download_infos2.clear(); 988 989 List<DownloadInfo> list = download_info_map.values(); 990 991 for ( DownloadInfo info: list ){ 992 993 byte nets = info.getNetworksInternal(); 994 995 if ( nets != NET_NONE ){ 996 997 if (( nets & NET_PUBLIC ) == 0 ){ 998 999 non_pub_download_infos1.add( info ); 1000 non_pub_download_infos2.add( info ); 1001 } 1002 } 1003 } 1004 1005 Collections.shuffle( non_pub_download_infos1 ); 1006 } 1007 } 1008 } 1009 1010 private boolean last_pub_was_pub; 1011 1012 private void publish()1013 publish() 1014 { 1015 if ( DISABLE_PUBLISHING ){ 1016 1017 return; 1018 } 1019 1020 while( true ){ 1021 1022 DownloadInfo info1 = null; 1023 DownloadInfo info2 = null; 1024 1025 synchronized( rcm_lock ){ 1026 1027 if ( TRACE ){ 1028 System.out.println( "publish: count=" + publishing_count + ", dim=" + download_info_map.size() + ", pub=" + pub_download_infos1.size() + "/" + pub_download_infos2.size() + ", nonpub=" + non_pub_download_infos1.size() + "/" + non_pub_download_infos2.size()); 1029 } 1030 1031 if ( publishing_count >= MAX_CONCURRENT_PUBLISH ){ 1032 1033 // too busy 1034 1035 return; 1036 } 1037 1038 if ( download_info_map.size() == 1 ){ 1039 1040 // only one download, nothing to pair up with 1041 1042 return; 1043 } 1044 1045 boolean pub_ok = false; 1046 1047 if ( pub_download_infos1.isEmpty() || 1048 ( pub_download_infos1.size() == 1 && 1049 pub_download_infos1.getFirst() == pub_download_infos2.getFirst())){ 1050 1051 // either none or only one download remaining to be published and it has no partner 1052 1053 }else{ 1054 1055 pub_ok = true; 1056 } 1057 1058 boolean non_pub_ok = false; 1059 1060 if ( non_pub_download_infos1.isEmpty() || 1061 ( non_pub_download_infos1.size() == 1 && 1062 non_pub_download_infos1.getFirst() == non_pub_download_infos2.getFirst())){ 1063 1064 // either none or only one download remaining to be published and it has no partner 1065 1066 }else{ 1067 1068 non_pub_ok = true; 1069 } 1070 1071 if ( !( pub_ok || non_pub_ok )){ 1072 1073 return; 1074 } 1075 1076 LinkedList<DownloadInfo> download_infos1; 1077 LinkedList<DownloadInfo> download_infos2; 1078 1079 if ( pub_ok && non_pub_ok ){ 1080 1081 if ( last_pub_was_pub ){ 1082 1083 pub_ok = false; 1084 } 1085 1086 last_pub_was_pub = !last_pub_was_pub; 1087 } 1088 1089 if ( pub_ok ){ 1090 1091 download_infos1 = pub_download_infos1; 1092 download_infos2 = pub_download_infos2; 1093 1094 }else{ 1095 1096 download_infos1 = non_pub_download_infos1; 1097 download_infos2 = non_pub_download_infos2; 1098 } 1099 1100 if ( download_infos1.isEmpty() || download_info_map.size() == 1 ){ 1101 1102 return; 1103 } 1104 1105 info1 = download_infos1.removeFirst(); 1106 1107 Iterator<DownloadInfo> it = download_infos2.iterator(); 1108 1109 while( it.hasNext()){ 1110 1111 info2 = it.next(); 1112 1113 if ( info1 != info2 || download_infos2.size() == 1 ){ 1114 1115 it.remove(); 1116 1117 break; 1118 } 1119 } 1120 1121 if ( info1 == info2 ){ 1122 1123 return; 1124 } 1125 1126 publishing_count++; 1127 } 1128 1129 try{ 1130 if ( !publish( info1, info2 )){ 1131 1132 synchronized( rcm_lock ){ 1133 1134 publishing_count--; 1135 } 1136 } 1137 }catch( Throwable e ){ 1138 1139 synchronized( rcm_lock ){ 1140 1141 publishing_count--; 1142 } 1143 1144 Debug.out( e ); 1145 } 1146 } 1147 } 1148 1149 private void publishNext()1150 publishNext() 1151 { 1152 synchronized( rcm_lock ){ 1153 1154 publishing_count--; 1155 1156 if ( publishing_count < 0 ){ 1157 1158 // shouldn't happen but whatever 1159 1160 publishing_count = 0; 1161 } 1162 } 1163 1164 publish(); 1165 } 1166 1167 private boolean publish( final DownloadInfo from_info, final DownloadInfo to_info )1168 publish( 1169 final DownloadInfo from_info, 1170 final DownloadInfo to_info ) 1171 1172 throws Exception 1173 { 1174 final DHTPluginInterface dht_plugin = selectDHT( from_info.getNetworksInternal()); 1175 1176 // System.out.println( "publish: " + from_info.getString() + " -> " + to_info.getString() + ": " + dht_plugin ); 1177 1178 if ( dht_plugin == null ){ 1179 1180 return( false ); 1181 } 1182 1183 final String from_hash = ByteFormatter.encodeString( from_info.getHash()); 1184 final String to_hash = ByteFormatter.encodeString( to_info.getHash()); 1185 1186 final byte[] key_bytes = ( "az:rcm:assoc:" + from_hash ).getBytes( "UTF-8" ); 1187 1188 String title = to_info.getTitle(); 1189 1190 if ( title.length() > MAX_TITLE_LENGTH ){ 1191 1192 title = title.substring( 0, MAX_TITLE_LENGTH ); 1193 } 1194 1195 Map<String,Object> map = new HashMap<String,Object>(); 1196 1197 map.put( "d", title ); 1198 map.put( "r", new Long( Math.abs( to_info.getRand()%1000 ))); 1199 1200 String tracker = to_info.getTracker(); 1201 1202 if ( tracker == null ){ 1203 1204 map.put( "h", to_info.getHash()); 1205 1206 }else{ 1207 1208 map.put( "t", tracker ); 1209 } 1210 1211 if ( to_info.getLevel() == 0 ){ 1212 1213 try{ 1214 Download d = to_info.getRelatedToDownload(); 1215 1216 if ( d != null ){ 1217 1218 int version = RelatedContent.VERSION_INITIAL; 1219 1220 Torrent torrent = d.getTorrent(); 1221 1222 if ( torrent != null ){ 1223 1224 long cnet = PlatformTorrentUtils.getContentNetworkID( PluginCoreUtils.unwrap( torrent )); 1225 1226 if ( cnet != ContentNetwork.CONTENT_NETWORK_UNKNOWN ){ 1227 1228 map.put( "c", new Long( cnet )); 1229 } 1230 1231 long secs = torrent.getCreationDate(); 1232 1233 long hours = secs/(60*60); 1234 1235 if ( hours > 0 ){ 1236 1237 map.put( "p", new Long( hours )); 1238 } 1239 } 1240 1241 DownloadManagerState state = PluginCoreUtils.unwrap( d ).getDownloadState(); 1242 1243 int leechers = -1; 1244 int seeds = -1; 1245 1246 int[] aggregate_seeds_leechers = getAggregateSeedsLeechers( state ); 1247 1248 if ( aggregate_seeds_leechers == null ){ 1249 1250 long cache = state.getLongAttribute( DownloadManagerState.AT_SCRAPE_CACHE ); 1251 1252 if ( cache != -1 ){ 1253 1254 seeds = (int)((cache>>32)&0x00ffffff); 1255 leechers = (int)(cache&0x00ffffff); 1256 } 1257 }else{ 1258 1259 seeds = aggregate_seeds_leechers[0]; 1260 leechers = aggregate_seeds_leechers[1]; 1261 1262 version = RelatedContent.VERSION_BETTER_SCRAPE; 1263 } 1264 1265 if ( version > 0 ){ 1266 map.put( "v", new Long( version )); 1267 } 1268 1269 if ( leechers > 0 ){ 1270 map.put( "l", new Long( leechers )); 1271 } 1272 if ( seeds > 0 ){ 1273 map.put( "z", new Long( seeds )); 1274 } 1275 1276 byte[][] keys = getKeys( d ); 1277 1278 if ( keys[0] != null ){ 1279 map.put( "k", keys[0] ); 1280 } 1281 if ( keys[1] != null ){ 1282 map.put( "w", keys[1] ); 1283 } 1284 1285 String[] _tags = getTags( d ); 1286 1287 if ( _tags != null ){ 1288 map.put( "g", encodeTags( _tags )); 1289 } 1290 1291 byte nets = getNetworks( d ); 1292 1293 if ( nets != NET_PUBLIC ){ 1294 map.put( "o", new Long( nets&0xff )); 1295 } 1296 } 1297 }catch( Throwable e ){ 1298 } 1299 } 1300 1301 final Set<String> my_tags = new HashSet<String>(); 1302 1303 try{ 1304 Download d = from_info.getRelatedToDownload(); 1305 1306 if ( d != null ){ 1307 1308 String[] _tags = getTags( d ); 1309 1310 if ( _tags != null ){ 1311 1312 map.put( "b", from_info.getRand() % 100 ); 1313 1314 map.put( "m", encodeTags( _tags )); 1315 1316 for ( String tag: _tags ){ 1317 1318 my_tags.add( tag ); 1319 } 1320 } 1321 } 1322 }catch( Throwable e ){ 1323 } 1324 1325 long size = to_info.getSize(); 1326 1327 if ( size != 0 ){ 1328 1329 map.put( "s", new Long( size )); 1330 } 1331 1332 final byte[] map_bytes = BEncoder.encode( map ); 1333 1334 //System.out.println( "rcmsize=" + map_bytes.length ); 1335 1336 final int max_hits = 30; 1337 1338 dht_plugin.get( 1339 key_bytes, 1340 "Content rel test: " + from_hash.substring( 0, 16 ), 1341 DHTPlugin.FLAG_SINGLE_VALUE, 1342 max_hits, 1343 30*1000, 1344 false, 1345 false, 1346 new DHTPluginOperationListener() 1347 { 1348 private boolean diversified; 1349 private int hits; 1350 1351 private Set<String> entries = new HashSet<String>(); 1352 1353 public void 1354 starts( 1355 byte[] key ) 1356 { 1357 } 1358 1359 public boolean 1360 diversified() 1361 { 1362 diversified = true; 1363 1364 return( false ); 1365 } 1366 1367 public void 1368 valueRead( 1369 DHTPluginContact originator, 1370 DHTPluginValue value ) 1371 { 1372 try{ 1373 Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue()); 1374 1375 DownloadInfo info = decodeInfo( map, from_info.getHash(), 1, false, entries ); 1376 1377 try{ 1378 String[] r_tags = decodeTags((byte[]) map.get( "m" )); 1379 1380 if ( r_tags != null ){ 1381 1382 Long b = (Long)map.get( "b" ); 1383 1384 // don't remove tags from the set that we actually published 1385 1386 if ( b == null || from_info.getRand()%100 != b%100 ){ 1387 1388 for ( String tag: r_tags ){ 1389 1390 synchronized( my_tags ){ 1391 1392 my_tags.remove( tag ); 1393 } 1394 } 1395 } 1396 } 1397 }catch( Throwable e ){ 1398 } 1399 1400 if ( info != null ){ 1401 1402 analyseResponse( info, null ); 1403 } 1404 }catch( Throwable e ){ 1405 } 1406 1407 hits++; 1408 } 1409 1410 public void 1411 valueWritten( 1412 DHTPluginContact target, 1413 DHTPluginValue value ) 1414 { 1415 1416 } 1417 1418 public void 1419 complete( 1420 byte[] key, 1421 boolean timeout_occurred ) 1422 { 1423 // if we have something to say prioritise it somewhat 1424 1425 int f_cutoff = my_tags.size()>0?20:10; 1426 1427 try{ 1428 boolean do_it; 1429 1430 // System.out.println( from_hash + ": hits=" + hits + ", div=" + diversified ); 1431 1432 if ( diversified || hits >= f_cutoff ){ 1433 1434 do_it = false; 1435 1436 }else if ( hits <= f_cutoff / 2 ){ 1437 1438 do_it = true; 1439 1440 }else{ 1441 1442 do_it = RandomUtils.nextInt( hits - (f_cutoff/2) + 1 ) == 0; 1443 } 1444 1445 if ( do_it ){ 1446 1447 try{ 1448 dht_plugin.put( 1449 key_bytes, 1450 "Content rel: " + from_hash.substring( 0, 16 ) + " -> " + to_hash.substring( 0, 16 ), 1451 map_bytes, 1452 DHTPlugin.FLAG_ANON, 1453 new DHTPluginOperationListener() 1454 { 1455 public boolean 1456 diversified() 1457 { 1458 return( true ); 1459 } 1460 1461 public void 1462 starts( 1463 byte[] key ) 1464 { 1465 } 1466 1467 public void 1468 valueRead( 1469 DHTPluginContact originator, 1470 DHTPluginValue value ) 1471 { 1472 } 1473 1474 public void 1475 valueWritten( 1476 DHTPluginContact target, 1477 DHTPluginValue value ) 1478 { 1479 } 1480 1481 public void 1482 complete( 1483 byte[] key, 1484 boolean timeout_occurred ) 1485 { 1486 publishNext(); 1487 } 1488 }); 1489 }catch( Throwable e ){ 1490 1491 Debug.printStackTrace(e); 1492 1493 publishNext(); 1494 } 1495 }else{ 1496 1497 publishNext(); 1498 } 1499 }finally{ 1500 1501 checkAlternativePubs( to_info, map_bytes, f_cutoff ); 1502 } 1503 } 1504 }); 1505 1506 return( true ); 1507 } 1508 1509 private void checkAlternativePubs( DownloadInfo to_info, final byte[] map_bytes, final int f_cutoff )1510 checkAlternativePubs( 1511 DownloadInfo to_info, 1512 final byte[] map_bytes, 1513 final int f_cutoff ) 1514 { 1515 Download dl = to_info.getRelatedToDownload(); 1516 1517 if ( dl != null ){ 1518 1519 DiskManagerFileInfo[] files = dl.getDiskManagerFileInfo(); 1520 1521 List<Long> sizes = new ArrayList<Long>(); 1522 1523 for ( DiskManagerFileInfo file: files ){ 1524 1525 long size = file.getLength(); 1526 1527 if ( size >= FILE_ASSOC_MIN_SIZE ){ 1528 1529 sizes.add( size ); 1530 } 1531 } 1532 1533 final DHTPluginInterface dht_plugin = selectDHT( to_info.getNetworksInternal()); 1534 1535 if ( dht_plugin != null && sizes.size() > 0 ){ 1536 1537 try{ 1538 final String to_hash = ByteFormatter.encodeString( to_info.getHash()); 1539 1540 final long selected_size = sizes.get( new Random().nextInt( sizes.size())); 1541 1542 final byte[] key_bytes = ( "az:rcm:size:assoc:" + selected_size ).getBytes( "UTF-8" ); 1543 1544 int max_hits = 30; 1545 1546 dht_plugin.get( 1547 key_bytes, 1548 "Content size rel test: " + to_hash.substring( 0, 16 ), 1549 DHTPlugin.FLAG_SINGLE_VALUE, 1550 max_hits, 1551 30*1000, 1552 false, 1553 false, 1554 new DHTPluginOperationListener() 1555 { 1556 private boolean diversified; 1557 private int hits; 1558 1559 private Set<String> entries = new HashSet<String>(); 1560 1561 public void 1562 starts( 1563 byte[] key ) 1564 { 1565 } 1566 1567 public boolean 1568 diversified() 1569 { 1570 diversified = true; 1571 1572 return( false ); 1573 } 1574 1575 public void 1576 valueRead( 1577 DHTPluginContact originator, 1578 DHTPluginValue value ) 1579 { 1580 hits++; 1581 } 1582 1583 public void 1584 valueWritten( 1585 DHTPluginContact target, 1586 DHTPluginValue value ) 1587 { 1588 1589 } 1590 1591 public void 1592 complete( 1593 byte[] key, 1594 boolean timeout_occurred ) 1595 { 1596 boolean do_it; 1597 1598 // System.out.println( from_hash + ": hits=" + hits + ", div=" + diversified ); 1599 1600 if ( diversified || hits >= f_cutoff ){ 1601 1602 do_it = false; 1603 1604 }else if ( hits <= f_cutoff / 2 ){ 1605 1606 do_it = true; 1607 1608 }else{ 1609 1610 do_it = RandomUtils.nextInt( hits - ( f_cutoff / 2 ) + 1 ) == 0; 1611 } 1612 1613 if ( do_it ){ 1614 1615 try{ 1616 dht_plugin.put( 1617 key_bytes, 1618 "Content size rel: " + selected_size + " -> " + to_hash.substring( 0, 16 ), 1619 map_bytes, 1620 DHTPlugin.FLAG_ANON, 1621 new DHTPluginOperationListener() 1622 { 1623 public boolean 1624 diversified() 1625 { 1626 return( true ); 1627 } 1628 1629 public void 1630 starts( 1631 byte[] key ) 1632 { 1633 } 1634 1635 public void 1636 valueRead( 1637 DHTPluginContact originator, 1638 DHTPluginValue value ) 1639 { 1640 } 1641 1642 public void 1643 valueWritten( 1644 DHTPluginContact target, 1645 DHTPluginValue value ) 1646 { 1647 } 1648 1649 public void 1650 complete( 1651 byte[] key, 1652 boolean timeout_occurred ) 1653 { 1654 } 1655 }); 1656 }catch( Throwable e ){ 1657 1658 Debug.printStackTrace(e); 1659 } 1660 } 1661 } 1662 }); 1663 }catch( Throwable e ){ 1664 1665 Debug.out( e); 1666 } 1667 } 1668 } 1669 } 1670 1671 protected DownloadInfo decodeInfo( Map map, byte[] from_hash, int level, boolean explicit, Set<String> unique_keys )1672 decodeInfo( 1673 Map map, 1674 byte[] from_hash, // will be null for those trawled from teh local DHT 1675 int level, 1676 boolean explicit, 1677 Set<String> unique_keys ) 1678 { 1679 try{ 1680 String title = new String((byte[])map.get( "d" ), "UTF-8" ); 1681 1682 String tracker = null; 1683 1684 byte[] hash = (byte[])map.get( "h" ); 1685 1686 if ( hash == null ){ 1687 1688 tracker = new String((byte[])map.get( "t" ), "UTF-8" ); 1689 } 1690 1691 int rand = ((Long)map.get( "r" )).intValue(); 1692 1693 String key = title + " % " + rand; 1694 1695 synchronized( unique_keys ){ 1696 1697 if ( unique_keys.contains( key )){ 1698 1699 return( null ); 1700 } 1701 1702 unique_keys.add( key ); 1703 } 1704 1705 Long l_version = (Long)map.get( "v" ); 1706 1707 int version = l_version==null?RelatedContent.VERSION_INITIAL:l_version.intValue(); 1708 1709 Long l_size = (Long)map.get( "s" ); 1710 1711 long size = l_size==null?0:l_size.longValue(); 1712 1713 Long cnet = (Long)map.get( "c" ); 1714 Long published = (Long)map.get( "p" ); 1715 Long leechers = (Long)map.get( "l" ); 1716 Long seeds = (Long)map.get( "z" ); 1717 1718 // System.out.println( "p=" + published + ", l=" + leechers + ", s=" + seeds ); 1719 1720 int seeds_leechers; 1721 1722 if ( leechers == null && seeds == null ){ 1723 1724 seeds_leechers = -1; 1725 1726 }else if ( leechers == null ){ 1727 1728 seeds_leechers = seeds.intValue()<<16; 1729 1730 }else if ( seeds == null ){ 1731 1732 seeds_leechers = leechers.intValue()&0xffff; 1733 1734 }else{ 1735 1736 seeds_leechers = (seeds.intValue()<<16)|(leechers.intValue()&0xffff); 1737 } 1738 1739 byte[] tracker_keys = (byte[])map.get( "k" ); 1740 byte[] ws_keys = (byte[])map.get( "w" ); 1741 1742 if ( tracker_keys != null && tracker_keys.length % 4 != 0 ){ 1743 1744 tracker_keys = null; 1745 } 1746 1747 if ( ws_keys != null && ws_keys.length % 4 != 0 ){ 1748 1749 ws_keys = null; 1750 } 1751 1752 byte[] _tags = (byte[])map.get( "g" ); 1753 1754 String[] tags = decodeTags( _tags ); 1755 1756 Long _nets = (Long)map.get( "o" ); 1757 1758 byte nets = _nets==null?NET_PUBLIC:_nets.byteValue(); 1759 1760 return( 1761 new DownloadInfo( 1762 version, 1763 from_hash, hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, level, explicit, size, 1764 published==null?0:published.intValue(), 1765 seeds_leechers, 1766 (byte)(cnet==null?ContentNetwork.CONTENT_NETWORK_UNKNOWN:cnet.byteValue()))); 1767 1768 }catch( Throwable e ){ 1769 1770 return( null ); 1771 } 1772 } 1773 1774 public void lookupAttributes( final byte[] from_hash, final RelatedAttributeLookupListener listener )1775 lookupAttributes( 1776 final byte[] from_hash, 1777 final RelatedAttributeLookupListener listener ) 1778 1779 throws ContentException 1780 { 1781 lookupAttributes( from_hash, new String[]{ AENetworkClassifier.AT_PUBLIC }, listener ); 1782 } 1783 1784 public void lookupAttributes( final byte[] from_hash, final String[] networks, final RelatedAttributeLookupListener listener )1785 lookupAttributes( 1786 final byte[] from_hash, 1787 final String[] networks, 1788 final RelatedAttributeLookupListener listener ) 1789 1790 throws ContentException 1791 { 1792 if ( from_hash == null ){ 1793 1794 throw( new ContentException( "hash is null" )); 1795 } 1796 1797 if ( !initialisation_complete_sem.isReleasedForever() || 1798 ( public_dht_plugin != null && public_dht_plugin.isInitialising())){ 1799 1800 AsyncDispatcher dispatcher = new AsyncDispatcher(); 1801 1802 dispatcher.dispatch( 1803 new AERunnable() 1804 { 1805 public void 1806 runSupport() 1807 { 1808 try{ 1809 initialisation_complete_sem.reserve(); 1810 1811 lookupAttributesSupport( from_hash, convertNetworks( networks ), listener ); 1812 1813 }catch( ContentException e ){ 1814 1815 Debug.out( e ); 1816 } 1817 } 1818 }); 1819 }else{ 1820 1821 lookupAttributesSupport( from_hash, convertNetworks( networks ), listener ); 1822 } 1823 } 1824 1825 private void lookupAttributesSupport( final byte[] from_hash, final byte networks, final RelatedAttributeLookupListener listener )1826 lookupAttributesSupport( 1827 final byte[] from_hash, 1828 final byte networks, 1829 final RelatedAttributeLookupListener listener ) 1830 1831 throws ContentException 1832 { 1833 try{ 1834 if ( !enabled ){ 1835 1836 throw( new ContentException( "rcm is disabled" )); 1837 } 1838 1839 final DHTPluginInterface dht_plugin = selectDHT( networks ); 1840 1841 if ( dht_plugin == null ){ 1842 1843 throw( new Exception( "DHT Plugin unavailable for networks " + getString( convertNetworks( networks )))); 1844 } 1845 1846 // really should implement a getNetwork() in DHTPluginInterface... 1847 1848 final String dht_plugin_network = dht_plugin == public_dht_plugin?AENetworkClassifier.AT_PUBLIC:AENetworkClassifier.AT_I2P; 1849 1850 final String from_hash_str = ByteFormatter.encodeString( from_hash ); 1851 1852 final byte[] key_bytes = ( "az:rcm:assoc:" + from_hash_str ).getBytes( "UTF-8" ); 1853 1854 String op_str = "Content attr read: " + from_hash_str.substring( 0, 16 ); 1855 1856 dht_plugin.get( 1857 key_bytes, 1858 op_str, 1859 DHTPlugin.FLAG_SINGLE_VALUE, 1860 512, 1861 30*1000, 1862 false, 1863 true, 1864 new DHTPluginOperationListener() 1865 { 1866 private Set<String> tags = new HashSet<String>(); 1867 1868 public void 1869 starts( 1870 byte[] key ) 1871 { 1872 if ( listener != null ){ 1873 1874 try{ 1875 listener.lookupStart(); 1876 1877 }catch( Throwable e ){ 1878 1879 Debug.out( e ); 1880 } 1881 1882 ContentCache content_cache = loadRelatedContent(); 1883 1884 DownloadInfo info = content_cache.related_content.get( Base32.encode( from_hash )); 1885 1886 if ( info != null ){ 1887 1888 String[] l_tags = info.getTags(); 1889 1890 if ( l_tags != null ){ 1891 1892 for ( String tag: l_tags ){ 1893 1894 synchronized( tags ){ 1895 1896 if ( tags.contains( tag )){ 1897 1898 continue; 1899 } 1900 1901 tags.add( tag ); 1902 } 1903 1904 try{ 1905 listener.tagFound( tag, dht_plugin_network ); 1906 1907 }catch( Throwable e ){ 1908 1909 Debug.out( e ); 1910 } 1911 } 1912 } 1913 } 1914 } 1915 } 1916 1917 public boolean 1918 diversified() 1919 { 1920 return( true ); 1921 } 1922 1923 public void 1924 valueRead( 1925 DHTPluginContact originator, 1926 DHTPluginValue value ) 1927 { 1928 try{ 1929 Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue()); 1930 1931 String[] r_tags = decodeTags((byte[]) map.get( "m" )); 1932 1933 if ( r_tags != null ){ 1934 1935 for ( String tag: r_tags ){ 1936 1937 synchronized( tags ){ 1938 1939 if ( tags.contains( tag )){ 1940 1941 continue; 1942 } 1943 1944 tags.add( tag ); 1945 } 1946 1947 try{ 1948 listener.tagFound( tag, dht_plugin_network ); 1949 1950 }catch( Throwable e ){ 1951 1952 Debug.out( e ); 1953 } 1954 } 1955 } 1956 1957 }catch( Throwable e ){ 1958 } 1959 } 1960 1961 public void 1962 valueWritten( 1963 DHTPluginContact target, 1964 DHTPluginValue value ) 1965 { 1966 1967 } 1968 1969 public void 1970 complete( 1971 byte[] key, 1972 boolean timeout_occurred ) 1973 { 1974 if ( listener != null ){ 1975 1976 try{ 1977 listener.lookupComplete(); 1978 1979 }catch( Throwable e ){ 1980 1981 Debug.out( e ); 1982 } 1983 } 1984 } 1985 }); 1986 }catch( Throwable e ){ 1987 1988 ContentException ce; 1989 1990 if ( ( e instanceof ContentException )){ 1991 1992 ce = (ContentException)e; 1993 1994 }else{ 1995 ce = new ContentException( "Lookup failed", e ); 1996 } 1997 1998 if ( listener != null ){ 1999 2000 try{ 2001 listener.lookupFailed( ce ); 2002 2003 }catch( Throwable f ){ 2004 2005 Debug.out( f ); 2006 } 2007 } 2008 2009 throw( ce ); 2010 } 2011 } 2012 2013 public void lookupContent( final byte[] hash, final RelatedContentLookupListener listener )2014 lookupContent( 2015 final byte[] hash, 2016 final RelatedContentLookupListener listener ) 2017 2018 throws ContentException 2019 { 2020 if ( hash == null ){ 2021 2022 throw( new ContentException( "hash is null" )); 2023 } 2024 2025 byte net = NET_PUBLIC; 2026 2027 try{ 2028 Download download = plugin_interface.getDownloadManager().getDownload( hash ); 2029 2030 if ( download != null ){ 2031 2032 net = getNetworks( download ); 2033 } 2034 }catch( Throwable e ){ 2035 2036 } 2037 2038 final byte f_net = net; 2039 2040 if ( !initialisation_complete_sem.isReleasedForever() || 2041 ( public_dht_plugin != null && public_dht_plugin.isInitialising())){ 2042 2043 AsyncDispatcher dispatcher = new AsyncDispatcher(); 2044 2045 dispatcher.dispatch( 2046 new AERunnable() 2047 { 2048 public void 2049 runSupport() 2050 { 2051 try{ 2052 initialisation_complete_sem.reserve(); 2053 2054 lookupContentSupport( hash, 0, f_net, true, listener ); 2055 2056 }catch( ContentException e ){ 2057 2058 Debug.out( e ); 2059 } 2060 } 2061 }); 2062 }else{ 2063 2064 lookupContentSupport( hash, 0, f_net, true, listener ); 2065 } 2066 } 2067 2068 public void lookupContent( final byte[] hash, final String[] networks, final RelatedContentLookupListener listener )2069 lookupContent( 2070 final byte[] hash, 2071 final String[] networks, 2072 final RelatedContentLookupListener listener ) 2073 2074 throws ContentException 2075 { 2076 if ( hash == null ){ 2077 2078 throw( new ContentException( "hash is null" )); 2079 } 2080 2081 final byte net = convertNetworks( networks ); 2082 2083 if ( net == 0 ){ 2084 2085 throw( new ContentException( "No networks specified" )); 2086 } 2087 2088 if ( !initialisation_complete_sem.isReleasedForever() || 2089 ( public_dht_plugin != null && public_dht_plugin.isInitialising())){ 2090 2091 AsyncDispatcher dispatcher = new AsyncDispatcher(); 2092 2093 dispatcher.dispatch( 2094 new AERunnable() 2095 { 2096 public void 2097 runSupport() 2098 { 2099 try{ 2100 initialisation_complete_sem.reserve(); 2101 2102 lookupContentSupport( hash, 0, net, true, listener ); 2103 2104 }catch( ContentException e ){ 2105 2106 Debug.out( e ); 2107 } 2108 } 2109 }); 2110 }else{ 2111 2112 lookupContentSupport( hash, 0, net, true, listener ); 2113 } 2114 } 2115 2116 public void lookupContent( final long file_size, final RelatedContentLookupListener listener )2117 lookupContent( 2118 final long file_size, 2119 final RelatedContentLookupListener listener ) 2120 2121 throws ContentException 2122 { 2123 if ( file_size < FILE_ASSOC_MIN_SIZE ){ 2124 2125 throw( new ContentException( "file size is invalid - min=" + FILE_ASSOC_MIN_SIZE )); 2126 } 2127 2128 if ( !initialisation_complete_sem.isReleasedForever() || 2129 ( public_dht_plugin != null && public_dht_plugin.isInitialising())){ 2130 2131 AsyncDispatcher dispatcher = new AsyncDispatcher(); 2132 2133 dispatcher.dispatch( 2134 new AERunnable() 2135 { 2136 public void 2137 runSupport() 2138 { 2139 try{ 2140 initialisation_complete_sem.reserve(); 2141 2142 lookupContentSupport( file_size, NET_PUBLIC, listener ); 2143 2144 }catch( ContentException e ){ 2145 2146 Debug.out( e ); 2147 } 2148 } 2149 }); 2150 }else{ 2151 2152 lookupContentSupport( file_size, NET_PUBLIC, listener ); 2153 } 2154 } 2155 2156 public void lookupContent( final long file_size, final String[] networks, final RelatedContentLookupListener listener )2157 lookupContent( 2158 final long file_size, 2159 final String[] networks, 2160 final RelatedContentLookupListener listener ) 2161 2162 throws ContentException 2163 { 2164 if ( file_size < FILE_ASSOC_MIN_SIZE ){ 2165 2166 throw( new ContentException( "file size is invalid - min=" + FILE_ASSOC_MIN_SIZE )); 2167 } 2168 2169 final byte net = convertNetworks( networks ); 2170 2171 if ( net == 0 ){ 2172 2173 throw( new ContentException( "No networks specified" )); 2174 } 2175 2176 if ( !initialisation_complete_sem.isReleasedForever() || 2177 ( public_dht_plugin != null && public_dht_plugin.isInitialising())){ 2178 2179 AsyncDispatcher dispatcher = new AsyncDispatcher(); 2180 2181 dispatcher.dispatch( 2182 new AERunnable() 2183 { 2184 public void 2185 runSupport() 2186 { 2187 try{ 2188 initialisation_complete_sem.reserve(); 2189 2190 lookupContentSupport( file_size, net, listener ); 2191 2192 }catch( ContentException e ){ 2193 2194 Debug.out( e ); 2195 } 2196 } 2197 }); 2198 }else{ 2199 2200 lookupContentSupport( file_size, net, listener ); 2201 } 2202 } 2203 2204 private void lookupContentSupport( final long file_size, final byte networks, final RelatedContentLookupListener listener )2205 lookupContentSupport( 2206 final long file_size, 2207 final byte networks, 2208 final RelatedContentLookupListener listener ) 2209 2210 throws ContentException 2211 { 2212 if ( !enabled ){ 2213 2214 throw( new ContentException( "rcm is disabled" )); 2215 } 2216 2217 try{ 2218 final byte[] key_bytes = ( "az:rcm:size:assoc:" + file_size ).getBytes( "UTF-8" ); 2219 2220 // we need something to use 2221 2222 final byte[] from_hash = new SHA1Simple().calculateHash( key_bytes ); 2223 2224 String op_str = "Content rel read: size=" + file_size; 2225 2226 lookupContentSupport0( from_hash, key_bytes, op_str, 0, networks, true, listener ); 2227 2228 }catch( ContentException e ){ 2229 2230 throw( e ); 2231 2232 }catch( Throwable e ){ 2233 2234 throw( new ContentException( "lookup failed", e )); 2235 } 2236 } 2237 2238 private void lookupContentSupport( final byte[] from_hash, final int level, final byte networks, final boolean explicit, final RelatedContentLookupListener listener )2239 lookupContentSupport( 2240 final byte[] from_hash, 2241 final int level, 2242 final byte networks, 2243 final boolean explicit, 2244 final RelatedContentLookupListener listener ) 2245 2246 throws ContentException 2247 { 2248 if ( !enabled ){ 2249 2250 throw( new ContentException( "rcm is disabled" )); 2251 } 2252 2253 try{ 2254 2255 final String from_hash_str = ByteFormatter.encodeString( from_hash ); 2256 2257 final byte[] key_bytes = ( "az:rcm:assoc:" + from_hash_str ).getBytes( "UTF-8" ); 2258 2259 String op_str = "Content rel read: " + from_hash_str.substring( 0, 16 ); 2260 2261 lookupContentSupport0( from_hash, key_bytes, op_str, level, networks, explicit, listener ); 2262 2263 }catch( ContentException e ){ 2264 2265 throw( e ); 2266 2267 }catch( Throwable e ){ 2268 2269 throw( new ContentException( "lookup failed", e )); 2270 } 2271 } 2272 2273 private String getString( String[] args )2274 getString( 2275 String[] args ) 2276 { 2277 String str = ""; 2278 2279 for ( String s: args ){ 2280 2281 str += (str.length()==0?"":",") + s; 2282 } 2283 2284 return( str ); 2285 } 2286 2287 private void lookupContentSupport0( final byte[] from_hash, final byte[] key_bytes, final String op_str, final int level, final byte networks, final boolean explicit, final RelatedContentLookupListener listener )2288 lookupContentSupport0( 2289 final byte[] from_hash, 2290 final byte[] key_bytes, 2291 final String op_str, 2292 final int level, 2293 final byte networks, 2294 final boolean explicit, 2295 final RelatedContentLookupListener listener ) 2296 2297 throws ContentException 2298 { 2299 try{ 2300 final int max_hits = 30; 2301 2302 DHTPluginInterface dht_plugin = selectDHT( networks ); 2303 2304 if ( dht_plugin == null ){ 2305 2306 throw( new Exception( "DHT Plugin unavailable for networks '" + getString( convertNetworks( networks )) + "'" )); 2307 } 2308 2309 dht_plugin.get( 2310 key_bytes, 2311 op_str, 2312 DHTPlugin.FLAG_SINGLE_VALUE, 2313 max_hits, 2314 60*1000, 2315 false, 2316 true, 2317 new DHTPluginOperationListener() 2318 { 2319 private Set<String> entries = new HashSet<String>(); 2320 2321 private RelatedContentManagerListener manager_listener = 2322 new RelatedContentManagerListener() 2323 { 2324 private Set<RelatedContent> content_list = new HashSet<RelatedContent>(); 2325 2326 public void 2327 contentFound( 2328 RelatedContent[] content ) 2329 { 2330 handle( content ); 2331 } 2332 2333 public void 2334 contentChanged( 2335 RelatedContent[] content ) 2336 { 2337 handle( content ); 2338 } 2339 2340 public void 2341 contentRemoved( 2342 RelatedContent[] content ) 2343 { 2344 } 2345 2346 public void 2347 contentChanged() 2348 { 2349 } 2350 2351 public void 2352 contentReset() 2353 { 2354 } 2355 2356 private void 2357 handle( 2358 RelatedContent[] content ) 2359 { 2360 List<RelatedContent> new_content = new ArrayList<RelatedContent>( content.length ); 2361 2362 synchronized( content_list ){ 2363 2364 for ( RelatedContent rc: content ){ 2365 2366 if ( !content_list.contains( rc )){ 2367 2368 new_content.add( rc ); 2369 } 2370 } 2371 2372 if ( new_content.size() == 0 ){ 2373 2374 return; 2375 } 2376 2377 content_list.addAll( new_content ); 2378 } 2379 2380 listener.contentFound( new_content.toArray( new RelatedContent[new_content.size()] )); 2381 } 2382 }; 2383 2384 public void 2385 starts( 2386 byte[] key ) 2387 { 2388 if ( listener != null ){ 2389 2390 try{ 2391 listener.lookupStart(); 2392 2393 }catch( Throwable e ){ 2394 2395 Debug.out( e ); 2396 } 2397 } 2398 } 2399 2400 public boolean 2401 diversified() 2402 { 2403 return( true ); 2404 } 2405 2406 public void 2407 valueRead( 2408 DHTPluginContact originator, 2409 DHTPluginValue value ) 2410 { 2411 try{ 2412 Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue()); 2413 2414 DownloadInfo info = decodeInfo( map, from_hash, level+1, explicit, entries ); 2415 2416 if ( info != null ){ 2417 2418 analyseResponse( info, listener==null?null:manager_listener ); 2419 } 2420 }catch( Throwable e ){ 2421 } 2422 } 2423 2424 public void 2425 valueWritten( 2426 DHTPluginContact target, 2427 DHTPluginValue value ) 2428 { 2429 2430 } 2431 2432 public void 2433 complete( 2434 byte[] key, 2435 boolean timeout_occurred ) 2436 { 2437 if ( listener != null ){ 2438 2439 try{ 2440 listener.lookupComplete(); 2441 2442 }catch( Throwable e ){ 2443 2444 Debug.out( e ); 2445 } 2446 } 2447 } 2448 }); 2449 }catch( Throwable e ){ 2450 2451 ContentException ce; 2452 2453 if ( ( e instanceof ContentException )){ 2454 2455 ce = (ContentException)e; 2456 2457 }else{ 2458 ce = new ContentException( "Lookup failed", e ); 2459 } 2460 2461 if ( listener != null ){ 2462 2463 try{ 2464 listener.lookupFailed( ce ); 2465 2466 }catch( Throwable f ){ 2467 2468 Debug.out( f ); 2469 } 2470 } 2471 2472 throw( ce ); 2473 } 2474 } 2475 2476 protected void popuplateSecondaryLookups( ContentCache content_cache )2477 popuplateSecondaryLookups( 2478 ContentCache content_cache ) 2479 { 2480 Random rand = new Random(); 2481 2482 secondary_lookups.clear(); 2483 2484 // stuff in a couple primarys 2485 2486 List<DownloadInfo> primaries = download_info_map.values(); 2487 2488 int primary_count = primaries.size(); 2489 2490 int primaries_to_add; 2491 2492 if ( primary_count < 2 ){ 2493 2494 primaries_to_add = 0; 2495 2496 }else if ( primary_count < 5 ){ 2497 2498 if ( rand.nextInt(4) == 0 ){ 2499 2500 primaries_to_add = 1; 2501 2502 }else{ 2503 2504 primaries_to_add = 0; 2505 } 2506 }else if ( primary_count < 10 ){ 2507 2508 primaries_to_add = 1; 2509 2510 }else{ 2511 2512 primaries_to_add = 2; 2513 } 2514 2515 if ( primaries_to_add > 0 ){ 2516 2517 Set<DownloadInfo> added = new HashSet<DownloadInfo>(); 2518 2519 for (int i=0;i<primaries_to_add;i++){ 2520 2521 DownloadInfo info = primaries.get( rand.nextInt( primaries.size())); 2522 2523 if ( !added.contains( info )){ 2524 2525 added.add( info ); 2526 2527 secondary_lookups.addLast(new SecondaryLookup(info.getHash(), info.getLevel(), info.getNetworksInternal())); 2528 } 2529 } 2530 } 2531 2532 Map<String,DownloadInfo> related_content = content_cache.related_content; 2533 2534 Iterator<DownloadInfo> it = related_content.values().iterator(); 2535 2536 List<DownloadInfo> secondary_cache_temp = new ArrayList<DownloadInfo>( related_content.size()); 2537 2538 while( it.hasNext()){ 2539 2540 DownloadInfo di = it.next(); 2541 2542 if ( di.getHash() != null && di.getLevel() < max_search_level ){ 2543 2544 secondary_cache_temp.add( di ); 2545 } 2546 } 2547 2548 final int cache_size = Math.min( secondary_cache_temp.size(), SECONDARY_LOOKUP_CACHE_MAX - secondary_lookups.size()); 2549 2550 if ( cache_size > 0 ){ 2551 2552 for( int i=0;i<cache_size;i++){ 2553 2554 int index = rand.nextInt( secondary_cache_temp.size()); 2555 2556 DownloadInfo x = secondary_cache_temp.get( index ); 2557 2558 secondary_cache_temp.set( index, secondary_cache_temp.get(i)); 2559 2560 secondary_cache_temp.set( i, x ); 2561 } 2562 2563 for ( int i=0;i<cache_size;i++){ 2564 2565 DownloadInfo x = secondary_cache_temp.get(i); 2566 2567 secondary_lookups.addLast(new SecondaryLookup(x.getHash(), x.getLevel(), x.getNetworksInternal())); 2568 } 2569 } 2570 } 2571 2572 protected void secondaryLookup()2573 secondaryLookup() 2574 { 2575 SecondaryLookup sl; 2576 2577 long now = SystemTime.getMonotonousTime(); 2578 2579 synchronized( rcm_lock ){ 2580 2581 if ( secondary_lookup_in_progress ){ 2582 2583 return; 2584 } 2585 2586 if ( now - secondary_lookup_complete_time < SECONDARY_LOOKUP_PERIOD ){ 2587 2588 return; 2589 } 2590 2591 if ( secondary_lookups.size() == 0 ){ 2592 2593 ContentCache cc = content_cache==null?null:content_cache.get(); 2594 2595 if ( cc == null ){ 2596 2597 // this will populate the cache 2598 2599 cc = loadRelatedContent(); 2600 2601 }else{ 2602 2603 popuplateSecondaryLookups( cc ); 2604 } 2605 } 2606 2607 if ( secondary_lookups.size() == 0 ){ 2608 2609 return; 2610 } 2611 2612 sl = secondary_lookups.removeFirst(); 2613 2614 secondary_lookup_in_progress = true; 2615 } 2616 2617 try{ 2618 lookupContentSupport( 2619 sl.getHash(), 2620 sl.getLevel(), 2621 sl.getNetworks(), 2622 false, 2623 new RelatedContentLookupListener() 2624 { 2625 public void 2626 lookupStart() 2627 { 2628 } 2629 2630 public void 2631 contentFound( 2632 RelatedContent[] content ) 2633 { 2634 } 2635 2636 public void 2637 lookupComplete() 2638 { 2639 next(); 2640 } 2641 2642 public void 2643 lookupFailed( 2644 ContentException error ) 2645 { 2646 next(); 2647 } 2648 2649 protected void 2650 next() 2651 { 2652 final SecondaryLookup next_sl; 2653 2654 synchronized( rcm_lock ){ 2655 2656 if ( secondary_lookups.size() == 0 ){ 2657 2658 secondary_lookup_in_progress = false; 2659 2660 secondary_lookup_complete_time = SystemTime.getMonotonousTime(); 2661 2662 return; 2663 2664 }else{ 2665 2666 next_sl = secondary_lookups.removeFirst(); 2667 } 2668 } 2669 2670 final RelatedContentLookupListener listener = this; 2671 2672 SimpleTimer.addEvent( 2673 "RCM:SLDelay", 2674 SystemTime.getOffsetTime( 30*1000 ), 2675 new TimerEventPerformer() 2676 { 2677 public void 2678 perform( 2679 TimerEvent event ) 2680 { 2681 try{ 2682 lookupContentSupport( next_sl.getHash(), next_sl.getLevel(), next_sl.getNetworks(), false, listener ); 2683 2684 }catch( Throwable e ){ 2685 2686 //Debug.out( e ); 2687 2688 synchronized( rcm_lock ){ 2689 2690 secondary_lookup_in_progress = false; 2691 2692 secondary_lookup_complete_time = SystemTime.getMonotonousTime(); 2693 } 2694 } 2695 } 2696 }); 2697 } 2698 }); 2699 2700 }catch( Throwable e ){ 2701 2702 //Debug.out( e ); 2703 2704 synchronized( rcm_lock ){ 2705 2706 secondary_lookup_in_progress = false; 2707 2708 secondary_lookup_complete_time = now; 2709 } 2710 } 2711 } 2712 2713 protected void contentChanged( final DownloadInfo info )2714 contentChanged( 2715 final DownloadInfo info ) 2716 { 2717 setConfigDirty(); 2718 2719 content_change_dispatcher.dispatch( 2720 new AERunnable() 2721 { 2722 public void 2723 runSupport() 2724 { 2725 for ( RelatedContentManagerListener l: listeners ){ 2726 2727 try{ 2728 l.contentChanged( new RelatedContent[]{ info }); 2729 2730 }catch( Throwable e ){ 2731 2732 Debug.out( e ); 2733 } 2734 } 2735 } 2736 }); 2737 } 2738 2739 protected void contentChanged( boolean is_dirty )2740 contentChanged( 2741 boolean is_dirty ) 2742 { 2743 if ( is_dirty ){ 2744 2745 setConfigDirty(); 2746 } 2747 2748 content_change_dispatcher.dispatch( 2749 new AERunnable() 2750 { 2751 public void 2752 runSupport() 2753 { 2754 for ( RelatedContentManagerListener l: listeners ){ 2755 2756 try{ 2757 l.contentChanged(); 2758 2759 }catch( Throwable e ){ 2760 2761 Debug.out( e ); 2762 } 2763 } 2764 } 2765 }); 2766 } 2767 2768 public void delete( RelatedContent[] content )2769 delete( 2770 RelatedContent[] content ) 2771 { 2772 synchronized( rcm_lock ){ 2773 2774 ContentCache content_cache = loadRelatedContent(); 2775 2776 delete( content, content_cache, true ); 2777 } 2778 } 2779 2780 protected void delete( final RelatedContent[] content, ContentCache content_cache, boolean persistent )2781 delete( 2782 final RelatedContent[] content, 2783 ContentCache content_cache, 2784 boolean persistent ) 2785 { 2786 if ( persistent ){ 2787 2788 addPersistentlyDeleted( content ); 2789 } 2790 2791 Map<String,DownloadInfo> related_content = content_cache.related_content; 2792 2793 Iterator<DownloadInfo> it = related_content.values().iterator(); 2794 2795 while( it.hasNext()){ 2796 2797 DownloadInfo di = it.next(); 2798 2799 for ( RelatedContent c: content ){ 2800 2801 if ( c == di ){ 2802 2803 it.remove(); 2804 2805 if ( di.isUnread()){ 2806 2807 decrementUnread(); 2808 } 2809 } 2810 } 2811 } 2812 2813 ByteArrayHashMapEx<ArrayList<DownloadInfo>> related_content_map = content_cache.related_content_map; 2814 2815 List<byte[]> delete = new ArrayList<byte[]>(); 2816 2817 for ( byte[] key: related_content_map.keys()){ 2818 2819 ArrayList<DownloadInfo> infos = related_content_map.get( key ); 2820 2821 for ( RelatedContent c: content ){ 2822 2823 if ( infos.remove( c )){ 2824 2825 if ( infos.size() == 0 ){ 2826 2827 delete.add( key ); 2828 2829 break; 2830 } 2831 } 2832 } 2833 } 2834 2835 for ( byte[] key: delete ){ 2836 2837 related_content_map.remove( key ); 2838 } 2839 2840 setConfigDirty(); 2841 2842 content_change_dispatcher.dispatch( 2843 new AERunnable() 2844 { 2845 public void 2846 runSupport() 2847 { 2848 for ( RelatedContentManagerListener l: listeners ){ 2849 2850 try{ 2851 l.contentRemoved( content ); 2852 2853 }catch( Throwable e ){ 2854 2855 Debug.out( e ); 2856 } 2857 } 2858 } 2859 }); 2860 } 2861 2862 protected String getPrivateInfoKey( RelatedContent info )2863 getPrivateInfoKey( 2864 RelatedContent info ) 2865 { 2866 return( info.getTitle() + ":" + info.getTracker()); 2867 } 2868 2869 protected void analyseResponse( DownloadInfo to_info, final RelatedContentManagerListener listener )2870 analyseResponse( 2871 DownloadInfo to_info, 2872 final RelatedContentManagerListener listener ) 2873 { 2874 try{ 2875 synchronized( rcm_lock ){ 2876 2877 byte[] target = to_info.getHash(); 2878 2879 String key; 2880 2881 if ( target != null ){ 2882 2883 if ( download_info_map.containsKey( target )){ 2884 2885 // target refers to downoad we already have 2886 2887 return; 2888 } 2889 2890 key = Base32.encode( target ); 2891 2892 }else{ 2893 2894 key = getPrivateInfoKey( to_info ); 2895 2896 if ( download_priv_set.contains( key )){ 2897 2898 // target refers to downoad we already have 2899 2900 return; 2901 } 2902 } 2903 2904 if ( isPersistentlyDeleted( to_info )){ 2905 2906 return; 2907 } 2908 2909 ContentCache content_cache = loadRelatedContent(); 2910 2911 DownloadInfo target_info = null; 2912 2913 boolean changed_content = false; 2914 boolean new_content = false; 2915 2916 2917 target_info = content_cache.related_content.get( key ); 2918 2919 if ( target_info == null ){ 2920 2921 if ( enoughSpaceFor( content_cache, to_info )){ 2922 2923 target_info = to_info; 2924 2925 content_cache.related_content.put( key, target_info ); 2926 2927 byte[] from_hash = to_info.getRelatedToHash(); 2928 2929 ArrayList<DownloadInfo> links = content_cache.related_content_map.get( from_hash ); 2930 2931 if ( links == null ){ 2932 2933 links = new ArrayList<DownloadInfo>(1); 2934 2935 content_cache.related_content_map.put( from_hash, links ); 2936 } 2937 2938 links.add( target_info ); 2939 2940 links.trimToSize(); 2941 2942 target_info.setPublic( content_cache ); 2943 2944 if ( secondary_lookups.size() < SECONDARY_LOOKUP_CACHE_MAX ){ 2945 2946 byte[] hash = target_info.getHash(); 2947 int level = target_info.getLevel(); 2948 2949 if ( hash != null && level < max_search_level ){ 2950 2951 secondary_lookups.add( new SecondaryLookup( hash, level, target_info.getNetworksInternal())); 2952 } 2953 } 2954 2955 new_content = true; 2956 2957 }else{ 2958 2959 transient_info_cache.put( key, to_info ); 2960 } 2961 }else{ 2962 2963 // we already know about this, see if new info - ignore lower versions 2964 2965 if ( to_info.getVersion() >= target_info.getVersion()){ 2966 2967 changed_content = target_info.addInfo( to_info ); 2968 } 2969 } 2970 2971 if ( target_info != null ){ 2972 2973 final RelatedContent[] f_target = new RelatedContent[]{ target_info }; 2974 final boolean f_change = changed_content; 2975 2976 final boolean something_changed = changed_content || new_content; 2977 2978 if ( something_changed ){ 2979 2980 setConfigDirty(); 2981 } 2982 2983 content_change_dispatcher.dispatch( 2984 new AERunnable() 2985 { 2986 public void 2987 runSupport() 2988 { 2989 if ( something_changed ){ 2990 2991 for ( RelatedContentManagerListener l: listeners ){ 2992 2993 try{ 2994 if ( f_change ){ 2995 2996 l.contentChanged( f_target ); 2997 2998 }else{ 2999 3000 l.contentFound( f_target ); 3001 } 3002 }catch( Throwable e ){ 3003 3004 Debug.out( e ); 3005 } 3006 } 3007 } 3008 3009 if ( listener != null ){ 3010 3011 try{ 3012 if ( f_change ){ 3013 3014 listener.contentChanged( f_target ); 3015 3016 }else{ 3017 3018 listener.contentFound( f_target ); 3019 } 3020 }catch( Throwable e ){ 3021 3022 Debug.out( e ); 3023 } 3024 } 3025 } 3026 }); 3027 } 3028 } 3029 3030 }catch( Throwable e ){ 3031 3032 Debug.out( e ); 3033 } 3034 } 3035 3036 protected boolean enoughSpaceFor( ContentCache content_cache, DownloadInfo fi )3037 enoughSpaceFor( 3038 ContentCache content_cache, 3039 DownloadInfo fi ) 3040 { 3041 Map<String,DownloadInfo> related_content = content_cache.related_content; 3042 3043 if ( related_content.size() < max_results + temporary_space.get()){ 3044 3045 return( true ); 3046 } 3047 3048 Iterator<Map.Entry<String,DownloadInfo>> it = related_content.entrySet().iterator(); 3049 3050 int max_level = fi.getLevel(); 3051 3052 // delete oldest at highest level >= level with minimum rank 3053 3054 Map<Integer,DownloadInfo> oldest_per_rank = new HashMap<Integer, DownloadInfo>(); 3055 3056 int min_rank = Integer.MAX_VALUE; 3057 int max_rank = -1; 3058 3059 while( it.hasNext()){ 3060 3061 Map.Entry<String,DownloadInfo> entry = it.next(); 3062 3063 DownloadInfo info = entry.getValue(); 3064 3065 if ( info.isExplicit()){ 3066 3067 continue; 3068 } 3069 3070 int info_level = info.getLevel(); 3071 3072 if ( info_level >= max_level ){ 3073 3074 if ( info_level > max_level ){ 3075 3076 max_level = info_level; 3077 3078 min_rank = Integer.MAX_VALUE; 3079 max_rank = -1; 3080 3081 oldest_per_rank.clear(); 3082 } 3083 3084 int rank = info.getRank(); 3085 3086 if ( rank < min_rank ){ 3087 3088 min_rank = rank; 3089 3090 }else if ( rank > max_rank ){ 3091 3092 max_rank = rank; 3093 } 3094 3095 DownloadInfo oldest = oldest_per_rank.get( rank ); 3096 3097 if ( oldest == null ){ 3098 3099 oldest_per_rank.put( rank, info ); 3100 3101 }else{ 3102 3103 if ( info.getLastSeenSecs() < oldest.getLastSeenSecs()){ 3104 3105 oldest_per_rank.put( rank, info ); 3106 } 3107 } 3108 } 3109 } 3110 3111 DownloadInfo to_remove = oldest_per_rank.get( min_rank ); 3112 3113 if ( to_remove != null ){ 3114 3115 delete( new RelatedContent[]{ to_remove }, content_cache, false ); 3116 3117 return( true ); 3118 } 3119 3120 // we don't want high-ranked entries to get stuck there and prevent newer stuff from getting in and rising up 3121 3122 if ( max_level == 1 ){ 3123 3124 to_remove = oldest_per_rank.get( max_rank ); 3125 3126 if ( to_remove != null ){ 3127 3128 int now_secs = (int)( SystemTime.getCurrentTime()/1000 ); 3129 3130 // give it a day at the top 3131 3132 if ( now_secs - to_remove.getLastSeenSecs() >= 24*60*60 ){ 3133 3134 delete( new RelatedContent[]{ to_remove }, content_cache, false ); 3135 3136 return( true ); 3137 } 3138 } 3139 } 3140 3141 return( false ); 3142 } 3143 3144 public RelatedContent[] getRelatedContent()3145 getRelatedContent() 3146 { 3147 synchronized( rcm_lock ){ 3148 3149 ContentCache content_cache = loadRelatedContent(); 3150 3151 return( content_cache.related_content.values().toArray( new DownloadInfo[ content_cache.related_content.size()])); 3152 } 3153 } 3154 3155 protected List<DownloadInfo> getRelatedContentAsList()3156 getRelatedContentAsList() 3157 { 3158 synchronized( rcm_lock ){ 3159 3160 ContentCache content_cache = loadRelatedContent(); 3161 3162 return( new ArrayList<DownloadInfo>( content_cache.related_content.values())); 3163 } 3164 } 3165 3166 public void reset()3167 reset() 3168 { 3169 reset( true ); 3170 } 3171 3172 protected void reset( boolean reset_perm_dels )3173 reset( 3174 boolean reset_perm_dels ) 3175 { 3176 synchronized( rcm_lock ){ 3177 3178 ContentCache cc = content_cache==null?null:content_cache.get(); 3179 3180 if ( cc == null ){ 3181 3182 FileUtil.deleteResilientConfigFile( CONFIG_FILE ); 3183 3184 }else{ 3185 3186 cc.related_content = new HashMap<String,DownloadInfo>(); 3187 cc.related_content_map = new ByteArrayHashMapEx<ArrayList<DownloadInfo>>(); 3188 } 3189 3190 pub_download_infos1.clear(); 3191 pub_download_infos2.clear(); 3192 3193 non_pub_download_infos1.clear(); 3194 non_pub_download_infos2.clear(); 3195 3196 List<DownloadInfo> list = download_info_map.values(); 3197 3198 for ( DownloadInfo info: list ){ 3199 3200 byte nets = info.getNetworksInternal(); 3201 3202 if ( nets != NET_NONE ){ 3203 3204 if (( nets & NET_PUBLIC ) != 0 ){ 3205 3206 pub_download_infos1.add( info ); 3207 pub_download_infos2.add( info ); 3208 3209 }else{ 3210 3211 non_pub_download_infos1.add( info ); 3212 non_pub_download_infos2.add( info ); 3213 } 3214 } 3215 } 3216 3217 Collections.shuffle( pub_download_infos1 ); 3218 Collections.shuffle( non_pub_download_infos1 ); 3219 3220 total_unread.set( 0 ); 3221 3222 if ( reset_perm_dels ){ 3223 3224 resetPersistentlyDeleted(); 3225 } 3226 3227 setConfigDirty(); 3228 } 3229 3230 content_change_dispatcher.dispatch( 3231 new AERunnable() 3232 { 3233 public void 3234 runSupport() 3235 { 3236 for ( RelatedContentManagerListener l: listeners ){ 3237 3238 l.contentReset(); 3239 } 3240 } 3241 }); 3242 } 3243 3244 public SearchInstance searchRCM( Map<String,Object> search_parameters, SearchObserver observer )3245 searchRCM( 3246 Map<String,Object> search_parameters, 3247 SearchObserver observer ) 3248 3249 throws SearchException 3250 { 3251 initialisation_complete_sem.reserve(); 3252 3253 if ( !enabled ){ 3254 3255 throw( new SearchException( "rcm is disabled" )); 3256 } 3257 3258 String[] networks = (String[])search_parameters.get( SearchProvider.SP_NETWORKS ); 3259 3260 String target_net = AENetworkClassifier.AT_PUBLIC; 3261 3262 if ( networks != null ){ 3263 3264 for ( String net: networks ){ 3265 3266 if ( net == AENetworkClassifier.AT_PUBLIC ){ 3267 3268 target_net = AENetworkClassifier.AT_PUBLIC; 3269 3270 break; 3271 3272 }else if ( net == AENetworkClassifier.AT_I2P ){ 3273 3274 target_net = AENetworkClassifier.AT_I2P; 3275 } 3276 } 3277 } 3278 3279 if ( target_net == AENetworkClassifier.AT_I2P ){ 3280 3281 checkI2PSearcher( true ); 3282 } 3283 3284 for ( RelatedContentSearcher searcher: searchers ){ 3285 3286 String net = searcher.getDHTPlugin().getNetwork(); 3287 3288 if ( net == target_net ){ 3289 3290 return( searcher.searchRCM( search_parameters, observer )); 3291 } 3292 } 3293 3294 throw( new SearchException( "no searchers available" )); 3295 } 3296 3297 3298 protected void setConfigDirty()3299 setConfigDirty() 3300 { 3301 synchronized( rcm_lock ){ 3302 3303 content_dirty = true; 3304 } 3305 } 3306 3307 protected ContentCache loadRelatedContent()3308 loadRelatedContent() 3309 { 3310 boolean fire_event = false; 3311 3312 try{ 3313 synchronized( rcm_lock ){ 3314 3315 last_config_access = SystemTime.getMonotonousTime(); 3316 3317 ContentCache cc = content_cache==null?null:content_cache.get(); 3318 3319 if ( cc == null ){ 3320 3321 if ( TRACE ){ 3322 System.out.println( "rcm: load new" ); 3323 } 3324 3325 fire_event = true; 3326 3327 cc = new ContentCache(); 3328 3329 content_cache = new WeakReference<ContentCache>( cc ); 3330 3331 try{ 3332 int new_total_unread = 0; 3333 3334 if ( FileUtil.resilientConfigFileExists( CONFIG_FILE )){ 3335 3336 Map map = FileUtil.readResilientConfigFile( CONFIG_FILE ); 3337 3338 Map<String,DownloadInfo> related_content = cc.related_content; 3339 ByteArrayHashMapEx<ArrayList<DownloadInfo>> related_content_map = cc.related_content_map; 3340 3341 Map<String,String> rcm_map; 3342 3343 byte[] data = (byte[])map.get( "d" ); 3344 3345 if ( data != null ){ 3346 3347 try{ 3348 map = BDecoder.decode(new BufferedInputStream( new GZIPInputStream( new ByteArrayInputStream( CryptoManagerFactory.getSingleton().deobfuscate( data ))))); 3349 3350 }catch( Throwable e ){ 3351 3352 // can get here is config's been deleted 3353 3354 map = new HashMap(); 3355 } 3356 } 3357 3358 rcm_map = (Map<String,String>)map.get( "rcm" ); 3359 3360 Object rc_map_stuff = map.get( "rc" ); 3361 3362 if ( rc_map_stuff != null && rcm_map != null ){ 3363 3364 Map<Integer,DownloadInfo> id_map = new HashMap<Integer, DownloadInfo>(); 3365 3366 if ( rc_map_stuff instanceof Map ){ 3367 3368 // migration from when it was a Map with non-ascii key issues 3369 3370 Map<String,Map<String,Object>> rc_map = (Map<String,Map<String,Object>>)rc_map_stuff; 3371 3372 for ( Map.Entry<String,Map<String,Object>> entry: rc_map.entrySet()){ 3373 3374 try{ 3375 3376 String key = entry.getKey(); 3377 3378 Map<String,Object> info_map = entry.getValue(); 3379 3380 DownloadInfo info = deserialiseDI( info_map, cc ); 3381 3382 if ( info.isUnread()){ 3383 3384 new_total_unread++; 3385 } 3386 3387 related_content.put( key, info ); 3388 3389 int id = ((Long)info_map.get( "_i" )).intValue(); 3390 3391 id_map.put( id, info ); 3392 3393 }catch( Throwable e ){ 3394 3395 Debug.out( e ); 3396 } 3397 } 3398 }else{ 3399 3400 List<Map<String,Object>> rc_map_list = (List<Map<String,Object>>)rc_map_stuff; 3401 3402 for ( Map<String,Object> info_map: rc_map_list ){ 3403 3404 try{ 3405 3406 String key = new String((byte[])info_map.get( "_k" ), "UTF-8" ); 3407 3408 DownloadInfo info = deserialiseDI( info_map, cc ); 3409 3410 if ( info.isUnread()){ 3411 3412 new_total_unread++; 3413 } 3414 3415 related_content.put( key, info ); 3416 3417 int id = ((Long)info_map.get( "_i" )).intValue(); 3418 3419 id_map.put( id, info ); 3420 3421 }catch( Throwable e ){ 3422 3423 Debug.out( e ); 3424 } 3425 } 3426 } 3427 3428 if ( rcm_map.size() != 0 && id_map.size() != 0 ){ 3429 3430 for ( String key: rcm_map.keySet()){ 3431 3432 try{ 3433 byte[] hash = Base32.decode( key ); 3434 3435 int[] ids = ImportExportUtils.importIntArray( rcm_map, key ); 3436 3437 if ( ids == null || ids.length == 0 ){ 3438 3439 // Debug.out( "Inconsistent - no ids" ); 3440 3441 }else{ 3442 3443 ArrayList<DownloadInfo> di_list = new ArrayList<DownloadInfo>(ids.length); 3444 3445 for ( int id: ids ){ 3446 3447 DownloadInfo di = id_map.get( id ); 3448 3449 if ( di == null ){ 3450 3451 // Debug.out( "Inconsistent: id " + id + " missing" ); 3452 3453 }else{ 3454 3455 // we don't currently remember all originators, just one that works 3456 3457 di.setRelatedToHash( hash ); 3458 3459 di_list.add( di ); 3460 } 3461 } 3462 3463 if ( di_list.size() > 0 ){ 3464 3465 related_content_map.put( hash, di_list ); 3466 } 3467 } 3468 }catch( Throwable e ){ 3469 3470 Debug.out( e ); 3471 } 3472 } 3473 } 3474 3475 Iterator<DownloadInfo> it = related_content.values().iterator(); 3476 3477 while( it.hasNext()){ 3478 3479 DownloadInfo di = it.next(); 3480 3481 if ( di.getRelatedToHash() == null ){ 3482 3483 // Debug.out( "Inconsistent: info not referenced" ); 3484 3485 if ( di.isUnread()){ 3486 3487 new_total_unread--; 3488 } 3489 3490 it.remove(); 3491 } 3492 } 3493 3494 popuplateSecondaryLookups( cc ); 3495 } 3496 } 3497 3498 if ( total_unread.get() != new_total_unread ){ 3499 3500 // Debug.out( "total_unread - inconsistent (" + total_unread + "/" + new_total_unread + ")" ); 3501 3502 total_unread.set( new_total_unread ); 3503 3504 COConfigurationManager.setParameter( CONFIG_TOTAL_UNREAD, new_total_unread ); 3505 } 3506 }catch( Throwable e ){ 3507 3508 Debug.out( e ); 3509 } 3510 3511 enforceMaxResults( cc, false ); 3512 } 3513 3514 content_cache_ref = cc; 3515 3516 return( cc ); 3517 } 3518 }finally{ 3519 3520 if ( fire_event ){ 3521 3522 contentChanged( false ); 3523 } 3524 } 3525 } 3526 3527 protected void saveRelatedContent( int tick_count )3528 saveRelatedContent( 3529 int tick_count ) 3530 { 3531 synchronized( rcm_lock ){ 3532 3533 COConfigurationManager.setParameter( CONFIG_TOTAL_UNREAD, total_unread.get()); 3534 3535 long now = SystemTime.getMonotonousTime();; 3536 3537 ContentCache cc = content_cache==null?null:content_cache.get(); 3538 3539 if ( !content_dirty ){ 3540 3541 if ( cc != null ){ 3542 3543 if ( now - last_config_access > CONFIG_DISCARD_MILLIS ){ 3544 3545 if ( content_cache_ref != null ){ 3546 3547 content_discard_ticks = 0; 3548 } 3549 3550 if ( TRACE ){ 3551 System.out.println( "rcm: discard: tick count=" + content_discard_ticks++ ); 3552 } 3553 3554 content_cache_ref = null; 3555 } 3556 }else{ 3557 3558 if ( TRACE ){ 3559 System.out.println( "rcm: discarded" ); 3560 } 3561 } 3562 3563 return; 3564 } 3565 3566 if ( tick_count % CONFIG_SAVE_TICKS != 0 ){ 3567 3568 return; 3569 } 3570 3571 last_config_access = now; 3572 3573 content_dirty = false; 3574 3575 if ( cc == null ){ 3576 3577 // Debug.out( "RCM: cache inconsistent" ); 3578 3579 }else{ 3580 3581 if ( persist ){ 3582 3583 if ( TRACE ){ 3584 System.out.println( "rcm: save" ); 3585 } 3586 3587 Map<String,DownloadInfo> related_content = cc.related_content; 3588 ByteArrayHashMapEx<ArrayList<DownloadInfo>> related_content_map = cc.related_content_map; 3589 3590 if ( related_content.size() == 0 ){ 3591 3592 FileUtil.deleteResilientConfigFile( CONFIG_FILE ); 3593 3594 }else{ 3595 3596 Map<String,Object> map = new HashMap<String, Object>(); 3597 3598 Set<Map.Entry<String,DownloadInfo>> rcs = related_content.entrySet(); 3599 3600 List<Map<String,Object>> rc_map_list = new ArrayList<Map<String, Object>>( rcs.size()); 3601 3602 map.put( "rc", rc_map_list ); 3603 3604 int id = 0; 3605 3606 Map<DownloadInfo,Integer> info_map = new HashMap<DownloadInfo, Integer>(); 3607 3608 for ( Map.Entry<String,DownloadInfo> entry: rcs ){ 3609 3610 DownloadInfo info = entry.getValue(); 3611 3612 Map<String,Object> di_map = serialiseDI( info, cc ); 3613 3614 if ( di_map != null ){ 3615 3616 info_map.put( info, id ); 3617 3618 di_map.put( "_i", new Long( id )); 3619 di_map.put( "_k", entry.getKey()); 3620 3621 rc_map_list.add( di_map ); 3622 3623 id++; 3624 } 3625 } 3626 3627 Map<String,Object> rcm_map = new HashMap<String, Object>(); 3628 3629 map.put( "rcm", rcm_map ); 3630 3631 for ( byte[] hash: related_content_map.keys()){ 3632 3633 List<DownloadInfo> dis = related_content_map.get( hash ); 3634 3635 int[] ids = new int[dis.size()]; 3636 3637 int pos = 0; 3638 3639 for ( DownloadInfo di: dis ){ 3640 3641 Integer index = info_map.get( di ); 3642 3643 if ( index == null ){ 3644 3645 // Debug.out( "inconsistent: info missing for " + di ); 3646 3647 break; 3648 3649 }else{ 3650 3651 ids[pos++] = index; 3652 } 3653 } 3654 3655 if ( pos == ids.length ){ 3656 3657 ImportExportUtils.exportIntArray( rcm_map, Base32.encode( hash), ids ); 3658 } 3659 } 3660 3661 if ( true ){ 3662 3663 ByteArrayOutputStream baos = new ByteArrayOutputStream( 100*1024 ); 3664 3665 try{ 3666 GZIPOutputStream gos = new GZIPOutputStream( baos ); 3667 3668 gos.write( BEncoder.encode( map )); 3669 3670 gos.close(); 3671 3672 }catch( Throwable e ){ 3673 3674 Debug.out( e ); 3675 } 3676 3677 map.clear(); 3678 3679 map.put( "d", CryptoManagerFactory.getSingleton().obfuscate( baos.toByteArray())); 3680 } 3681 3682 FileUtil.writeResilientConfigFile( CONFIG_FILE, map ); 3683 } 3684 }else{ 3685 3686 deleteRelatedContent(); 3687 } 3688 3689 for ( RelatedContentSearcher searcher: searchers ){ 3690 3691 searcher.updateKeyBloom( cc ); 3692 } 3693 } 3694 } 3695 } 3696 3697 private void deleteRelatedContent()3698 deleteRelatedContent() 3699 { 3700 FileUtil.deleteResilientConfigFile( CONFIG_FILE ); 3701 FileUtil.deleteResilientConfigFile( PERSIST_DEL_FILE ); 3702 } 3703 3704 public int getNumUnread()3705 getNumUnread() 3706 { 3707 return( total_unread.get()); 3708 } 3709 3710 public void setAllRead()3711 setAllRead() 3712 { 3713 boolean changed = false; 3714 3715 synchronized( rcm_lock ){ 3716 3717 DownloadInfo[] content = (DownloadInfo[])getRelatedContent(); 3718 3719 for ( DownloadInfo c: content ){ 3720 3721 if ( c.isUnread()){ 3722 3723 changed = true; 3724 3725 c.setUnreadInternal( false ); 3726 } 3727 } 3728 3729 total_unread.set( 0 ); 3730 } 3731 3732 if ( changed ){ 3733 3734 contentChanged( true ); 3735 } 3736 } 3737 3738 public void deleteAll()3739 deleteAll() 3740 { 3741 synchronized( rcm_lock ){ 3742 3743 ContentCache content_cache = loadRelatedContent(); 3744 3745 addPersistentlyDeleted( content_cache.related_content.values().toArray( new DownloadInfo[ content_cache.related_content.size()])); 3746 3747 reset( false ); 3748 } 3749 } 3750 3751 protected void incrementUnread()3752 incrementUnread() 3753 { 3754 total_unread.incrementAndGet(); 3755 } 3756 3757 protected void decrementUnread()3758 decrementUnread() 3759 { 3760 synchronized( rcm_lock ){ 3761 3762 int val = total_unread.decrementAndGet(); 3763 3764 if ( val < 0 ){ 3765 3766 // Debug.out( "inconsistent" ); 3767 3768 total_unread.set( 0 ); 3769 } 3770 } 3771 } 3772 3773 protected Download getDownload( byte[] hash )3774 getDownload( 3775 byte[] hash ) 3776 { 3777 try{ 3778 return( plugin_interface.getDownloadManager().getDownload( hash )); 3779 3780 }catch( Throwable e ){ 3781 3782 return( null ); 3783 } 3784 } 3785 3786 private byte[][] getKeys( Download download )3787 getKeys( 3788 Download download ) 3789 { 3790 byte[] tracker_keys = null; 3791 byte[] ws_keys = null; 3792 3793 try{ 3794 Torrent torrent = download.getTorrent(); 3795 3796 if ( torrent != null ){ 3797 3798 TOTorrent to_torrent = PluginCoreUtils.unwrap( torrent ); 3799 3800 Set<String> tracker_domains = new HashSet<String>(); 3801 3802 addURLToDomainKeySet( tracker_domains, to_torrent.getAnnounceURL()); 3803 3804 TOTorrentAnnounceURLGroup group = to_torrent.getAnnounceURLGroup(); 3805 3806 TOTorrentAnnounceURLSet[] sets = group.getAnnounceURLSets(); 3807 3808 for ( TOTorrentAnnounceURLSet set: sets ){ 3809 3810 URL[] urls = set.getAnnounceURLs(); 3811 3812 for ( URL u: urls ){ 3813 3814 addURLToDomainKeySet( tracker_domains, u ); 3815 } 3816 } 3817 3818 tracker_keys = domainsToArray( tracker_domains, 8 ); 3819 3820 Set<String> ws_domains = new HashSet<String>(); 3821 3822 List getright = BDecoder.decodeStrings( getURLList( to_torrent, "url-list" )); 3823 List webseeds = BDecoder.decodeStrings( getURLList( to_torrent, "httpseeds" )); 3824 3825 for ( List l: new List[]{ getright, webseeds }){ 3826 3827 for ( Object o: l ){ 3828 3829 if ( o instanceof String ){ 3830 3831 try{ 3832 addURLToDomainKeySet( ws_domains, new URL((String)o)); 3833 3834 }catch( Throwable e ){ 3835 3836 } 3837 } 3838 } 3839 } 3840 3841 ws_keys = domainsToArray( ws_domains, 3 ); 3842 } 3843 }catch( Throwable e ){ 3844 } 3845 3846 return( new byte[][]{ tracker_keys, ws_keys }); 3847 } 3848 3849 protected byte[] domainsToArray( Set<String> domains, int max )3850 domainsToArray( 3851 Set<String> domains, 3852 int max ) 3853 { 3854 int entries = Math.min( domains.size(), max ); 3855 3856 if ( entries > 0 ){ 3857 3858 byte[] keys = new byte[ entries*4 ]; 3859 3860 int pos = 0; 3861 3862 for ( String dom: domains ){ 3863 3864 int hash = dom.hashCode(); 3865 3866 byte[] bytes = { (byte)(hash>>24), (byte)(hash>>16),(byte)(hash>>8),(byte)hash }; 3867 3868 System.arraycopy( bytes, 0, keys, pos, 4 ); 3869 3870 pos += 4; 3871 } 3872 3873 return( keys ); 3874 } 3875 3876 return( null ); 3877 } 3878 3879 protected List getURLList( TOTorrent torrent, String key )3880 getURLList( 3881 TOTorrent torrent, 3882 String key ) 3883 { 3884 Object obj = torrent.getAdditionalProperty( key ); 3885 3886 if ( obj instanceof byte[] ){ 3887 3888 List l = new ArrayList(); 3889 3890 l.add(obj); 3891 3892 return( l ); 3893 3894 }else if ( obj instanceof List ){ 3895 3896 return (List)BEncoder.clone(obj); 3897 3898 }else{ 3899 3900 return( new ArrayList()); 3901 } 3902 } 3903 3904 private void addURLToDomainKeySet( Set<String> set, URL u )3905 addURLToDomainKeySet( 3906 Set<String> set, 3907 URL u ) 3908 { 3909 String prot = u.getProtocol(); 3910 3911 if ( prot != null ){ 3912 3913 if ( prot.equalsIgnoreCase( "http" ) || prot.equalsIgnoreCase( "udp" )){ 3914 3915 String host = u.getHost().toLowerCase( Locale.US ); 3916 3917 if ( host.contains( ":" )){ 3918 3919 // ipv6 raw 3920 3921 return; 3922 } 3923 3924 String[] bits = host.split( "\\." ); 3925 3926 int len = bits.length; 3927 3928 if ( len >= 2 ){ 3929 3930 String end = bits[len-1]; 3931 3932 char[] chars = end.toCharArray(); 3933 3934 // simple check for ipv4 raw 3935 3936 boolean all_digits = true; 3937 3938 for ( char c: chars ){ 3939 3940 if ( !Character.isDigit( c )){ 3941 3942 all_digits = false; 3943 3944 break; 3945 } 3946 } 3947 3948 if ( !all_digits ){ 3949 3950 set.add( bits[len-2] + "." + end ); 3951 } 3952 } 3953 } 3954 } 3955 } 3956 3957 private byte getNetworks( Download download )3958 getNetworks( 3959 Download download ) 3960 { 3961 String[] networks = download.getListAttribute( ta_networks ); 3962 3963 if ( networks == null ){ 3964 3965 return( NET_NONE ); 3966 3967 }else{ 3968 3969 return( convertNetworks( networks )); 3970 } 3971 } 3972 3973 public static String[] convertNetworks( byte net )3974 convertNetworks( 3975 byte net ) 3976 { 3977 if ( net == NET_NONE ){ 3978 return( new String[0] ); 3979 }else if ( net == NET_PUBLIC ){ 3980 return( NET_PUBLIC_ARRAY ); 3981 }else if ( net == NET_I2P ){ 3982 return( NET_I2P_ARRAY ); 3983 }else if ( net == NET_TOR ){ 3984 return( NET_TOR_ARRAY ); 3985 }else if ( net == (NET_PUBLIC | NET_I2P )){ 3986 return( NET_PUBLIC_AND_I2P_ARRAY ); 3987 }else{ 3988 List<String> nets = new ArrayList<String>(); 3989 3990 if (( net & NET_PUBLIC ) != 0 ){ 3991 nets.add( AENetworkClassifier.AT_PUBLIC ); 3992 } 3993 if (( net & NET_I2P ) != 0 ){ 3994 nets.add( AENetworkClassifier.AT_I2P ); 3995 } 3996 if (( net & NET_TOR ) != 0 ){ 3997 nets.add( AENetworkClassifier.AT_TOR ); 3998 } 3999 4000 return( nets.toArray( new String[ nets.size()])); 4001 } 4002 } 4003 4004 public static byte convertNetworks( String[] networks )4005 convertNetworks( 4006 String[] networks ) 4007 { 4008 byte nets = NET_NONE; 4009 4010 for ( int i=0;i<networks.length;i++ ){ 4011 4012 String n = networks[i]; 4013 4014 if (n.equalsIgnoreCase( AENetworkClassifier.AT_PUBLIC )){ 4015 4016 nets |= NET_PUBLIC; 4017 4018 }else if ( n.equalsIgnoreCase( AENetworkClassifier.AT_I2P )){ 4019 4020 nets |= NET_I2P; 4021 4022 }else if ( n.equalsIgnoreCase( AENetworkClassifier.AT_TOR )){ 4023 4024 nets |= NET_TOR; 4025 } 4026 } 4027 4028 return( nets ); 4029 } 4030 4031 private String[] getTags( Download download )4032 getTags( 4033 Download download ) 4034 { 4035 Set<String> all_tags = new HashSet<String>(); 4036 4037 if ( tag_manager.isEnabled()){ 4038 4039 String cat_name = ta_category==null?null:download.getAttribute( ta_category ); 4040 4041 if ( cat_name != null ){ 4042 4043 Tag cat_tag = tag_manager.getTagType( TagType.TT_DOWNLOAD_CATEGORY ).getTag( cat_name, true ); 4044 4045 if ( cat_tag != null && cat_tag.isPublic()){ 4046 4047 all_tags.add( cat_name.toLowerCase( Locale.US )); 4048 } 4049 } 4050 4051 List<Tag> tags = tag_manager.getTagType( TagType.TT_DOWNLOAD_MANUAL ).getTagsForTaggable( PluginCoreUtils.unwrap( download )); 4052 4053 for ( Tag t: tags ){ 4054 4055 if ( t.isPublic()){ 4056 4057 all_tags.add( t.getTagName( true ).toLowerCase( Locale.US )); 4058 } 4059 } 4060 } 4061 4062 String[] networks = download.getListAttribute( ta_networks ); 4063 4064 for ( String network: networks ){ 4065 4066 if ( !network.equals( "Public" )){ 4067 4068 if ( AEPluginProxyHandler.hasPluginProxyForNetwork( network, true )){ 4069 4070 all_tags.add( "_" + network.toLowerCase( Locale.US ) + "_" ); 4071 } 4072 } 4073 } 4074 4075 if ( all_tags.size() == 0 ){ 4076 4077 return( null ); 4078 4079 }else if ( all_tags.size() == 1 ){ 4080 4081 return( new String[]{ all_tags.iterator().next()}); 4082 4083 }else{ 4084 4085 List<String> temp = new ArrayList<String>( all_tags ); 4086 4087 Collections.shuffle( temp ); 4088 4089 return( temp.toArray( new String[ temp.size()] )); 4090 } 4091 } 4092 4093 private static final int MAX_TAG_LENGTH = 20; 4094 private static final int MAX_TAGS_TOTAL_LENGTH = 64; 4095 4096 protected byte[] encodeTags( String[] tags )4097 encodeTags( 4098 String[] tags ) 4099 { 4100 if ( tags == null || tags.length == 0 ){ 4101 4102 return( null ); 4103 } 4104 4105 byte[] temp = new byte[MAX_TAGS_TOTAL_LENGTH]; 4106 int pos = 0; 4107 int rem = temp.length; 4108 4109 for ( int i=0;i<tags.length;i++){ 4110 4111 String tag = tags[i]; 4112 4113 tag = truncateTag( tag ); 4114 4115 try{ 4116 byte[] tag_bytes = tag.getBytes( "UTF-8" ); 4117 4118 int tb_len = tag_bytes.length; 4119 4120 if ( rem < tb_len + 1 ){ 4121 4122 break; 4123 } 4124 4125 temp[pos++] = (byte)tb_len; 4126 4127 System.arraycopy( tag_bytes, 0, temp, pos, tb_len ); 4128 4129 pos += tb_len; 4130 rem -= (tb_len+1); 4131 4132 }catch( Throwable e ){ 4133 4134 } 4135 } 4136 4137 if ( pos == 0 ){ 4138 4139 return( null ); 4140 4141 }else{ 4142 4143 byte[] result = new byte[pos]; 4144 4145 System.arraycopy( temp, 0, result, 0, pos ); 4146 4147 return( result ); 4148 } 4149 } 4150 4151 protected String truncateTag( String tag )4152 truncateTag( 4153 String tag ) 4154 { 4155 if ( tag.length() > MAX_TAG_LENGTH ){ 4156 4157 tag = tag.substring( 0, MAX_TAG_LENGTH ); 4158 } 4159 4160 while( tag.length() > 0 ){ 4161 4162 try{ 4163 byte[] tag_bytes = tag.getBytes( "UTF-8" ); 4164 4165 if ( tag_bytes.length <= MAX_TAG_LENGTH ){ 4166 4167 break; 4168 4169 }else{ 4170 4171 tag = tag.substring( 0, tag.length() - 1 ); 4172 } 4173 4174 }catch( Throwable e ){ 4175 4176 break; 4177 } 4178 } 4179 4180 return( tag ); 4181 } 4182 4183 protected String[] decodeTags( byte[] bytes )4184 decodeTags( 4185 byte[] bytes ) 4186 { 4187 if ( bytes == null || bytes.length == 0 ){ 4188 4189 return( null ); 4190 } 4191 4192 List<String> tags = new ArrayList<String>( 10 ); 4193 4194 int pos = 0; 4195 4196 while( pos < bytes.length ){ 4197 4198 int tag_len = bytes[pos++]&0x000000ff; 4199 4200 if ( tag_len > MAX_TAG_LENGTH ){ 4201 4202 break; 4203 } 4204 4205 try{ 4206 tags.add( new String( bytes, pos, tag_len, "UTF-8" )); 4207 4208 pos += tag_len; 4209 4210 }catch( Throwable e ){ 4211 4212 break; 4213 } 4214 } 4215 4216 if ( tags.size() == 0 ){ 4217 4218 return( null ); 4219 4220 }else{ 4221 4222 return( tags.toArray( new String[ tags.size()] )); 4223 } 4224 } 4225 4226 private static final int PD_BLOOM_INITIAL_SIZE = 1000; 4227 private static final int PD_BLOOM_INCREMENT_SIZE = 1000; 4228 4229 4230 private BloomFilter persist_del_bloom; 4231 4232 protected byte[] getPermDelKey( RelatedContent info )4233 getPermDelKey( 4234 RelatedContent info ) 4235 { 4236 byte[] bytes = info.getHash(); 4237 4238 if ( bytes == null ){ 4239 4240 try{ 4241 bytes = new SHA1Simple().calculateHash( getPrivateInfoKey(info).getBytes( "ISO-8859-1" )); 4242 4243 }catch( Throwable e ){ 4244 4245 Debug.out( e ); 4246 4247 return( null ); 4248 } 4249 } 4250 4251 byte[] key = new byte[8]; 4252 4253 System.arraycopy( bytes, 0, key, 0, 8 ); 4254 4255 return( key ); 4256 } 4257 4258 protected List<byte[]> loadPersistentlyDeleted()4259 loadPersistentlyDeleted() 4260 { 4261 List<byte[]> entries = null; 4262 4263 if ( FileUtil.resilientConfigFileExists( PERSIST_DEL_FILE )){ 4264 4265 Map<String,Object> map = (Map<String,Object>)FileUtil.readResilientConfigFile( PERSIST_DEL_FILE ); 4266 4267 entries = (List<byte[]>)map.get( "entries" ); 4268 } 4269 4270 if ( entries == null ){ 4271 4272 entries = new ArrayList<byte[]>(0); 4273 } 4274 4275 return( entries ); 4276 } 4277 4278 protected void addPersistentlyDeleted( RelatedContent[] content )4279 addPersistentlyDeleted( 4280 RelatedContent[] content ) 4281 { 4282 if ( content.length == 0 ){ 4283 4284 return; 4285 } 4286 4287 List<byte[]> entries = loadPersistentlyDeleted(); 4288 4289 List<byte[]> new_keys = new ArrayList<byte[]>( content.length ); 4290 4291 for ( RelatedContent rc: content ){ 4292 4293 byte[] key = getPermDelKey( rc ); 4294 4295 new_keys.add( key ); 4296 4297 entries.add( key ); 4298 } 4299 4300 Map<String,Object> map = new HashMap<String, Object>(); 4301 4302 map.put( "entries", entries ); 4303 4304 FileUtil.writeResilientConfigFile( PERSIST_DEL_FILE, map ); 4305 4306 if ( persist_del_bloom != null ){ 4307 4308 if ( persist_del_bloom.getSize() / ( persist_del_bloom.getEntryCount() + content.length ) < 10 ){ 4309 4310 persist_del_bloom = BloomFilterFactory.createAddOnly( Math.max( PD_BLOOM_INITIAL_SIZE, persist_del_bloom.getSize() *10 + PD_BLOOM_INCREMENT_SIZE + content.length )); 4311 4312 for ( byte[] k: entries ){ 4313 4314 persist_del_bloom.add( k ); 4315 } 4316 }else{ 4317 4318 for ( byte[] k: new_keys ){ 4319 4320 persist_del_bloom.add( k ); 4321 } 4322 } 4323 } 4324 } 4325 4326 protected boolean isPersistentlyDeleted( RelatedContent content )4327 isPersistentlyDeleted( 4328 RelatedContent content ) 4329 { 4330 if ( persist_del_bloom == null ){ 4331 4332 List<byte[]> entries = loadPersistentlyDeleted(); 4333 4334 persist_del_bloom = BloomFilterFactory.createAddOnly( Math.max( PD_BLOOM_INITIAL_SIZE, entries.size()*10 + PD_BLOOM_INCREMENT_SIZE )); 4335 4336 for ( byte[] k: entries ){ 4337 4338 persist_del_bloom.add( k ); 4339 } 4340 } 4341 4342 byte[] key = getPermDelKey( content ); 4343 4344 return( persist_del_bloom.contains( key )); 4345 } 4346 4347 protected void resetPersistentlyDeleted()4348 resetPersistentlyDeleted() 4349 { 4350 FileUtil.deleteResilientConfigFile( PERSIST_DEL_FILE ); 4351 4352 persist_del_bloom = BloomFilterFactory.createAddOnly( PD_BLOOM_INITIAL_SIZE ); 4353 } 4354 4355 public void reserveTemporarySpace()4356 reserveTemporarySpace() 4357 { 4358 temporary_space.addAndGet( TEMPORARY_SPACE_DELTA ); 4359 } 4360 4361 public void releaseTemporarySpace()4362 releaseTemporarySpace() 4363 { 4364 boolean reset_explicit = temporary_space.addAndGet( -TEMPORARY_SPACE_DELTA ) == 0; 4365 4366 enforceMaxResults( reset_explicit ); 4367 } 4368 4369 protected void enforceMaxResults( boolean reset_explicit )4370 enforceMaxResults( 4371 boolean reset_explicit ) 4372 { 4373 synchronized( rcm_lock ){ 4374 4375 ContentCache content_cache = loadRelatedContent(); 4376 4377 enforceMaxResults( content_cache, reset_explicit ); 4378 } 4379 } 4380 4381 protected void enforceMaxResults( ContentCache content_cache, boolean reset_explicit )4382 enforceMaxResults( 4383 ContentCache content_cache, 4384 boolean reset_explicit ) 4385 { 4386 Map<String,DownloadInfo> related_content = content_cache.related_content; 4387 4388 int num_to_remove = related_content.size() - ( max_results + temporary_space.get()); 4389 4390 if ( num_to_remove > 0 ){ 4391 4392 List<DownloadInfo> infos = new ArrayList<DownloadInfo>(related_content.values()); 4393 4394 if ( reset_explicit ){ 4395 4396 for ( DownloadInfo info: infos ){ 4397 4398 if ( info.isExplicit()){ 4399 4400 info.setExplicit( false ); 4401 } 4402 } 4403 } 4404 4405 Collections.sort( 4406 infos, 4407 new Comparator<DownloadInfo>() 4408 { 4409 public int 4410 compare( 4411 DownloadInfo o1, 4412 DownloadInfo o2) 4413 { 4414 int res = o2.getLevel() - o1.getLevel(); 4415 4416 if ( res != 0 ){ 4417 4418 return( res ); 4419 } 4420 4421 res = o1.getRank() - o2.getRank(); 4422 4423 if ( res != 0 ){ 4424 4425 return( res ); 4426 } 4427 4428 return( o1.getLastSeenSecs() - o2.getLastSeenSecs()); 4429 } 4430 }); 4431 4432 List<RelatedContent> to_remove = new ArrayList<RelatedContent>(); 4433 4434 for (int i=0;i<Math.min( num_to_remove, infos.size());i++ ){ 4435 4436 to_remove.add( infos.get(i)); 4437 } 4438 4439 if ( to_remove.size() > 0 ){ 4440 4441 delete( to_remove.toArray( new RelatedContent[to_remove.size()]), content_cache, false ); 4442 } 4443 } 4444 } 4445 4446 public void addListener( RelatedContentManagerListener listener )4447 addListener( 4448 RelatedContentManagerListener listener ) 4449 { 4450 listeners.add( listener ); 4451 } 4452 4453 public void removeListener( RelatedContentManagerListener listener )4454 removeListener( 4455 RelatedContentManagerListener listener ) 4456 { 4457 listeners.remove( listener ); 4458 } 4459 4460 protected static class 4461 ByteArrayHashMapEx<T> 4462 extends ByteArrayHashMap<T> 4463 { 4464 public T getRandomValueExcluding( T excluded )4465 getRandomValueExcluding( 4466 T excluded ) 4467 { 4468 int num = RandomUtils.nextInt( size ); 4469 4470 T result = null; 4471 4472 for (int j = 0; j < table.length; j++) { 4473 4474 Entry<T> e = table[j]; 4475 4476 while( e != null ){ 4477 4478 T value = e.value; 4479 4480 if ( value != excluded ){ 4481 4482 result = value; 4483 } 4484 4485 if ( num <= 0 && result != null ){ 4486 4487 return( result ); 4488 } 4489 4490 num--; 4491 4492 e = e.next; 4493 } 4494 } 4495 4496 return( result ); 4497 } 4498 } 4499 4500 private Map<String,Object> serialiseDI( DownloadInfo info, ContentCache cc )4501 serialiseDI( 4502 DownloadInfo info, 4503 ContentCache cc ) 4504 { 4505 try{ 4506 Map<String,Object> info_map = new HashMap<String,Object>(); 4507 4508 ImportExportUtils.exportLong( info_map, "v", info.getVersion()); 4509 4510 info_map.put( "h", info.getHash()); 4511 4512 ImportExportUtils.exportString( info_map, "d", info.getTitle()); 4513 ImportExportUtils.exportInt( info_map, "r", info.getRand()); 4514 ImportExportUtils.exportString( info_map, "t", info.getTracker()); 4515 ImportExportUtils.exportLong( info_map, "z", info.getSize()); 4516 4517 ImportExportUtils.exportInt( info_map, "p", (int)( info.getPublishDate()/(60*60*1000))); 4518 ImportExportUtils.exportInt( info_map, "q", (info.getSeeds()<<16)|(info.getLeechers()&0xffff)); 4519 ImportExportUtils.exportInt( info_map, "c", (int)info.getContentNetwork()); 4520 4521 byte[] tracker_keys = info.getTrackerKeys(); 4522 if ( tracker_keys != null ){ 4523 info_map.put( "k", tracker_keys ); 4524 } 4525 4526 byte[] ws_keys = info.getWebSeedKeys(); 4527 if ( ws_keys != null ){ 4528 info_map.put( "w", ws_keys ); 4529 } 4530 4531 String[] tags = info.getTags(); 4532 if ( tags != null ){ 4533 info_map.put( "g", encodeTags(tags)); 4534 } 4535 4536 byte nets = info.getNetworksInternal(); 4537 if (nets != NET_PUBLIC ){ 4538 info_map.put( "o", new Long(nets&0x00ff )); 4539 } 4540 4541 if ( cc != null ){ 4542 4543 ImportExportUtils.exportBoolean( info_map, "u", info.isUnread()); 4544 ImportExportUtils.exportIntArray( info_map, "l", info.getRandList()); 4545 ImportExportUtils.exportInt( info_map, "s", info.getLastSeenSecs()); 4546 ImportExportUtils.exportInt( info_map, "e", info.getLevel()); 4547 } 4548 4549 ImportExportUtils.exportLong(info_map, "cl", info.getChangedLocallyOn()); 4550 4551 return( info_map ); 4552 4553 }catch( Throwable e ){ 4554 4555 Debug.out( e ); 4556 4557 return( null ); 4558 } 4559 } 4560 4561 private DownloadInfo deserialiseDI( Map<String,Object> info_map, ContentCache cc )4562 deserialiseDI( 4563 Map<String,Object> info_map, 4564 ContentCache cc ) 4565 { 4566 try{ 4567 int version = (int)ImportExportUtils.importLong( info_map, "v", RelatedContent.VERSION_INITIAL ); 4568 byte[] hash = (byte[])info_map.get("h"); 4569 String title = ImportExportUtils.importString( info_map, "d" ); 4570 int rand = ImportExportUtils.importInt( info_map, "r" ); 4571 String tracker = ImportExportUtils.importString( info_map, "t" ); 4572 long size = ImportExportUtils.importLong( info_map, "z" ); 4573 4574 int date = ImportExportUtils.importInt( info_map, "p", 0 ); 4575 int seeds_leechers = ImportExportUtils.importInt( info_map, "q", -1 ); 4576 byte cnet = (byte)ImportExportUtils.importInt( info_map, "c", (int)ContentNetwork.CONTENT_NETWORK_UNKNOWN ); 4577 byte[] tracker_keys = (byte[])info_map.get( "k"); 4578 byte[] ws_keys = (byte[])info_map.get( "w" ); 4579 long lastChangedLocally = ImportExportUtils.importLong( info_map, "cl" ); 4580 4581 if ( tracker_keys != null && tracker_keys.length % 4 != 0 ){ 4582 4583 tracker_keys = null; 4584 } 4585 4586 if ( ws_keys != null && ws_keys.length % 4 != 0 ){ 4587 4588 ws_keys = null; 4589 } 4590 4591 byte[] _tags = (byte[])info_map.get( "g" ); 4592 4593 String[] tags = decodeTags( _tags ); 4594 4595 Long _nets = (Long)info_map.get( "o" ); 4596 4597 byte nets = _nets==null?NET_PUBLIC:_nets.byteValue(); 4598 4599 if ( cc == null ){ 4600 4601 DownloadInfo info = new DownloadInfo( version, hash, hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, 0, false, size, date, seeds_leechers, cnet ); 4602 4603 info.setChangedLocallyOn( lastChangedLocally ); 4604 4605 return( info ); 4606 4607 }else{ 4608 4609 boolean unread = ImportExportUtils.importBoolean( info_map, "u" ); 4610 4611 int[] rand_list = ImportExportUtils.importIntArray( info_map, "l" ); 4612 4613 int last_seen = ImportExportUtils.importInt( info_map, "s" ); 4614 4615 int level = ImportExportUtils.importInt( info_map, "e" ); 4616 4617 DownloadInfo info = new DownloadInfo( version, hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, unread, rand_list, last_seen, level, size, date, seeds_leechers, cnet, cc ); 4618 4619 info.setChangedLocallyOn( lastChangedLocally ); 4620 4621 return( info ); 4622 } 4623 }catch( Throwable e ){ 4624 4625 Debug.out( e ); 4626 4627 return( null ); 4628 } 4629 } 4630 4631 private void dump()4632 dump() 4633 { 4634 RelatedContent[] related_content = getRelatedContent(); 4635 4636 ByteArrayHashMap<List<String>> tk_map = new ByteArrayHashMap<List<String>>(); 4637 ByteArrayHashMap<List<String>> ws_map = new ByteArrayHashMap<List<String>>(); 4638 4639 for ( RelatedContent rc: related_content ){ 4640 4641 byte[] tracker_keys = rc.getTrackerKeys(); 4642 4643 if ( tracker_keys != null ){ 4644 4645 for (int i=0;i<tracker_keys.length;i+=4 ){ 4646 4647 byte[] tk = new byte[4]; 4648 4649 System.arraycopy( tracker_keys, i, tk, 0, 4 ); 4650 4651 List<String> titles = tk_map.get( tk ); 4652 4653 if ( titles == null ){ 4654 4655 titles = new ArrayList<String>(); 4656 4657 tk_map.put( tk, titles ); 4658 } 4659 4660 titles.add( rc.getTitle()); 4661 } 4662 } 4663 byte[] ws_keys = rc.getWebSeedKeys(); 4664 4665 if ( ws_keys != null ){ 4666 4667 for (int i=0;i<ws_keys.length;i+=4 ){ 4668 4669 byte[] wk = new byte[4]; 4670 4671 System.arraycopy( ws_keys, i, wk, 0, 4 ); 4672 4673 List<String> titles = ws_map.get( wk ); 4674 4675 if ( titles == null ){ 4676 4677 titles = new ArrayList<String>(); 4678 4679 ws_map.put( wk, titles ); 4680 } 4681 4682 titles.add( rc.getTitle()); 4683 } 4684 } 4685 } 4686 4687 System.out.println( "-- Trackers --" ); 4688 4689 for ( byte[] key: tk_map.keys()){ 4690 4691 List<String> titles = tk_map.get( key ); 4692 4693 System.out.println( ByteFormatter.encodeString( key )); 4694 4695 for ( String title: titles ){ 4696 4697 System.out.println( " " + title ); 4698 } 4699 } 4700 4701 System.out.println( "-- Web Seeds --" ); 4702 4703 for ( byte[] key: ws_map.keys()){ 4704 4705 List<String> titles = ws_map.get( key ); 4706 4707 System.out.println( ByteFormatter.encodeString( key )); 4708 4709 for ( String title: titles ){ 4710 4711 System.out.println( " " + title ); 4712 } 4713 } 4714 } 4715 4716 protected class 4717 DownloadInfo 4718 extends RelatedContent 4719 { 4720 final private int rand; 4721 4722 private boolean unread = true; 4723 private int[] rand_list; 4724 private int last_seen; 4725 private int level; 4726 private boolean explicit; 4727 4728 // we *need* this reference here to manage garbage collection correctly 4729 4730 private ContentCache cc; 4731 4732 protected DownloadInfo( int _version, byte[] _related_to, byte[] _hash, String _title, int _rand, String _tracker, byte[] _tracker_keys, byte[] _ws_keys, String[] _tags, byte _nets, int _level, boolean _explicit, long _size, int _date, int _seeds_leechers, byte _cnet )4733 DownloadInfo( 4734 int _version, 4735 byte[] _related_to, 4736 byte[] _hash, 4737 String _title, 4738 int _rand, 4739 String _tracker, 4740 byte[] _tracker_keys, 4741 byte[] _ws_keys, 4742 String[] _tags, 4743 byte _nets, 4744 int _level, 4745 boolean _explicit, 4746 long _size, 4747 int _date, 4748 int _seeds_leechers, 4749 byte _cnet ) 4750 { 4751 super( _version, _related_to, _title, _hash, _tracker, _tracker_keys, _ws_keys, _tags, _nets, _size, _date, _seeds_leechers, _cnet ); 4752 4753 rand = _rand; 4754 level = _level; 4755 explicit = _explicit; 4756 4757 updateLastSeen(); 4758 } 4759 4760 protected DownloadInfo( int _version, byte[] _hash, String _title, int _rand, String _tracker, byte[] _tracker_keys, byte[] _ws_keys, String[] _tags, byte _nets, boolean _unread, int[] _rand_list, int _last_seen, int _level, long _size, int _date, int _seeds_leechers, byte _cnet, ContentCache _cc )4761 DownloadInfo( 4762 int _version, 4763 byte[] _hash, 4764 String _title, 4765 int _rand, 4766 String _tracker, 4767 byte[] _tracker_keys, 4768 byte[] _ws_keys, 4769 String[] _tags, 4770 byte _nets, 4771 boolean _unread, 4772 int[] _rand_list, 4773 int _last_seen, 4774 int _level, 4775 long _size, 4776 int _date, 4777 int _seeds_leechers, 4778 byte _cnet, 4779 ContentCache _cc ) 4780 { 4781 super( _version, _title, _hash, _tracker, _tracker_keys, _ws_keys, _tags, _nets, _size, _date, _seeds_leechers, _cnet ); 4782 4783 rand = _rand; 4784 unread = _unread; 4785 rand_list = _rand_list; 4786 last_seen = _last_seen; 4787 level = _level; 4788 cc = _cc; 4789 4790 if ( rand_list != null ){ 4791 4792 if ( rand_list.length > MAX_RANK ){ 4793 4794 int[] temp = new int[ MAX_RANK ]; 4795 4796 System.arraycopy( rand_list, 0, temp, 0, MAX_RANK ); 4797 4798 rand_list = temp; 4799 } 4800 } 4801 } 4802 4803 protected boolean addInfo( DownloadInfo info )4804 addInfo( 4805 DownloadInfo info ) 4806 { 4807 boolean result = false; 4808 4809 synchronized( this ){ 4810 4811 updateLastSeen(); 4812 4813 int r = info.getRand(); 4814 4815 if ( rand_list == null ){ 4816 4817 rand_list = new int[]{ r }; 4818 4819 result = true; 4820 4821 }else{ 4822 4823 boolean match = false; 4824 4825 for (int i=0;i<rand_list.length;i++){ 4826 4827 if ( rand_list[i] == r ){ 4828 4829 match = true; 4830 4831 break; 4832 } 4833 } 4834 4835 if ( !match && rand_list.length < MAX_RANK ){ 4836 4837 int len = rand_list.length; 4838 4839 int[] new_rand_list = new int[len+1]; 4840 4841 System.arraycopy( rand_list, 0, new_rand_list, 0, len ); 4842 4843 new_rand_list[len] = r; 4844 4845 rand_list = new_rand_list; 4846 4847 result = true; 4848 } 4849 } 4850 } 4851 4852 if ( info.getVersion() > getVersion()){ 4853 4854 setVersion( info.getVersion()); 4855 4856 result = true; 4857 } 4858 4859 if ( info.getLevel() < level ){ 4860 4861 level = info.getLevel(); 4862 4863 result = true; 4864 } 4865 4866 long cn = info.getContentNetwork(); 4867 4868 if ( cn != ContentNetwork.CONTENT_NETWORK_UNKNOWN && 4869 getContentNetwork() == ContentNetwork.CONTENT_NETWORK_UNKNOWN ){ 4870 4871 setContentNetwork( cn ); 4872 } 4873 4874 if ( info.getVersion() >= getVersion()){ 4875 4876 // don't update seeds/leechers with older version (less accurate) values 4877 4878 int sl = info.getSeedsLeechers(); 4879 4880 if ( sl != -1 && sl != getSeedsLeechers()){ 4881 4882 setSeedsLeechers( sl ); 4883 4884 result = true; 4885 } 4886 } 4887 4888 int d = info.getDateHours(); 4889 4890 if ( d > 0 && getDateHours() == 0 ){ 4891 4892 setDateHours( d ); 4893 4894 result = true; 4895 } 4896 4897 String[] other_tags = info.getTags(); 4898 4899 if ( other_tags != null && other_tags.length > 0 ){ 4900 4901 String[] existing_tags = getTags(); 4902 4903 if ( existing_tags == NO_TAGS ){ 4904 4905 setTags( other_tags ); 4906 4907 result = true; 4908 4909 }else{ 4910 4911 boolean same; 4912 4913 if ( other_tags.length == existing_tags.length ){ 4914 4915 if ( existing_tags.length == 1 ){ 4916 4917 same = other_tags[0].equals( existing_tags[0] ); 4918 4919 }else{ 4920 4921 same = true; 4922 4923 for ( int i=0;i<existing_tags.length;i++ ){ 4924 4925 String e_tag = existing_tags[i]; 4926 4927 boolean found = false; 4928 4929 for ( int j=0;j<other_tags.length;j++){ 4930 4931 if ( e_tag.equals( other_tags[j])){ 4932 4933 found = true; 4934 4935 break; 4936 } 4937 } 4938 4939 if ( !found ){ 4940 4941 same = false; 4942 4943 break; 4944 } 4945 } 4946 } 4947 }else{ 4948 4949 same = false; 4950 } 4951 4952 if ( !same ){ 4953 4954 Set<String> tags = new HashSet<String>(); 4955 4956 for ( String t: existing_tags ){ 4957 tags.add( t ); 4958 } 4959 for ( String t: other_tags ){ 4960 tags.add( t ); 4961 } 4962 4963 setTags( tags.toArray( new String[tags.size()])); 4964 4965 result = true; 4966 } 4967 } 4968 } 4969 4970 byte other_nets = info.getNetworksInternal(); 4971 byte existing_nets = getNetworksInternal(); 4972 4973 if ( other_nets != existing_nets ){ 4974 4975 setNetworksInternal((byte)( other_nets | existing_nets )); 4976 4977 result = true; 4978 } 4979 4980 if (result) { 4981 setChangedLocallyOn(0); 4982 } 4983 return( result ); 4984 } 4985 4986 public int getLevel()4987 getLevel() 4988 { 4989 return( level ); 4990 } 4991 4992 protected boolean isExplicit()4993 isExplicit() 4994 { 4995 return( explicit ); 4996 } 4997 4998 protected void setExplicit( boolean b )4999 setExplicit( 5000 boolean b ) 5001 { 5002 explicit = b; 5003 } 5004 5005 protected void updateLastSeen()5006 updateLastSeen() 5007 { 5008 // persistence of this is piggy-backed on other saves to limit resource usage 5009 // only therefore a vague measure 5010 5011 last_seen = (int)( SystemTime.getCurrentTime()/1000 ); 5012 } 5013 5014 public int getRank()5015 getRank() 5016 { 5017 return( rand_list==null?0:rand_list.length ); 5018 } 5019 5020 public boolean isUnread()5021 isUnread() 5022 { 5023 return( unread ); 5024 } 5025 5026 protected void setPublic( ContentCache _cc )5027 setPublic( 5028 ContentCache _cc ) 5029 { 5030 cc = _cc; 5031 5032 if ( unread ){ 5033 5034 incrementUnread(); 5035 } 5036 5037 rand_list = new int[]{ rand }; 5038 setChangedLocallyOn(0); 5039 } 5040 5041 public int getLastSeenSecs()5042 getLastSeenSecs() 5043 { 5044 return( last_seen ); 5045 } 5046 5047 protected void setUnreadInternal( boolean _unread )5048 setUnreadInternal( 5049 boolean _unread ) 5050 { 5051 synchronized( this ){ 5052 5053 unread = _unread; 5054 } 5055 } 5056 5057 public void setUnread( boolean _unread )5058 setUnread( 5059 boolean _unread ) 5060 { 5061 boolean changed = false; 5062 5063 synchronized( this ){ 5064 5065 if ( unread != _unread ){ 5066 5067 unread = _unread; 5068 5069 changed = true; 5070 } 5071 } 5072 5073 if ( changed ){ 5074 5075 if ( _unread ){ 5076 5077 incrementUnread(); 5078 5079 }else{ 5080 5081 decrementUnread(); 5082 } 5083 5084 setChangedLocallyOn(0); 5085 contentChanged( this ); 5086 } 5087 } 5088 5089 protected int getRand()5090 getRand() 5091 { 5092 return( rand ); 5093 } 5094 5095 protected int[] getRandList()5096 getRandList() 5097 { 5098 return( rand_list ); 5099 } 5100 5101 public Download getRelatedToDownload()5102 getRelatedToDownload() 5103 { 5104 try{ 5105 return( getDownload( getRelatedToHash())); 5106 5107 }catch( Throwable e ){ 5108 5109 Debug.out( e ); 5110 5111 return( null ); 5112 } 5113 } 5114 5115 public void delete()5116 delete() 5117 { 5118 setChangedLocallyOn(0); 5119 RelatedContentManager.this.delete( new RelatedContent[]{ this }); 5120 } 5121 5122 public String getString()5123 getString() 5124 { 5125 return( super.getString() + ", " + rand + ", rl=" + rand_list + ", last_seen=" + last_seen + ", level=" + level ); 5126 } 5127 } 5128 5129 // can't move this class out of here as the key for the transfer type is based on its 5130 // class name... 5131 5132 protected class 5133 RCMSearchXFer 5134 implements DistributedDatabaseTransferType 5135 { 5136 } 5137 5138 protected static class 5139 ContentCache 5140 { 5141 protected Map<String,DownloadInfo> related_content = new HashMap<String, DownloadInfo>(); 5142 protected ByteArrayHashMapEx<ArrayList<DownloadInfo>> related_content_map = new ByteArrayHashMapEx<ArrayList<DownloadInfo>>(); 5143 } 5144 5145 private static class 5146 SecondaryLookup 5147 { 5148 final private byte[] hash; 5149 final private int level; 5150 final private byte nets; 5151 5152 private SecondaryLookup( byte[] _hash, int _level, byte _nets )5153 SecondaryLookup( 5154 byte[] _hash, 5155 int _level, 5156 byte _nets ) 5157 { 5158 hash = _hash; 5159 level = _level; 5160 nets = _nets; 5161 } 5162 5163 private byte[] getHash()5164 getHash() 5165 { 5166 return( hash ); 5167 } 5168 5169 private int getLevel()5170 getLevel() 5171 { 5172 return( level ); 5173 } 5174 5175 private byte getNetworks()5176 getNetworks() 5177 { 5178 return( nets ); 5179 } 5180 } 5181 } 5182