1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* 3 * Pan - A Newsreader for Gtk+ 4 * Copyright (C) 2002-2006 Charles Kerr <charles@rebelbase.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; version 2 of the License. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 #ifndef __DataImpl_h__ 21 #define __DataImpl_h__ 22 23 #include <iosfwd> 24 #include <list> 25 #include <map> 26 #include <string> 27 #include <vector> 28 #include <deque> 29 30 #include <pan/tasks/queue.h> 31 #include <pan/general/quark.h> 32 #include <pan/general/macros.h> 33 #include <pan/general/map-vector.h> 34 #include <pan/general/sorted-vector.h> 35 #include <pan/usenet-utils/numbers.h> 36 #include <pan/usenet-utils/scorefile.h> 37 #include <pan/usenet-utils/blowfish.h> 38 #include <pan/data/article.h> 39 #include <pan/data/article-cache.h> 40 #include <pan/data/encode-cache.h> 41 #include <pan/data/data.h> 42 #include <pan/data-impl/data-io.h> 43 #include <pan/data-impl/article-filter.h> 44 #include <pan/data-impl/rules-filter.h> 45 #include <pan/data-impl/profiles.h> 46 #include <pan/data-impl/memchunk.h> 47 #include <pan/gui/prefs.h> 48 49 #ifdef HAVE_GNUTLS 50 #include <pan/data/cert-store.h> 51 #include <gnutls/gnutls.h> 52 #endif 53 54 namespace pan 55 { 56 typedef std::vector<const Article*> articles_t; 57 typedef Data::PasswordData PasswordData; 58 59 /** 60 * File-based implementation of the `Data' backend interface. 61 * 62 * Most of the files are stored in $PAN_HOME, which defaults to 63 * $HOME/.pan2 if the PAN_HOME environmental variable isn't set. 64 * 65 * @ingroup data_impl 66 */ 67 class DataImpl: 68 public Data, 69 public TaskArchive, 70 public ProfilesImpl 71 { 72 73 /** 74 *** SERVERS 75 **/ 76 77 public: 78 /* The ProfilesImpl will own and destruct the DataIO object */ 79 DataImpl (const StringView& cache_ext, Prefs& prefs, bool unit_test=false, int cache_megs=10, DataIO * source=new DataIO()); 80 virtual ~DataImpl (); 81 virtual void save_state (); 82 83 public: get_cache()84 virtual ArticleCache& get_cache () { return _cache; } get_cache()85 virtual const ArticleCache& get_cache () const { return _cache; } 86 get_encode_cache()87 virtual EncodeCache& get_encode_cache () { return _encode_cache; } get_encode_cache()88 virtual const EncodeCache& get_encode_cache () const { return _encode_cache; } 89 get_certstore()90 virtual CertStore& get_certstore () { return _certstore; } get_certstore()91 virtual const CertStore& get_certstore () const { return _certstore; } 92 get_prefs()93 virtual Prefs& get_prefs () { return _prefs; } get_prefs()94 virtual const Prefs& get_prefs () const { return _prefs; } 95 get_queue()96 virtual Queue* get_queue () { return _queue; } get_queue()97 virtual const Queue* get_queue () const { return _queue; } 98 set_queue(Queue * q)99 virtual void set_queue (Queue* q) { _queue = q; } 100 101 private: 102 ArticleCache _cache; 103 EncodeCache _encode_cache; 104 CertStore _certstore; 105 Queue* _queue; 106 107 public: 108 #ifdef HAVE_GKR 109 #if GTK_CHECK_VERSION(3,0,0) 110 gboolean password_encrypt (const PasswordData&); 111 gchar* password_decrypt (PasswordData&) const; 112 #else 113 GnomeKeyringResult password_encrypt (const PasswordData&); 114 GnomeKeyringResult password_decrypt (PasswordData&) const; 115 #endif /* GTK_CHECK_VERSION(3,0,0) */ 116 #endif 117 private: 118 119 void rebuild_backend (); 120 const bool _unit_test; 121 DataIO * _data_io; 122 Prefs& _prefs; 123 124 /** 125 *** SERVERS 126 **/ 127 128 private: // implementation 129 130 void load_server_properties (const DataIO&); 131 132 void save_server_properties (DataIO&, Prefs&); 133 134 typedef Loki::AssocVector<Quark,Server> servers_t; 135 136 servers_t _servers; 137 138 public: 139 virtual const Server* find_server (const Quark& server) const; 140 virtual Server* find_server (const Quark& server); 141 virtual bool find_server_by_hn (const std::string& server, Quark& setme) const; 142 143 public: // mutators 144 145 virtual void delete_server (const Quark& server); 146 147 virtual Quark add_new_server (); 148 149 150 virtual void set_server_auth (const Quark & server, 151 const StringView & username, 152 gchar *&password, 153 bool use_gkr); 154 155 virtual void set_server_trust (const Quark & servername, 156 int setme); 157 158 virtual void set_server_addr (const Quark & server, 159 const StringView & host, 160 const int port); 161 162 virtual void set_server_limits (const Quark & server, 163 int max_connections); 164 165 virtual void set_server_rank (const Quark& server, int rank); 166 167 virtual void set_server_ssl_support (const Quark& server, int ssl); 168 169 virtual void set_server_cert (const Quark & server, const StringView & cert); 170 171 virtual void set_server_article_expiration_age (const Quark & server, 172 int days); 173 174 virtual void set_server_compression_type (const Quark & server, 175 const int setme); 176 177 virtual void save_server_info (const Quark& server); 178 179 public: // accessors 180 get_servers()181 virtual quarks_t get_servers () const { 182 quarks_t servers; 183 foreach_const (servers_t, _servers, it) 184 servers.insert (it->first); 185 return servers; 186 } 187 188 virtual bool get_server_auth (const Quark & server, 189 std::string & setme_username, 190 gchar *&setme_password, 191 bool use_gkr); 192 193 virtual bool get_server_trust (const Quark & servername, int&) const; 194 195 virtual bool get_server_compression_type (const Quark & servername, CompressionType&) const; 196 197 virtual bool get_server_addr (const Quark & server, 198 std::string & setme_host, 199 int & setme_port) const; 200 201 virtual std::string get_server_address (const Quark& servername) const; 202 203 virtual bool get_server_ssl_support (const Quark & server) const; 204 205 virtual std::string get_server_cert (const Quark & server) const; 206 207 virtual int get_server_rank (const Quark& server) const; 208 209 virtual int get_server_limits (const Quark & server) const; 210 211 virtual int get_server_article_expiration_age (const Quark & server) const; 212 213 /** 214 *** GROUPS 215 **/ 216 217 private: // implementation 218 219 typedef std::map<Quark,std::string> descriptions_t; 220 mutable descriptions_t _descriptions; // groupname -> description 221 mutable bool _descriptions_loaded; 222 223 typedef sorted_vector<Quark,true> unique_sorted_quarks_t; 224 typedef sorted_vector<Quark,true> groups_t; 225 groups_t _moderated; // groups which are moderated 226 groups_t _nopost; // groups which do not allow posting 227 228 typedef sorted_vector<Quark,true,AlphabeticalQuarkOrdering> alpha_groups_t; 229 alpha_groups_t _subscribed; // subscribed groups, sorted alphabetically 230 alpha_groups_t _unsubscribed; // non-subscribed groups, sorted alphabetically 231 232 /** 233 * Represents a newsgroup that's been read. 234 * 235 * Since most groups are never read, the `read' fields are separated 236 * out into this structure, so that it can be instantiated on demand. 237 * Since most news servers have tens of thousands of newsgroups, 238 * this represents a big memory savings. 239 * 240 * This private class should only be used by code in the data-impl module. 241 */ 242 struct ReadGroup 243 { 244 /** 245 * Per-server for a newsgroup that's been read. 246 * 247 * This private class should only be used by code in the data-impl module. 248 */ 249 struct Server { 250 Numbers _read; 251 uint64_t _xover_high; ServerReadGroup::Server252 Server(): _xover_high(0) {} 253 }; 254 typedef Loki::AssocVector<Quark,Server> servers_t; 255 servers_t _servers; 256 257 unsigned long _article_count; 258 unsigned long _unread_count; 259 ReadGroupReadGroup260 ReadGroup(): _article_count(0), _unread_count(0) {} 261 262 Server& operator[](const Quark& s) { return _servers[s]; } 263 decrement_safeReadGroup264 static void decrement_safe (unsigned long& lhs, unsigned long dec) { lhs = (lhs>dec) ? lhs-dec : 0; } decrement_unreadReadGroup265 void decrement_unread (unsigned long dec) { decrement_safe (_unread_count, dec); } decrement_countReadGroup266 void decrement_count (unsigned long dec) { decrement_safe (_article_count, dec); } 267 find_serverReadGroup268 Server* find_server (const Quark& s) { 269 servers_t::iterator it (_servers.find (s)); 270 return it == _servers.end() ? 0 : &it->second; 271 } 272 find_serverReadGroup273 const Server* find_server (const Quark& s) const { 274 servers_t::const_iterator it (_servers.find (s)); 275 return it == _servers.end() ? 0 : &it->second; 276 } 277 }; 278 279 typedef Loki::AssocVector<Quark,ReadGroup> read_groups_t; 280 read_groups_t _read_groups; 281 find_read_group(const Quark & g)282 ReadGroup* find_read_group (const Quark& g) { 283 read_groups_t::iterator it (_read_groups.find (g)); 284 return it == _read_groups.end() ? 0 : &it->second; 285 } find_read_group(const Quark & g)286 const ReadGroup* find_read_group (const Quark& g) const { 287 read_groups_t::const_iterator it (_read_groups.find (g)); 288 return it == _read_groups.end() ? 0 : &it->second; 289 } find_read_group_server(const Quark & g,const Quark & s)290 ReadGroup::Server* find_read_group_server (const Quark& g, const Quark& s) { 291 ReadGroup * read_group = find_read_group (g); 292 return read_group ? read_group->find_server (s) : 0; 293 } find_read_group_server(const Quark & g,const Quark & s)294 const ReadGroup::Server* find_read_group_server (const Quark& g, const Quark& s) const { 295 const ReadGroup * read_group = find_read_group (g); 296 return read_group ? read_group->find_server (s) : 0; 297 } 298 299 void ensure_descriptions_are_loaded () const; 300 void load_group_descriptions (const DataIO&) const; 301 void save_group_descriptions (DataIO&) const; 302 303 void load_group_xovers (const DataIO&); 304 void save_group_xovers (DataIO&) const; 305 306 void load_group_permissions (const DataIO&); 307 void save_group_permissions (DataIO&) const; 308 309 std::string get_newsrc_filename (const Quark& server) const; 310 void load_newsrc (const Quark& server, LineReader*, alpha_groups_t&, alpha_groups_t&); 311 void load_newsrc_files (const DataIO&); 312 void save_newsrc_files (DataIO&) const; 313 314 public: // mutators 315 316 virtual void add_groups (const Quark & server, 317 const NewGroup * new_groups, 318 size_t group_count); 319 320 virtual void mark_group_read (const Quark & group); 321 322 virtual void set_group_subscribed (const Quark & group, 323 bool sub); 324 325 public: // accessors 326 327 virtual const std::string& get_group_description (const Quark& group) const; 328 virtual void get_subscribed_groups (std::vector<Quark>&) const; 329 virtual void get_other_groups (std::vector<Quark>&) const; 330 virtual void get_group_counts (const Quark & group, 331 unsigned long & setme_unread, 332 unsigned long & setme_total) const; 333 virtual char get_group_permission (const Quark & group) const; 334 virtual void group_get_servers (const Quark& group, quarks_t&) const; 335 virtual void server_get_groups (const Quark& server, quarks_t&) const; 336 337 /** 338 *** HEADERS 339 **/ 340 341 private: // implementation 342 343 /** 'article' MUST have been allocated by GroupHeaders::alloc_new_article()!! */ 344 void load_article (const Quark& g, Article * article, const StringView& references); 345 346 /** the contents of `part' are given up wholesale to our 347 local GroupHeaders. As a side-effect, the value of `part' 348 after this call is undefined. This is an ugly interface, 349 but it's fast and only called by one client. */ 350 void load_part (const Quark& g, const Quark& mid, 351 int number, 352 const StringView& part_mid, 353 unsigned long lines, 354 unsigned long bytes); 355 356 void load_headers (const DataIO&, const Quark& group); 357 void save_headers (DataIO&, const Quark& group) const; 358 bool save_headers (DataIO&, const Quark& group, 359 const std::vector<Article*>&, 360 unsigned long&, unsigned long&) const; 361 362 363 /** 364 * ArticleNode is a Tree node used for threading Article objects. 365 * 366 * GroupHeaders owns these, and also contains a lookup table from 367 * Message-ID to ArticleNode for finding a starting point in the tree. 368 * 369 * Note that _article can be NULL here; we instantiate nodes from 370 * Articles' References: header so that we can get the threading model 371 * right even during an xover where we get children in before the 372 * parent. This way we never need to rethread; new articles just 373 * fill in the missing pieces as they come in. 374 * 375 * @see GroupHeaders 376 */ 377 struct ArticleNode 378 { 379 Quark _mid; 380 Article * _article; 381 382 ArticleNode * _parent; 383 typedef std::list<ArticleNode*> children_t; 384 children_t _children; 385 ArticleNodeArticleNode386 ArticleNode(): _article(0), _parent(0) {} 387 }; 388 389 typedef std::map<Quark,ArticleNode*> nodes_t; 390 typedef std::vector<ArticleNode*> nodes_v; 391 typedef std::vector<const ArticleNode*> const_nodes_v; 392 393 struct NodeWeakOrdering 394 { 395 typedef std::pair<Quark,ArticleNode*> nodes_t_element; 396 operatorNodeWeakOrdering397 bool operator () (const nodes_t_element& a, const Quark& b) const { 398 return a.first < b; 399 } operatorNodeWeakOrdering400 bool operator () (const Quark& a, const nodes_t_element& b) const { 401 return a < b.first; 402 } 403 }; 404 405 /*** 406 ** 407 ***/ 408 virtual void fire_article_flag_changed (articles_t& a, const Quark& group); 409 410 struct GroupHeaders 411 { 412 int _ref; 413 bool _dirty; 414 nodes_t _nodes; 415 MemChunk<Article> _art_chunk; 416 MemChunk<ArticleNode> _node_chunk; 417 418 GroupHeaders(); 419 ~GroupHeaders (); 420 alloc_new_articleGroupHeaders421 Article& alloc_new_article () { 422 static const Article blank_article; 423 _art_chunk.push_back (blank_article); 424 return _art_chunk.back(); 425 } 426 427 ArticleNode* find_node (const Quark& mid); 428 const ArticleNode* find_node (const Quark& mid) const; 429 430 const Quark& find_parent_message_id (const Quark& mid) const; 431 Article* find_article (const Quark& mid); 432 const Article* find_article (const Quark& mid) const; 433 void remove_articles (const quarks_t& mids); 434 void build_references_header (const Article* article, std::string& setme) const; 435 436 }; 437 438 static void find_nodes (const quarks_t & mids, 439 nodes_t & nodes, 440 nodes_v & setme); 441 442 static void find_nodes (const quarks_t & mids, 443 const nodes_t & nodes, 444 const_nodes_v & setme); 445 446 virtual void get_article_references (const Quark& group, const Article*, std::string& setme) const; 447 448 /** 449 * For a given ArticleNode, returns the first ancestor whose mid is in mid_pool. 450 * FIXME: these should be member functions of ArticleNode 451 */ 452 static ArticleNode* find_closest_ancestor (ArticleNode * node, 453 const unique_sorted_quarks_t & mid_pool); 454 static const ArticleNode* find_closest_ancestor (const ArticleNode * node, 455 const unique_sorted_quarks_t & mid_pool); 456 457 static ArticleNode* find_ancestor (ArticleNode * node, 458 const Quark & ancestor_mid); 459 460 typedef Loki::AssocVector<Quark,GroupHeaders*> group_to_headers_t; 461 group_to_headers_t _group_to_headers; 462 463 GroupHeaders* get_group_headers (const Quark& group); 464 const GroupHeaders* get_group_headers (const Quark& group) const; 465 void free_group_headers_memory (const Quark& group); 466 bool is_read (const Xref&) const; 467 468 void ref_group (const Quark& group); 469 void unref_group (const Quark& group); 470 471 private: 472 473 class MyTree: public Data::ArticleTree 474 { 475 friend class DataImpl; 476 477 public: // life cycle 478 MyTree (DataImpl & data_impl, 479 const Quark & group, 480 const Quark & save_path, // for auto-download 481 const Data::ShowType show_type, 482 const FilterInfo * filter_info=0, 483 const RulesInfo * rules=0); 484 virtual ~MyTree (); 485 486 public: // from ArticleTree 487 virtual void get_children (const Quark& mid, articles_t& setme) const; 488 virtual const Article* get_parent (const Quark& mid) const; 489 virtual const Article* get_article (const Quark& mid) const; 490 virtual size_t size () const; 491 virtual void set_filter (const ShowType show_type = SHOW_ARTICLES, 492 const FilterInfo * criteria = 0); 493 virtual void set_rules (const ShowType show_type = SHOW_ARTICLES, 494 const RulesInfo * rules = 0); 495 496 public: 497 void articles_changed (const quarks_t& mids, bool do_refilter); 498 void add_articles (const quarks_t& mids); 499 void remove_articles (const quarks_t& mids); 500 501 private: // implementation fields 502 const Quark _group; 503 DataImpl & _data; 504 const Quark _save_path; // for auto-download 505 nodes_t _nodes; 506 MemChunk<ArticleNode> _node_chunk; 507 FilterInfo _filter; 508 RulesInfo _rules; 509 Data::ShowType _show_type; 510 struct NodeMidCompare; 511 struct TwoNodes; 512 513 private: 514 typedef std::set<const ArticleNode*,NodeMidCompare> unique_nodes_t; 515 void accumulate_descendants (unique_nodes_t&, const ArticleNode*) const; 516 void add_articles (const const_nodes_v&); 517 void apply_filter (const const_nodes_v&); 518 void apply_rules (const_nodes_v& candidates); 519 520 private: 521 void cache_articles (std::set<const Article*> s); 522 void download_articles (std::set<const Article*> s); 523 524 }; 525 526 527 528 std::set<MyTree*> _trees; 529 void on_articles_removed (const quarks_t& mids) const; 530 void on_articles_added (const Quark& group, const quarks_t& mids); 531 void on_articles_changed (const Quark& group, const quarks_t& mids, bool do_refilter); 532 void remove_articles_from_tree (MyTree*, const quarks_t& mids) const; 533 void add_articles_to_tree (MyTree*, const quarks_t& mids); 534 535 public: // Data interface 536 537 virtual void delete_articles (const unique_articles_t&); 538 539 virtual ArticleTree* group_get_articles (const Quark & group, 540 const Quark & save_path, 541 const ShowType show_type = SHOW_ARTICLES, 542 const FilterInfo * criteria=0, 543 const RulesInfo * rules=0) const; 544 545 virtual void group_clear_articles (const Quark & group); 546 547 virtual bool is_read (const Article *) const; 548 549 virtual void mark_read (const Article & article, 550 bool mark_read); 551 552 virtual void mark_read (const Article ** articles, 553 unsigned long article_count, 554 bool mark_read=true); 555 556 virtual void get_article_scores (const Quark & newsgroup, 557 const Article & article, 558 Scorefile::items_t & setme) const; 559 560 virtual void add_score (const StringView & section_wildmat, 561 int score_value, 562 bool score_assign_flag, 563 int lifespan_days, 564 bool all_items_must_be_true, 565 const Scorefile::AddItem * items, 566 size_t item_count, 567 bool do_rescore); 568 569 virtual void comment_out_scorefile_line (const StringView & filename, 570 size_t begin_line, 571 size_t end_line, 572 bool do_rescore); 573 574 virtual void rescore_articles (const Quark& group, const quarks_t mids); 575 576 virtual void rescore_group_articles (const Quark& group); 577 578 virtual void rescore (); 579 580 private: 581 582 Scorefile _scorefile; 583 584 /** 585 *** XOVER 586 **/ 587 588 private: // implementation 589 590 /** 591 * This is a workarea used when we processing an XOVER command. 592 */ 593 struct XOverEntry 594 { 595 time_t _last_flush_time; 596 597 /** These are the articles which have been recently added. 598 The patch is periodically flushed to on_articles_added() 599 from xover_line() and xref_unref(). */ 600 quarks_t _added_batch; 601 602 /** Same as _added_batch, but for changed articles. */ 603 quarks_t _changed_batch; 604 605 typedef std::multimap<Quark,Quark> subject_to_mid_t; 606 607 /** This is for multipart detection. Pan folds multipart posts into 608 a single Article holding all the parts. This lookup helps decide, 609 when we get a new multipart post, which Article to fold 610 it into. We strip out the unique part info from the Subject header 611 (such as the "15" in [15/42]) and use it as a key in this lookup 612 table that gives the Message-ID of the Article owning this post. */ 613 subject_to_mid_t _subject_lookup; 614 615 /** We must refcount because multiple server connections can 616 be assigned to the same XOVER task. */ 617 int refcount; 618 XOverEntryXOverEntry619 XOverEntry(): _last_flush_time(0), refcount(0) { } 620 }; 621 622 typedef Loki::AssocVector<Quark,XOverEntry> xovers_t; 623 xovers_t _xovers; 624 625 XOverEntry * _cached_xover_entry; 626 Quark _cached_xover_group; 627 628 /** 629 * Destroy the workarea. 630 * Don't call this directly -- xover_unref() do its job. 631 */ 632 void xover_clear_workarea (const Quark& group); 633 634 /** 635 * Finds the XOverEntry workarea for the specified group. 636 * This must be called inside an xover_ref() / xover_unref() block, 637 * as the workarea is instantiated when the group's xover refcount 638 * increases to one and destroyed when it goes down to zero. 639 */ 640 XOverEntry& xover_get_workarea (const Quark& group); 641 642 public: // Data interface 643 644 virtual void xover_ref (const Quark & group); 645 646 virtual const Article* xover_add (const Quark & server, 647 const Quark & group, 648 const StringView & subject, 649 const StringView & author, 650 const time_t date, 651 const StringView & message_id, 652 const StringView & references, 653 const unsigned long byte_count, 654 const unsigned long line_count, 655 const StringView & xref, 656 const bool is_virtual=false); 657 658 /** useful for xover unit testing */ 659 virtual void xover_flush (const Quark & group); 660 661 virtual void xover_unref (const Quark & group); 662 663 virtual uint64_t get_xover_high (const Quark & group, 664 const Quark & server) const; 665 666 virtual void set_xover_high (const Quark & group, 667 const Quark & server, 668 const uint64_t high); 669 670 virtual void set_xover_low (const Quark & group, 671 const Quark & server, 672 const uint64_t low); 673 674 675 /** 676 *** TaskArchive 677 **/ 678 679 public: 680 681 virtual void save_tasks (const std::vector<Task*>& saveme); 682 683 virtual void load_tasks (std::vector<Task*>& setme); 684 685 686 687 public: 688 689 const ArticleFilter _article_filter; 690 RulesFilter _rules_filter; 691 692 private: 693 mutable guint newsrc_autosave_id; 694 guint newsrc_autosave_timeout; 695 public: set_newsrc_autosave_timeout(guint seconds)696 void set_newsrc_autosave_timeout(guint seconds) 697 {newsrc_autosave_timeout = seconds;} save_newsrc_files()698 void save_newsrc_files() 699 { // Called from rc_as_cb(...). 700 // The newsrc_autosave_id is now (soon) invalid since the timeout will be 701 // cancelled when our caller returns FALSE. So forget about it already. 702 newsrc_autosave_id = 0; 703 save_newsrc_files(*_data_io); 704 } 705 }; 706 } 707 708 #endif 709