1 /*
2    BAREOS® - Backup Archiving REcovery Open Sourced
3 
4    Copyright (C) 2013-2014 Planets Communications B.V.
5    Copyright (C) 2013-2020 Bareos GmbH & Co. KG
6 
7    This program is Free Software; you can redistribute it and/or
8    modify it under the terms of version three of the GNU Affero General Public
9    License as published by the Free Software Foundation and included
10    in the file LICENSE.
11 
12    This program is distributed in the hope that it will be useful, but
13    WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15    Affero General Public License for more details.
16 
17    You should have received a copy of the GNU Affero General Public License
18    along with this program; if not, write to the Free Software
19    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20    02110-1301, USA.
21 */
22 
23 #ifndef BAREOS_FILED_ACCURATE_H_
24 #define BAREOS_FILED_ACCURATE_H_
25 
26 /*
27  * Generic accurate payload is the same for any storage class.
28  *
29  * We store the lstat field as a character field as it makes storing of the
30  * data much more efficient than storing it in a normal stat structure.
31  * Nowadays most of these structures are 64 bits and they will use 136
32  * bytes (32 bits) and 128 bytes (64 bits) on Solaris. The ASCII encoding
33  * of the lstat field most of the time is between 43 and 75 bytes with an
34  * average of 60 bytes. (From live data of 152562393 files.)
35  *
36  * The seen flag is not in the actual accurate payload but stored in a
37  * bitmap. This makes it much more efficient for a storage engine that has
38  * to update the whole data when something changes. For things like a in
39  * memory hash table it doesn't make much difference and it has the
40  * disadvantage that we need to keep a filenr to index the bitmap which
41  * also cost some bytes.
42  */
43 
44 #include "include/config.h"
45 
46 #ifdef HAVE_HPUX_OS
47 #  pragma pack(push, 4)
48 #endif
49 
50 namespace filedaemon {
51 
52 struct accurate_payload {
53   int64_t filenr;
54   int32_t delta_seq;
55   char* lstat;
56   char* chksum;
57 };
58 
59 
60 /*
61  * Accurate payload storage abstraction classes.
62  */
63 class BareosAccurateFilelist {
64  protected:
65   int64_t filenr_ = 0;
66   char* seen_bitmap_ = nullptr;
67   JobControlRecord* jcr_ = nullptr;
68   uint32_t number_of_previous_files_ = 0;
69 
70  public:
71   BareosAccurateFilelist() = default;
~BareosAccurateFilelist()72   virtual ~BareosAccurateFilelist() {}
73 
74   virtual bool init() = 0;
75   virtual bool AddFile(char* fname,
76                        int fname_length,
77                        char* lstat,
78                        int lstat_length,
79                        char* chksum,
80                        int checksum_length,
81                        int32_t delta_seq)
82       = 0;
83   virtual bool EndLoad() = 0;
84   virtual accurate_payload* lookup_payload(char* fname) = 0;
85   virtual bool UpdatePayload(char* fname, accurate_payload* payload) = 0;
86   virtual bool SendBaseFileList() = 0;
87   virtual bool SendDeletedList() = 0;
MarkFileAsSeen(accurate_payload * payload)88   void MarkFileAsSeen(accurate_payload* payload)
89   {
90     SetBit(payload->filenr, seen_bitmap_);
91   }
92 
UnmarkFileAsSeen(accurate_payload * payload)93   void UnmarkFileAsSeen(accurate_payload* payload)
94   {
95     ClearBit(payload->filenr, seen_bitmap_);
96   }
97 
MarkAllFilesAsSeen()98   void MarkAllFilesAsSeen() { SetBitRange(0, filenr_ - 1, seen_bitmap_); }
99 
UnmarkAllFilesAsSeen()100   void UnmarkAllFilesAsSeen() { ClearBitRange(0, filenr_ - 1, seen_bitmap_); }
101 };
102 
103 /*
104  * Hash table specific storage abstraction class using the internal htable
105  * datastructure.
106  */
107 struct CurFile {
108   hlink link;
109   char* fname;
110   accurate_payload payload;
111 };
112 
113 #ifdef HAVE_HPUX_OS
114 #  pragma pack(pop)
115 #endif
116 
117 class BareosAccurateFilelistHtable : public BareosAccurateFilelist {
118  protected:
119   htable* file_list_;
120   void destroy();
121 
122  public:
123   /* methods */
124   BareosAccurateFilelistHtable() = delete;
125   BareosAccurateFilelistHtable(JobControlRecord* jcr, uint32_t number_of_files);
~BareosAccurateFilelistHtable()126   ~BareosAccurateFilelistHtable() { destroy(); }
127 
init()128   bool init() override { return true; }
129 
130   bool AddFile(char* fname,
131                int fname_length,
132                char* lstat,
133                int lstat_length,
134                char* chksum,
135                int checksum_length,
136                int32_t delta_seq) override;
137   bool EndLoad() override;
138   accurate_payload* lookup_payload(char* fname) override;
139   bool UpdatePayload(char* fname, accurate_payload* payload) override;
140   bool SendBaseFileList() override;
141   bool SendDeletedList() override;
142 };
143 
144 #ifdef HAVE_LMDB
145 
146 #  include "lmdb/lmdb.h"
147 
148 /*
149  * Lightning Memory DataBase (LMDB) specific storage abstraction class using the
150  * Symas LMDB.
151  */
152 class BareosAccurateFilelistLmdb : public BareosAccurateFilelist {
153  protected:
154   int pay_load_length_;
155   POOLMEM* pay_load_;
156   POOLMEM* lmdb_name_;
157   MDB_env* db_env_;
158   MDB_dbi db_dbi_;
159   MDB_txn* db_rw_txn_;
160   MDB_txn* db_ro_txn_;
161 
162   void destroy();
163 
164  public:
165   /* methods */
166   BareosAccurateFilelistLmdb() = delete;
167   BareosAccurateFilelistLmdb(JobControlRecord* jcr, uint32_t number_of_files);
~BareosAccurateFilelistLmdb()168   ~BareosAccurateFilelistLmdb() { destroy(); }
169   bool init() override;
170   bool AddFile(char* fname,
171                int fname_length,
172                char* lstat,
173                int lstat_length,
174                char* chksum,
175                int checksum_length,
176                int32_t delta_seq) override;
177   bool EndLoad() override;
178   accurate_payload* lookup_payload(char* fname) override;
179   bool UpdatePayload(char* fname, accurate_payload* payload) override;
180   bool SendBaseFileList() override;
181   bool SendDeletedList() override;
182 };
183 #endif /* HAVE_LMDB */
184 
185 bool AccurateFinish(JobControlRecord* jcr);
186 bool AccurateCheckFile(JobControlRecord* jcr, FindFilesPacket* ff_pkt);
187 bool AccurateMarkFileAsSeen(JobControlRecord* jcr, char* fname);
188 bool accurate_unMarkFileAsSeen(JobControlRecord* jcr, char* fname);
189 bool AccurateMarkAllFilesAsSeen(JobControlRecord* jcr);
190 bool accurate_unMarkAllFilesAsSeen(JobControlRecord* jcr);
191 void AccurateFree(JobControlRecord* jcr);
192 
193 
194 } /* namespace filedaemon */
195 #endif /* BAREOS_FILED_ACCURATE_H_ */
196