1 /*
2 Bacula(R) - The Network Backup Solution
3
4 Copyright (C) 2000-2020 Kern Sibbald
5
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
8
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
13
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
16
17 Bacula(R) is a registered trademark of Kern Sibbald.
18 */
19 /*
20 *
21 * Program to test batch mode
22 *
23 * Eric Bollengier, March 2007
24 *
25 */
26
27 /*
28 to create datafile
29
30 for i in $(seq 10000 99999) ; do
31 j=$((($i % 1000) + 555))
32 echo "$i;/tmp/totabofds$j/fiddddle${j}$i;xxxLSTATxxxx;xxxxxxxMD5xxxxxx"
33 done > dat1
34
35 or
36
37 j=0
38 find / | while read a; do
39 j=$(($j+1))
40 echo "$j;$a;xxxLSTATxxxx;xxxxxxxMD5xxxxxx"
41 done > dat1
42 */
43
44 #include "bacula.h"
45 #include "stored/stored.h"
46 #include "findlib/find.h"
47 #include "cats/cats.h"
48
49 /* Forward referenced functions */
50 static void *do_batch(void *);
51
52
53 /* Local variables */
54 static BDB *db;
55
56 static const char *db_name = "bacula";
57 static const char *db_user = "bacula";
58 static const char *db_password = "";
59 static const char *db_host = NULL;
60 static const char *db_ssl_mode = NULL;
61 static const char *db_ssl_key = NULL;
62 static const char *db_ssl_cert = NULL;
63 static const char *db_ssl_ca = NULL;
64 static const char *db_ssl_capath = NULL;
65 static const char *db_ssl_cipher = NULL;
66
67 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
68
usage()69 static void usage()
70 {
71 fprintf(stderr, _(
72 PROG_COPYRIGHT
73 "\n%sVersion: %s (%s)\n"
74 "Example : bbatch -w /path/to/workdir -h localhost -f dat1 -f dat -f datx\n"
75 " will start 3 thread and load dat1, dat and datx in your catalog\n"
76 "See bbatch.c to generate datafile\n\n"
77 "Usage: bbatch [ options ] -w working/dir -f datafile\n"
78 " -b with batch mode\n"
79 " -B without batch mode\n"
80 " -d <nn> set debug level to <nn>\n"
81 " -dt print timestamp in debug output\n"
82 " -n <name> specify the database name (default bacula)\n"
83 " -u <user> specify database user name (default bacula)\n"
84 " -P <password specify database password (default none)\n"
85 " -h <host> specify database host (default NULL)\n"
86 " -k <sslkey> path name to the key file (default NULL)\n"
87 " -e <sslcert> path name to the certificate file (default NULL)\n"
88 " -a <sslca> path name to the CA certificate file (default NULL)\n"
89 " -w <working> specify working directory\n"
90 " -r <jobids> call restore code with given jobids\n"
91 " -v verbose\n"
92 " -f <file> specify data file\n"
93 " -? print this message\n\n"), 2001, "", VERSION, BDATE);
94 exit(1);
95 }
96
97 /* number of thread started */
98 int nb=0;
99
list_handler(void * ctx,int num_fields,char ** row)100 static int list_handler(void *ctx, int num_fields, char **row)
101 {
102 uint64_t *a = (uint64_t*) ctx;
103 (*a)++;
104 return 0;
105 }
106
main(int argc,char * argv[])107 int main (int argc, char *argv[])
108 {
109 int ch;
110 bool use_batch_insert = true;
111 char *restore_list=NULL;
112 setlocale(LC_ALL, "");
113 bindtextdomain("bacula", LOCALEDIR);
114 textdomain("bacula");
115 init_stack_dump();
116 lmgr_init_thread();
117
118 char **files = (char **) malloc (10 * sizeof(char *));
119 int i;
120 my_name_is(argc, argv, "bbatch");
121 init_msg(NULL, NULL);
122
123 OSDependentInit();
124
125 while ((ch = getopt(argc, argv, "bBh:o:k:e:a:c:d:n:P:Su:vf:w:r:?")) != -1) {
126 switch (ch) {
127 case 'r':
128 restore_list=bstrdup(optarg);
129 break;
130 case 'B':
131 use_batch_insert = true;
132 break;
133 case 'b':
134 use_batch_insert = false;
135 break;
136 case 'd': /* debug level */
137 if (*optarg == 't') {
138 dbg_timestamp = true;
139 } else {
140 debug_level = atoi(optarg);
141 if (debug_level <= 0) {
142 debug_level = 1;
143 }
144 }
145 break;
146
147 case 'h':
148 db_host = optarg;
149 break;
150
151 case 'o':
152 db_ssl_mode = optarg;
153 break;
154
155 case 'k':
156 db_ssl_key = optarg;
157 break;
158
159 case 'e':
160 db_ssl_cert = optarg;
161 break;
162
163 case 'a':
164 db_ssl_ca = optarg;
165 break;
166
167 case 'n':
168 db_name = optarg;
169 break;
170
171 case 'w':
172 working_directory = optarg;
173 break;
174
175 case 'u':
176 db_user = optarg;
177 break;
178
179 case 'P':
180 db_password = optarg;
181 break;
182
183 case 'v':
184 verbose++;
185 break;
186
187 case 'f':
188 if (nb < 10 ) {
189 files[nb++] = optarg;
190 }
191 break;
192
193 case '?':
194 default:
195 usage();
196
197 }
198 }
199 argc -= optind;
200 argv += optind;
201
202 if (argc != 0) {
203 Pmsg0(0, _("Wrong number of arguments: \n"));
204 usage();
205 }
206
207 if (restore_list) {
208 uint64_t nb_file=0;
209 btime_t start, end;
210 /* To use the -r option, the catalog should already contains records */
211
212 if ((db = db_init_database(NULL, NULL, db_name, db_user, db_password,
213 db_host, 0, NULL,
214 db_ssl_mode, db_ssl_key,
215 db_ssl_cert, db_ssl_ca,
216 db_ssl_capath, db_ssl_cipher,
217 false, !use_batch_insert)) == NULL) {
218 Emsg0(M_ERROR_TERM, 0, _("Could not init Bacula database\n"));
219 }
220 if (!db_open_database(NULL, db)) {
221 Emsg0(M_ERROR_TERM, 0, db_strerror(db));
222 }
223
224 start = get_current_btime();
225 db_get_file_list(NULL, db, restore_list, DBL_NONE, list_handler, &nb_file);
226 end = get_current_btime();
227
228 Pmsg3(0, _("Computing file list for jobid=%s files=%lld secs=%d\n"),
229 restore_list, nb_file, (uint32_t)btime_to_unix(end-start));
230
231 free(restore_list);
232 return 0;
233 }
234
235 if (use_batch_insert) {
236 printf("With Batch Insert mode\n");
237 } else {
238 printf("Without Batch Insert mode\n");
239 }
240
241 i = nb;
242 while (--i >= 0) {
243 pthread_t thid;
244 JCR *bjcr = new_jcr(sizeof(JCR), NULL);
245 bjcr->bsr = NULL;
246 bjcr->VolSessionId = 1;
247 bjcr->VolSessionTime = (uint32_t)time(NULL);
248 bjcr->NumReadVolumes = 0;
249 bjcr->NumWriteVolumes = 0;
250 bjcr->JobId = getpid();
251 bjcr->setJobType(JT_CONSOLE);
252 bjcr->setJobLevel(L_FULL);
253 bjcr->JobStatus = JS_Running;
254 bjcr->where = bstrdup(files[i]);
255 bjcr->job_name = get_pool_memory(PM_FNAME);
256 pm_strcpy(bjcr->job_name, "Dummy.Job.Name");
257 bjcr->client_name = get_pool_memory(PM_FNAME);
258 pm_strcpy(bjcr->client_name, "Dummy.Client.Name");
259 bstrncpy(bjcr->Job, "bbatch", sizeof(bjcr->Job));
260 bjcr->fileset_name = get_pool_memory(PM_FNAME);
261 pm_strcpy(bjcr->fileset_name, "Dummy.fileset.name");
262 bjcr->fileset_md5 = get_pool_memory(PM_FNAME);
263 pm_strcpy(bjcr->fileset_md5, "Dummy.fileset.md5");
264
265 if ((db = db_init_database(NULL, NULL, db_name, db_user, db_password,
266 db_host, 0, NULL,
267 db_ssl_mode, db_ssl_key,
268 db_ssl_cert, db_ssl_ca,
269 db_ssl_capath, db_ssl_cipher,
270 false, false)) == NULL) {
271 Emsg0(M_ERROR_TERM, 0, _("Could not init Bacula database\n"));
272 }
273 if (!db_open_database(NULL, db)) {
274 Emsg0(M_ERROR_TERM, 0, db_strerror(db));
275 }
276 Dmsg0(200, "Database opened\n");
277 if (verbose) {
278 Pmsg2(000, _("Using Database: %s, User: %s\n"), db_name, db_user);
279 }
280
281 bjcr->db = db;
282
283 pthread_create(&thid, NULL, do_batch, bjcr);
284 }
285
286 while (nb > 0) {
287 bmicrosleep(1,0);
288 }
289
290 return 0;
291 }
292
fill_attr(ATTR_DBR * ar,char * data)293 static void fill_attr(ATTR_DBR *ar, char *data)
294 {
295 char *p;
296 char *b;
297 int index=0;
298 ar->Stream = STREAM_UNIX_ATTRIBUTES;
299 ar->JobId = getpid();
300
301 for(p = b = data; *p; p++) {
302 if (*p == ';') {
303 *p = '\0';
304 switch (index) {
305 case 0:
306 ar->FileIndex = str_to_int64(b);
307 break;
308 case 1:
309 ar->fname = b;
310 break;
311 case 2:
312 ar->attr = b;
313 break;
314 case 3:
315 ar->Digest = b;
316 break;
317 }
318 index++;
319 b = ++p;
320 }
321 }
322 }
323
do_batch(void * jcr)324 static void *do_batch(void *jcr)
325 {
326 JCR *bjcr = (JCR *)jcr;
327 char data[1024];
328 int lineno = 0;
329 struct ATTR_DBR ar;
330 memset(&ar, 0, sizeof(ar));
331 btime_t begin = get_current_btime();
332 char *datafile = bjcr->where;
333
334 FILE *fd = fopen(datafile, "r");
335 if (!fd) {
336 Emsg1(M_ERROR_TERM, 0, _("Error opening datafile %s\n"), datafile);
337 }
338 while (fgets(data, sizeof(data)-1, fd)) {
339 strip_trailing_newline(data);
340 lineno++;
341 if (verbose && ((lineno % 5000) == 1)) {
342 printf("\r%i", lineno);
343 }
344 fill_attr(&ar, data);
345 if (!db_create_attributes_record(bjcr, bjcr->db, &ar)) {
346 Emsg0(M_ERROR_TERM, 0, _("Error while inserting file\n"));
347 }
348 }
349 fclose(fd);
350 db_write_batch_file_records(bjcr);
351 btime_t end = get_current_btime();
352
353 P(mutex);
354 char ed1[200], ed2[200];
355 printf("\rbegin = %s, end = %s\n", edit_int64(begin, ed1),edit_int64(end, ed2));
356 printf("Insert time = %sms\n", edit_int64((end - begin) / 10000, ed1));
357 printf("Create %u files at %.2f/s\n", lineno,
358 (lineno / ((float)((end - begin) / 1000000))));
359 nb--;
360 V(mutex);
361 pthread_exit(NULL);
362 return NULL;
363 }
364