1 // This file is part of BOINC.
2 // http://boinc.berkeley.edu
3 // Copyright (C) 2015 University of California
4 //
5 // BOINC is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU Lesser General Public License
7 // as published by the Free Software Foundation,
8 // either version 3 of the License, or (at your option) any later version.
9 //
10 // BOINC is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 // See the GNU Lesser General Public License for more details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with BOINC. If not, see <http://www.gnu.org/licenses/>.
17
18 // Create workunit(s).
19 // see http://boinc.berkeley.edu/trac/wiki/JobSubmission
20
21 #include "config.h"
22
23 #include <cstdio>
24 #include <cstdlib>
25 #include <cstring>
26 #include <ctime>
27 #include <string>
28 #include <map>
29 #include <sys/param.h>
30 #include <unistd.h>
31
32 #include "boinc_db.h"
33 #include "common_defs.h"
34 #include "crypt.h"
35 #include "filesys.h"
36 #include "sched_config.h"
37 #include "str_replace.h"
38 #include "str_util.h"
39 #include "util.h"
40
41 #include "backend_lib.h"
42
43 using std::string;
44 using std::map;
45
46 bool verbose = false;
47 bool continue_on_error = false;
48
usage()49 void usage() {
50 fprintf(stderr,
51 "usage: create_work [options] infile1 infile2 ...\n"
52 "\n"
53 "Options:\n"
54 " --appname name\n"
55 " [ --additional_xml x ]\n"
56 " [ --batch n ]\n"
57 " [ --broadcast ]\n"
58 " [ --broadcast_user ID ]\n"
59 " [ --broadcast_team ID ]\n"
60 " [ --command_line \"X\" ]\n"
61 " [ --config_dir path ]\n"
62 " [ -d n ]\n"
63 " [ --delay_bound x ]\n"
64 " [ --hr_class n ]\n"
65 " [ --max_error_results n ]\n"
66 " [ --max_success_results n ]\n"
67 " [ --max_total_results n ]\n"
68 " [ --min_quorum n ]\n"
69 " [ --priority n ]\n"
70 " [ --result_template filename ] default: appname_out\n"
71 " [ --rsc_disk_bound x ]\n"
72 " [ --rsc_fpops_est x ]\n"
73 " [ --rsc_fpops_bound x ]\n"
74 " [ --rsc_memory_bound x ]\n"
75 " [ --size_class n ]\n"
76 " [ --stdin ]\n"
77 " [ --target_host ID ]\n"
78 " [ --target_nresults n ]\n"
79 " [ --target_team ID ]\n"
80 " [ --target_user ID ]\n"
81 " [ --verbose ]\n"
82 " [ --wu_id ID ] ID of existing workunit record (used by boinc_submit)\n"
83 " [ --wu_name name ] default: generate a name based on app name\n"
84 " [ --wu_template filename ] default: appname_in\n"
85 "\nSee http://boinc.berkeley.edu/trac/wiki/JobSubmission\n"
86 );
87 exit(1);
88 }
89
arg(char ** argv,int i,const char * name)90 bool arg(char** argv, int i, const char* name) {
91 char buf[256];
92 sprintf(buf, "-%s", name);
93 if (!strcmp(argv[i], buf)) return true;
94 sprintf(buf, "--%s", name);
95 if (!strcmp(argv[i], buf)) return true;
96 return false;
97 }
98
check_assign_id(int x)99 void check_assign_id(int x) {
100 if (x == 0) {
101 fprintf(stderr,
102 "you must specify a nonzero database ID for assigning jobs to users, teams, or hosts.\n"
103 );
104 exit(1);
105 }
106 }
107
108 // describes a job.
109 // Also used to store batch-level info such as template names
110 // and assignment info
111 //
112 struct JOB_DESC {
113 DB_WORKUNIT wu;
114 char wu_template[BLOB_SIZE];
115 char wu_template_file[256];
116 char result_template_file[256];
117 char result_template_path[MAXPATHLEN];
118 vector <INFILE_DESC> infiles;
119 char* command_line;
120 char additional_xml[256];
121 bool assign_flag;
122 bool assign_multi;
123 int assign_id;
124 int assign_type;
125
JOB_DESCJOB_DESC126 JOB_DESC() {
127 wu.clear();
128 command_line = NULL;
129 assign_flag = false;
130 assign_multi = false;
131 strcpy(wu_template_file, "");
132 strcpy(result_template_file, "");
133 strcpy(additional_xml, "");
134 assign_id = 0;
135 assign_type = ASSIGN_NONE;
136
137 // defaults (in case they're not in WU template)
138 //
139 wu.id = 0;
140 wu.min_quorum = 2;
141 wu.target_nresults = 2;
142 wu.max_error_results = 3;
143 wu.max_total_results = 10;
144 wu.max_success_results = 6;
145 wu.rsc_fpops_est = 3600e9;
146 wu.rsc_fpops_bound = 86400e9;
147 wu.rsc_memory_bound = 5e8;
148 wu.rsc_disk_bound = 1e9;
149 wu.rsc_bandwidth_bound = 0.0;
150 wu.delay_bound = 7*86400;
151
152 }
153 void create();
154 void parse_cmdline(int, char**);
155 };
156
157 // parse additional job-specific info when using --stdin
158 //
parse_cmdline(int argc,char ** argv)159 void JOB_DESC::parse_cmdline(int argc, char** argv) {
160 for (int i=0; i<argc; i++) {
161 if (arg(argv, i, (char*)"command_line")) {
162 command_line = argv[++i];
163 } else if (arg(argv, i, (char*)"wu_name")) {
164 safe_strcpy(wu.name, argv[++i]);
165 } else if (arg(argv, i, (char*)"wu_template")) {
166 safe_strcpy(wu_template_file, argv[++i]);
167 } else if (arg(argv, i, (char*)"result_template")) {
168 safe_strcpy(result_template_file, argv[++i]);
169 } else if (arg(argv, i, (char*)"remote_file")) {
170 INFILE_DESC id;
171 id.is_remote = true;
172 safe_strcpy(id.url, argv[++i]);
173 id.nbytes = atof(argv[++i]);
174 safe_strcpy(id.md5, argv[++i]);
175 infiles.push_back(id);
176 } else if (arg(argv, i, "target_host")) {
177 assign_flag = true;
178 assign_type = ASSIGN_HOST;
179 assign_id = atoi(argv[++i]);
180 check_assign_id(assign_id);
181 } else if (arg(argv, i, "target_user")) {
182 assign_flag = true;
183 assign_type = ASSIGN_USER;
184 assign_id = atoi(argv[++i]);
185 check_assign_id(assign_id);
186 } else {
187 if (!strncmp("-", argv[i], 1)) {
188 fprintf(stderr, "create_work: bad stdin argument '%s'\n", argv[i]);
189 exit(1);
190 }
191 INFILE_DESC id;
192 id.is_remote = false;
193 safe_strcpy(id.name, argv[i]);
194 infiles.push_back(id);
195 }
196 }
197 }
198
199 // See if WU template was given for job.
200 // Many jobs may have the same ones.
201 // To avoid rereading files, cache them in a map.
202 // Get from cache if there, else read the file and add to cache
203 //
get_wu_template(JOB_DESC & jd2)204 void get_wu_template(JOB_DESC& jd2) {
205 // the jobs may specify WU templates.
206 //
207 static map<string, char*> wu_templates;
208
209 string s = string(jd2.wu_template_file);
210 if (wu_templates.count(s) == 0) {
211 char* p;
212 int retval = read_file_malloc(jd2.wu_template_file, p, 0, false);
213 if (retval) {
214 fprintf(
215 stderr, "Can't read WU template %s\n", jd2.wu_template_file
216 );
217 exit(1);
218 }
219 wu_templates[s] = p;
220 }
221 strcpy(jd2.wu_template, wu_templates[s]);
222 }
223
main(int argc,char ** argv)224 int main(int argc, char** argv) {
225 DB_APP app;
226 int retval;
227 int i;
228 char download_dir[256], db_name[256], db_passwd[256];
229 char db_user[256],db_host[256];
230 char buf[4096];
231 JOB_DESC jd;
232 bool show_wu_name = true;
233 bool use_stdin = false;
234
235 strcpy(app.name, "");
236 strcpy(db_passwd, "");
237 const char* config_dir = 0;
238 i = 1;
239
240 while (i < argc) {
241 if (arg(argv, i, "appname")) {
242 safe_strcpy(app.name, argv[++i]);
243 } else if (arg(argv, i, "d")) {
244 int dl = atoi(argv[++i]);
245 log_messages.set_debug_level(dl);
246 if (dl ==4) g_print_queries = true;
247 } else if (arg(argv, i, "wu_name")) {
248 show_wu_name = false;
249 safe_strcpy(jd.wu.name, argv[++i]);
250 } else if (arg(argv, i, "wu_template")) {
251 safe_strcpy(jd.wu_template_file, argv[++i]);
252 } else if (arg(argv, i, "result_template")) {
253 safe_strcpy(jd.result_template_file, argv[++i]);
254 } else if (arg(argv, i, "config_dir")) {
255 config_dir = argv[++i];
256 } else if (arg(argv, i, "batch")) {
257 jd.wu.batch = atoi(argv[++i]);
258 } else if (arg(argv, i, "priority")) {
259 jd.wu.priority = atoi(argv[++i]);
260 } else if (arg(argv, i, "rsc_fpops_est")) {
261 jd.wu.rsc_fpops_est = atof(argv[++i]);
262 } else if (arg(argv, i, "rsc_fpops_bound")) {
263 jd.wu.rsc_fpops_bound = atof(argv[++i]);
264 } else if (arg(argv, i, "rsc_memory_bound")) {
265 jd.wu.rsc_memory_bound = atof(argv[++i]);
266 } else if (arg(argv, i, "size_class")) {
267 jd.wu.size_class = atoi(argv[++i]);
268 } else if (arg(argv, i, "rsc_disk_bound")) {
269 jd.wu.rsc_disk_bound = atof(argv[++i]);
270 } else if (arg(argv, i, "delay_bound")) {
271 jd.wu.delay_bound = atoi(argv[++i]);
272 } else if (arg(argv, i, "hr_class")) {
273 jd.wu.hr_class = atoi(argv[++i]);
274 } else if (arg(argv, i, "min_quorum")) {
275 jd.wu.min_quorum = atoi(argv[++i]);
276 } else if (arg(argv, i, "target_nresults")) {
277 jd.wu.target_nresults = atoi(argv[++i]);
278 } else if (arg(argv, i, "max_error_results")) {
279 jd.wu.max_error_results = atoi(argv[++i]);
280 } else if (arg(argv, i, "max_total_results")) {
281 jd.wu.max_total_results = atoi(argv[++i]);
282 } else if (arg(argv, i, "max_success_results")) {
283 jd.wu.max_success_results = atoi(argv[++i]);
284 } else if (arg(argv, i, "opaque")) {
285 jd.wu.opaque = atoi(argv[++i]);
286 } else if (arg(argv, i, "command_line")) {
287 jd.command_line= argv[++i];
288 } else if (arg(argv, i, "additional_xml")) {
289 safe_strcpy(jd.additional_xml, argv[++i]);
290 } else if (arg(argv, i, "wu_id")) {
291 jd.wu.id = atoi(argv[++i]);
292 } else if (arg(argv, i, "broadcast")) {
293 jd.assign_multi = true;
294 jd.assign_flag = true;
295 jd.assign_type = ASSIGN_NONE;
296 } else if (arg(argv, i, "broadcast_user")) {
297 jd.assign_flag = true;
298 jd.assign_type = ASSIGN_USER;
299 jd.assign_multi = true;
300 jd.assign_id = atoi(argv[++i]);
301 check_assign_id(jd.assign_id);
302 } else if (arg(argv, i, "broadcast_team")) {
303 jd.assign_flag = true;
304 jd.assign_type = ASSIGN_TEAM;
305 jd.assign_multi = true;
306 jd.assign_id = atoi(argv[++i]);
307 check_assign_id(jd.assign_id);
308 } else if (arg(argv, i, "target_host")) {
309 jd.assign_flag = true;
310 jd.assign_type = ASSIGN_HOST;
311 jd.assign_id = atoi(argv[++i]);
312 check_assign_id(jd.assign_id);
313 } else if (arg(argv, i, "target_user")) {
314 jd.assign_flag = true;
315 jd.assign_type = ASSIGN_USER;
316 jd.assign_id = atoi(argv[++i]);
317 check_assign_id(jd.assign_id);
318 } else if (arg(argv, i, "target_team")) {
319 jd.assign_flag = true;
320 jd.assign_type = ASSIGN_TEAM;
321 jd.assign_id = atoi(argv[++i]);
322 check_assign_id(jd.assign_id);
323 } else if (arg(argv, i, "help")) {
324 usage();
325 exit(0);
326 } else if (arg(argv, i, "stdin")) {
327 use_stdin = true;
328 } else if (arg(argv, i, (char*)"remote_file")) {
329 INFILE_DESC id;
330 id.is_remote = true;
331 safe_strcpy(id.url, argv[++i]);
332 id.nbytes = atof(argv[++i]);
333 safe_strcpy(id.md5, argv[++i]);
334 jd.infiles.push_back(id);
335 } else if (arg(argv, i, "verbose")) {
336 verbose = true;
337 } else if (arg(argv, i, "continue_on_error")) {
338 continue_on_error = true;
339 } else {
340 if (!strncmp("-", argv[i], 1)) {
341 fprintf(stderr, "create_work: bad argument '%s'\n", argv[i]);
342 exit(1);
343 }
344 INFILE_DESC id;
345 id.is_remote = false;
346 safe_strcpy(id.name, argv[i]);
347 jd.infiles.push_back(id);
348 }
349 i++;
350 }
351
352 if (!strlen(app.name)) {
353 usage();
354 }
355 if (!strlen(jd.wu.name)) {
356 sprintf(jd.wu.name, "%s_%d_%f", app.name, getpid(), dtime());
357 }
358 if (!strlen(jd.wu_template_file)) {
359 sprintf(jd.wu_template_file, "templates/%s_in", app.name);
360 }
361 if (!strlen(jd.result_template_file)) {
362 sprintf(jd.result_template_file, "templates/%s_out", app.name);
363 }
364
365 retval = config.parse_file(config_dir);
366 if (retval) {
367 fprintf(stderr, "Can't parse config file: %s\n", boincerror(retval));
368 exit(1);
369 } else {
370 strcpy(db_name, config.db_name);
371 strcpy(db_passwd, config.db_passwd);
372 strcpy(db_user, config.db_user);
373 strcpy(db_host, config.db_host);
374 strcpy(download_dir, config.download_dir);
375 }
376
377 retval = boinc_db.open(db_name, db_host, db_user, db_passwd);
378 if (retval) {
379 fprintf(stderr,
380 "create_work: error opening database: %s\n", boincerror(retval)
381 );
382 exit(1);
383 }
384 boinc_db.set_isolation_level(READ_UNCOMMITTED);
385 sprintf(buf, "where name='%s'", app.name);
386 retval = app.lookup(buf);
387 if (retval) {
388 fprintf(stderr, "create_work: app not found\n");
389 exit(1);
390 }
391
392 // read the WU template file.
393 // this won't get used if we're creating a batch
394 // with job-level WU templates
395 //
396 if (boinc_file_exists(jd.wu_template_file)) {
397 retval = read_filename(
398 jd.wu_template_file, jd.wu_template, sizeof(jd.wu_template)
399 );
400 if (retval) {
401 fprintf(stderr,
402 "create_work: can't open input template %s\n", jd.wu_template_file
403 );
404 exit(1);
405 }
406 }
407
408 jd.wu.appid = app.id;
409
410 strcpy(jd.result_template_path, "./");
411 strcat(jd.result_template_path, jd.result_template_file);
412
413 if (use_stdin) {
414 // clear the WU template name so we'll recognize a job-level one
415 //
416 strcpy(jd.wu_template_file, "");
417
418 if (jd.assign_flag) {
419 // if we're doing assignment we can't use the bulk-query method;
420 // create the jobs one at a time.
421 //
422 int _argc;
423 char* _argv[100];
424 for (int j=0; ; j++) {
425 char* p = fgets(buf, sizeof(buf), stdin);
426 if (p == NULL) break;
427 JOB_DESC jd2 = jd;
428 strcpy(jd2.wu.name, "");
429 _argc = parse_command_line(buf, _argv);
430 jd2.parse_cmdline(_argc, _argv);
431 if (!strlen(jd2.wu.name)) {
432 sprintf(jd2.wu.name, "%s_%d", jd.wu.name, j);
433 }
434 if (strlen(jd2.wu_template_file)) {
435 get_wu_template(jd2);
436 }
437 if (!strlen(jd2.wu_template)) {
438 fprintf(stderr, "job is missing input template\n");
439 exit(1);
440 }
441 jd2.create();
442 }
443 } else {
444 string values;
445 DB_WORKUNIT wu;
446 int _argc;
447 char* _argv[100], value_buf[MAX_QUERY_LEN];
448 for (int j=0; ; j++) {
449 char* p = fgets(buf, sizeof(buf), stdin);
450 if (p == NULL) break;
451 JOB_DESC jd2 = jd;
452 strcpy(jd2.wu.name, "");
453 _argc = parse_command_line(buf, _argv);
454 jd2.parse_cmdline(_argc, _argv);
455 if (!strlen(jd2.wu.name)) {
456 sprintf(jd2.wu.name, "%s_%d", jd.wu.name, j);
457 }
458 // if the stdin line specified assignment,
459 // create the job individually
460 //
461 if (jd2.assign_flag) {
462 jd2.create();
463 continue;
464 }
465 // otherwise accumulate a SQL query so that we can
466 // create jobs en masse
467 //
468 if (strlen(jd2.wu_template_file)) {
469 get_wu_template(jd2);
470 }
471 if (!strlen(jd2.wu_template)) {
472 fprintf(stderr, "job is missing input template\n");
473 exit(1);
474 }
475 retval = create_work2(
476 jd2.wu,
477 jd2.wu_template,
478 jd2.result_template_file,
479 jd2.result_template_path,
480 jd2.infiles,
481 config,
482 jd2.command_line,
483 jd2.additional_xml,
484 value_buf
485 );
486 if (retval) {
487 fprintf(stderr, "create_work() failed: %d\n", retval);
488 if (continue_on_error) {
489 continue;
490 } else {
491 exit(1);
492 }
493 }
494 if (values.size()) {
495 values += ",";
496 values += value_buf;
497 } else {
498 values = value_buf;
499 }
500 // MySQL can handles queries at least 1 MB
501 //
502 int n = strlen(value_buf);
503 if (values.size() + 2*n > 1000000) {
504 retval = wu.insert_batch(values);
505 if (retval) {
506 fprintf(stderr,
507 "wu.insert_batch() failed: %d; size %d\n",
508 retval, (int)values.size()
509 );
510 fprintf(stderr,
511 "MySQL error: %s\n", boinc_db.error_string()
512 );
513 exit(1);
514 }
515 values.clear();
516 }
517 }
518 if (values.size()) {
519 retval = wu.insert_batch(values);
520 if (retval) {
521 fprintf(stderr,
522 "wu.insert_batch() failed: %d\n", retval
523 );
524 fprintf(stderr,
525 "MySQL error: %s\n", boinc_db.error_string()
526 );
527 exit(1);
528 }
529 }
530 }
531 } else {
532 jd.create();
533 if (show_wu_name) {
534 printf("workunit name: %s\n", jd.wu.name);
535 }
536 }
537 boinc_db.close();
538 }
539
create()540 void JOB_DESC::create() {
541 if (assign_flag) {
542 wu.transitioner_flags = assign_multi?TRANSITION_NONE:TRANSITION_NO_NEW_RESULTS;
543 }
544 int retval = create_work2(
545 wu,
546 wu_template,
547 result_template_file,
548 result_template_path,
549 infiles,
550 config,
551 command_line,
552 additional_xml
553 );
554 if (retval) {
555 fprintf(stderr, "create_work: %s\n", boincerror(retval));
556 exit(1);
557 }
558 if (verbose) {
559 fprintf(stderr, "created workunit; name %s, ID %lu\n", wu.name, wu.id);
560 }
561 if (assign_flag) {
562 DB_ASSIGNMENT assignment;
563 assignment.clear();
564 assignment.create_time = time(0);
565 assignment.target_id = assign_id;
566 assignment.target_type = assign_type;
567 assignment.multi = assign_multi;
568 assignment.workunitid = wu.id;
569 retval = assignment.insert();
570 if (retval) {
571 fprintf(stderr,
572 "assignment.insert() failed: %s\n", boincerror(retval)
573 );
574 exit(1);
575 }
576 }
577 }
578
579 const char *BOINC_RCSID_3865dbbf46 = "$Id$";
580