1 // This file is part of BOINC.
2 // http://boinc.berkeley.edu
3 // Copyright (C) 2008 University of California
4 //
5 // BOINC is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU Lesser General Public License
7 // as published by the Free Software Foundation,
8 // either version 3 of the License, or (at your option) any later version.
9 //
10 // BOINC is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 // See the GNU Lesser General Public License for more details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
17 
18 // initialization and starting of applications
19 
20 #include "cpp.h"
21 
22 #ifdef _WIN32
23 #include "boinc_win.h"
24 #include "win_util.h"
25 #define unlink   _unlink
26 #ifdef _MSC_VER
27 #define snprintf _snprintf
28 #define strdup   _strdup
29 #define getcwd   _getcwd
30 #endif
31 #else
32 #include "config.h"
33 #if HAVE_SCHED_SETSCHEDULER && defined (__linux__)
34 #include <sched.h>
35 #endif
36 #if HAVE_SYS_TIME_H
37 #include <sys/time.h>
38 #endif
39 #if HAVE_SYS_RESOURCE_H
40 #include <sys/resource.h>
41 #endif
42 #if HAVE_SYS_IPC_H
43 #include <sys/ipc.h>
44 #endif
45 #if HAVE_SYS_WAIT_H
46 #include <sys/wait.h>
47 #endif
48 #include <unistd.h>
49 #include <cerrno>
50 #include <sys/stat.h>
51 #include <string>
52 #endif
53 
54 #ifdef __EMX__
55 #include <process.h>
56 #endif
57 
58 #if (defined (__APPLE__) && (defined(__i386__) || defined(__x86_64__)))
59 #include <mach-o/loader.h>
60 #include <mach-o/fat.h>
61 #include <mach/machine.h>
62 #include <libkern/OSByteOrder.h>
63 #endif
64 
65 #if(!defined (_WIN32) && !defined (__EMX__))
66 #include <fcntl.h>
67 #endif
68 
69 #include <vector>
70 
71 using std::vector;
72 using std::string;
73 
74 #include "base64.h"
75 #include "error_numbers.h"
76 #include "filesys.h"
77 #include "shmem.h"
78 #include "str_replace.h"
79 #include "str_util.h"
80 #include "util.h"
81 
82 #include "async_file.h"
83 #include "client_msgs.h"
84 #include "client_state.h"
85 #include "file_names.h"
86 #include "result.h"
87 #include "sandbox.h"
88 
89 #ifdef _WIN32
90 #include "run_app_windows.h"
91 #endif
92 
93 #include "cs_proxy.h"
94 
95 #include "app.h"
96 
97 
98 #ifdef _WIN32
99 // Dynamically link to these functions at runtime;
100 // otherwise BOINC cannot run on Win98
101 
102 // CreateEnvironmentBlock
103 typedef BOOL (WINAPI *tCEB)(LPVOID *lpEnvironment, HANDLE hToken, BOOL bInherit);
104 // DestroyEnvironmentBlock
105 typedef BOOL (WINAPI *tDEB)(LPVOID lpEnvironment);
106 
107 #endif
108 
109 // print each string in an array
110 //
111 #ifndef _WIN32
debug_print_argv(char ** argv)112 static void debug_print_argv(char** argv) {
113     msg_printf(0, MSG_INFO, "[task] Arguments:");
114     for (int i=0; argv[i]; i++) {
115         msg_printf(0, MSG_INFO, "[task]    argv[%d]: %s\n", i, argv[i]);
116     }
117 }
118 #endif
119 
120 // For apps that use coprocessors, append "--device x" to the command line.
121 // NOTE: this is deprecated.  Use app_init_data instead.
122 //
coproc_cmdline(int rsc_type,RESULT * rp,double ninstances,char * cmdline,int cmdline_len)123 static void coproc_cmdline(
124     int rsc_type, RESULT* rp, double ninstances, char* cmdline, int cmdline_len
125 ) {
126     char buf[256];
127     COPROC* coproc = &coprocs.coprocs[rsc_type];
128     for (int j=0; j<ninstances; j++) {
129         int k = rp->coproc_indices[j];
130         // sanity check
131         //
132         if (k < 0 || k >= coproc->count) {
133             msg_printf(0, MSG_INTERNAL_ERROR,
134                 "coproc_cmdline: coproc index %d out of range", k
135             );
136             k = 0;
137         }
138         sprintf(buf, " --device %d", coproc->device_nums[k]);
139         strlcat(cmdline, buf, cmdline_len);
140     }
141 }
142 
143 // Make a unique key for client/app shared memory segment.
144 // Windows: also create and attach to the segment.
145 //
get_shmem_seg_name()146 int ACTIVE_TASK::get_shmem_seg_name() {
147 #ifdef _WIN32
148     int i;
149     char seg_name[256];
150 
151     bool try_global = (sandbox_account_service_token != NULL);
152     for (i=0; i<1024; i++) {
153         sprintf(seg_name, "%sboinc_%d", SHM_PREFIX, i);
154         shm_handle = create_shmem(
155             seg_name, sizeof(SHARED_MEM), (void**)&app_client_shm.shm,
156             try_global
157         );
158         if (shm_handle) break;
159     }
160     if (!shm_handle) return ERR_SHMGET;
161     sprintf(shmem_seg_name, "boinc_%d", i);
162 #else
163     char init_data_path[MAXPATHLEN];
164 #ifndef __EMX__
165     // shmem_seg_name is not used with mmap() shared memory
166     if (app_version->api_version_at_least(6, 0)) {
167         shmem_seg_name = -1;
168         return 0;
169     }
170 #endif
171     sprintf(init_data_path, "%s/%s", slot_dir, INIT_DATA_FILE);
172 
173     // ftok() only works if there's a file at the given location
174     //
175     if (!boinc_file_exists(init_data_path)) {
176         FILE* f = boinc_fopen(init_data_path, "w");
177         if (f) {
178             fclose(f);
179         } else {
180             msg_printf(wup->project, MSG_INTERNAL_ERROR,
181                 "error: can't open file for shmem seg name"
182             );
183         }
184     }
185     shmem_seg_name = ftok(init_data_path, 1);
186     if (shmem_seg_name == -1) {
187         msg_printf(wup->project, MSG_INTERNAL_ERROR,
188             "error: can't open file for shmem seg name: %d", errno
189         );
190         perror("ftok");
191         return ERR_SHMEM_NAME;
192     }
193 #endif
194     return 0;
195 }
196 
init_app_init_data(APP_INIT_DATA & aid)197 void ACTIVE_TASK::init_app_init_data(APP_INIT_DATA& aid) {
198     PROJECT* project = wup->project;
199     aid.major_version = BOINC_MAJOR_VERSION;
200     aid.minor_version = BOINC_MINOR_VERSION;
201     aid.release = BOINC_RELEASE;
202     aid.app_version = app_version->version_num;
203     safe_strcpy(aid.app_name, wup->app->name);
204     safe_strcpy(aid.symstore, project->symstore);
205     safe_strcpy(aid.acct_mgr_url, gstate.acct_mgr_info.master_url);
206     if (project->project_specific_prefs.length()) {
207         aid.project_preferences = strdup(
208             project->project_specific_prefs.c_str()
209         );
210     }
211     aid.userid = project->userid;
212     aid.teamid = project->teamid;
213     aid.hostid = project->hostid;
214     safe_strcpy(aid.user_name, project->user_name);
215     safe_strcpy(aid.team_name, project->team_name);
216     safe_strcpy(aid.project_dir, project->project_dir_absolute());
217     relative_to_absolute("", aid.boinc_dir);
218     safe_strcpy(aid.authenticator, project->authenticator);
219     aid.slot = slot;
220 #ifdef _WIN32
221     aid.client_pid = GetCurrentProcessId();
222 #else
223     aid.client_pid = getpid();
224 #endif
225     safe_strcpy(aid.wu_name, wup->name);
226     safe_strcpy(aid.result_name, result->name);
227     aid.user_total_credit = project->user_total_credit;
228     aid.user_expavg_credit = project->user_expavg_credit;
229     aid.host_total_credit = project->host_total_credit;
230     aid.host_expavg_credit = project->host_expavg_credit;
231     double rrs = gstate.runnable_resource_share(RSC_TYPE_CPU);
232     if (rrs) {
233         aid.resource_share_fraction = project->resource_share/rrs;
234     } else {
235         aid.resource_share_fraction = 1;
236     }
237     aid.host_info = gstate.host_info;
238     aid.proxy_info = working_proxy_info;
239     aid.global_prefs = gstate.global_prefs;
240     aid.starting_elapsed_time = checkpoint_elapsed_time;
241     aid.using_sandbox = g_use_sandbox;
242     aid.vm_extensions_disabled = gstate.host_info.p_vm_extensions_disabled;
243     aid.rsc_fpops_est = wup->rsc_fpops_est;
244     aid.rsc_fpops_bound = wup->rsc_fpops_bound;
245     aid.rsc_memory_bound = wup->rsc_memory_bound;
246     aid.rsc_disk_bound = wup->rsc_disk_bound;
247     aid.computation_deadline = result->computation_deadline();
248     int rt = app_version->gpu_usage.rsc_type;
249     if (rt) {
250         COPROC& cp = coprocs.coprocs[rt];
251         if (coproc_type_name_to_num(cp.type) >= 0) {
252             // Standardized vendor name ("NVIDIA", "ATI" or "intel_gpu")
253             safe_strcpy(aid.gpu_type, cp.type);
254         } else {
255             // For other vendors, use vendor name as returned by OpenCL
256             safe_strcpy(aid.gpu_type, cp.opencl_prop.vendor);
257         }
258         int k = result->coproc_indices[0];
259         if (k<0 || k>=cp.count) {
260             msg_printf(0, MSG_INTERNAL_ERROR,
261                 "init_app_init_data(): coproc index %d out of range", k
262             );
263             k = 0;
264         }
265         aid.gpu_device_num = cp.device_nums[k];
266         aid.gpu_opencl_dev_index = cp.opencl_device_indexes[k];
267         aid.gpu_usage = app_version->gpu_usage.usage;
268     } else {
269         safe_strcpy(aid.gpu_type, "");
270         aid.gpu_device_num = -1;
271         aid.gpu_opencl_dev_index = -1;
272         aid.gpu_usage = 0;
273     }
274     aid.ncpus = app_version->avg_ncpus;
275     aid.vbox_window = cc_config.vbox_window;
276     aid.checkpoint_period = gstate.global_prefs.disk_interval;
277     aid.fraction_done_start = 0;
278     aid.fraction_done_end = 1;
279 #ifdef _WIN32
280     safe_strcpy(aid.shmem_seg_name, shmem_seg_name);
281 #else
282     aid.shmem_seg_name = shmem_seg_name;
283 #endif
284     aid.wu_cpu_time = checkpoint_cpu_time;
285     APP_VERSION* avp = app_version;
286     for (unsigned int i=0; i<avp->app_files.size(); i++) {
287         FILE_REF& fref = avp->app_files[i];
288         aid.app_files.push_back(string(fref.file_name));
289     }
290 }
291 
292 // write the app init file.
293 // This is done before starting or restarting the app,
294 // and when project prefs have changed during app execution
295 //
write_app_init_file(APP_INIT_DATA & aid)296 int ACTIVE_TASK::write_app_init_file(APP_INIT_DATA& aid) {
297     FILE *f;
298     char init_data_path[MAXPATHLEN];
299 
300 #if 0
301     msg_printf(wup->project, MSG_INFO,
302         "writing app_init.xml for %s; slot %d rt %s gpu_device_num %d", result->name, slot, aid.gpu_type, aid.gpu_device_num
303     );
304 #endif
305 
306     sprintf(init_data_path, "%s/%s", slot_dir, INIT_DATA_FILE);
307 
308     // delete the file using the switcher (Unix)
309     // in case it's owned by another user and we don't have write access
310     //
311     delete_project_owned_file(init_data_path, false);
312     f = boinc_fopen(init_data_path, "w");
313     if (!f) {
314         msg_printf(wup->project, MSG_INTERNAL_ERROR,
315             "Failed to open init file %s",
316             init_data_path
317         );
318         return ERR_FOPEN;
319     }
320 
321     int retval = write_init_data_file(f, aid);
322     fclose(f);
323     return retval;
324 }
325 
326 // Given a logical name of the form D1/D2/.../Dn/F,
327 // create the directories D1 ... Dn in the slot dir
328 //
create_dirs_for_logical_name(const char * name,const char * slot_dir)329 static int create_dirs_for_logical_name(
330     const char* name, const char* slot_dir
331 ) {
332     char buf[1024];
333     char dir_path[MAXPATHLEN];
334     int retval;
335 
336     safe_strcpy(buf, name);
337     safe_strcpy(dir_path, slot_dir);
338     char* p = buf;
339     while (1) {
340         char* q = strstr(p, "/");
341         if (!q) break;
342         *q = 0;
343         safe_strcat(dir_path, "/");
344         safe_strcat(dir_path, p);
345         retval = boinc_mkdir(dir_path);
346         if (retval) return retval;
347         p = q+1;
348     }
349     return 0;
350 }
351 
prepend_prefix(APP_VERSION * avp,char * in,char * out,int len)352 static void prepend_prefix(APP_VERSION* avp, char* in, char* out, int len) {
353     if (strlen(avp->file_prefix)) {
354         snprintf(out, len, "%s/%s", avp->file_prefix, in);
355     } else {
356         strlcpy(out, in, len);
357     }
358 }
359 
360 // an input/output file must be copied if either
361 // - the FILE_REFERENCE says so or
362 // - the APP_VERSION has a non-empty file_prefix
363 //
must_copy_file(FILE_REF & fref,bool is_io_file)364 bool ACTIVE_TASK::must_copy_file(FILE_REF& fref, bool is_io_file) {
365     if (fref.copy_file) return true;
366     if (is_io_file && strlen(app_version->file_prefix)) return true;
367     return false;
368 }
369 
370 // set up a file reference, given a slot dir and project dir.
371 // This means:
372 // 1) copy the file to slot dir, if reference is by copy
373 // 2) else make a soft link
374 //
setup_file(FILE_INFO * fip,FILE_REF & fref,char * file_path,bool input,bool is_io_file)375 int ACTIVE_TASK::setup_file(
376     FILE_INFO* fip, FILE_REF& fref, char* file_path, bool input, bool is_io_file
377 ) {
378     char link_path[MAXPATHLEN], rel_file_path[MAXPATHLEN], open_name[256];
379     int retval;
380     PROJECT* project = result->project;
381 
382     if (log_flags.slot_debug) {
383         msg_printf(wup->project, MSG_INFO,
384             "setup_file: %s (%s)", file_path, input?"input":"output"
385         );
386     }
387 
388     if (strlen(fref.open_name)) {
389         if (is_io_file) {
390             prepend_prefix(
391                 app_version, fref.open_name, open_name, sizeof(open_name)
392             );
393         } else {
394             safe_strcpy(open_name, fref.open_name);
395         }
396         retval = create_dirs_for_logical_name(open_name, slot_dir);
397         if (retval) return retval;
398         snprintf(link_path, sizeof(link_path), "%s/%s", slot_dir, open_name);
399     } else {
400         snprintf(link_path, sizeof(link_path), "%s/%s", slot_dir, fip->name);
401     }
402 
403     snprintf(rel_file_path, sizeof(rel_file_path), "../../%s", file_path );
404 
405     if (boinc_file_exists(link_path)) {
406         return 0;
407     }
408 
409     if (must_copy_file(fref, is_io_file)) {
410         if (input) {
411             // the file may be there already (async copy case)
412             //
413             if (boinc_file_exists(link_path)) {
414                 return 0;
415             }
416             if (fip->nbytes > ASYNC_FILE_THRESHOLD) {
417                 ASYNC_COPY* ac = new ASYNC_COPY;
418                 retval = ac->init(this, fip, file_path, link_path);
419                 if (retval) return retval;
420                 return ERR_IN_PROGRESS;
421             } else {
422                 retval = boinc_copy(file_path, link_path);
423                 if (retval) {
424                     msg_printf(project, MSG_INTERNAL_ERROR,
425                         "Can't copy %s to %s: %s", file_path, link_path,
426                         boincerror(retval)
427                     );
428                     return retval;
429                 }
430                 retval = fip->set_permissions(link_path);
431                 if (retval) return retval;
432             }
433         }
434         return 0;
435     }
436 
437 #ifdef _WIN32
438     retval = make_soft_link(project, link_path, rel_file_path);
439     if (retval) return retval;
440 #else
441     if (project->use_symlinks) {
442         retval = symlink(rel_file_path, link_path);
443     } else {
444         retval = make_soft_link(project, link_path, rel_file_path);
445     }
446     if (retval) return retval;
447 #endif
448     if (g_use_sandbox) set_to_project_group(link_path);
449     return 0;
450 }
451 
link_user_files()452 int ACTIVE_TASK::link_user_files() {
453     PROJECT* project = wup->project;
454     unsigned int i;
455     FILE_REF fref;
456     FILE_INFO* fip;
457     char file_path[MAXPATHLEN];
458 
459     for (i=0; i<project->user_files.size(); i++) {
460         fref = project->user_files[i];
461         fip = fref.file_info;
462         if (fip->status != FILE_PRESENT) continue;
463         get_pathname(fip, file_path, sizeof(file_path));
464         setup_file(fip, fref, file_path, true, false);
465     }
466     return 0;
467 }
468 
copy_output_files()469 int ACTIVE_TASK::copy_output_files() {
470     char slotfile[256], projfile[256], open_name[256];
471     unsigned int i;
472     for (i=0; i<result->output_files.size(); i++) {
473         FILE_REF& fref = result->output_files[i];
474         if (!must_copy_file(fref, true)) continue;
475         FILE_INFO* fip = fref.file_info;
476         prepend_prefix(
477             app_version, fref.open_name, open_name, sizeof(open_name)
478         );
479         snprintf(slotfile, sizeof(slotfile), "%s/%s", slot_dir, open_name);
480         get_pathname(fip, projfile, sizeof(projfile));
481         int retval = boinc_rename(slotfile, projfile);
482         // the rename fails if the output file isn't there.
483         //
484         if (retval) {
485             if (retval == ERR_FILE_MISSING) {
486                 if (log_flags.slot_debug) {
487                     msg_printf(wup->project, MSG_INFO,
488                         "[slot] output file %s missing, not copying", slotfile
489                     );
490                 }
491             } else {
492                 msg_printf(wup->project, MSG_INTERNAL_ERROR,
493                     "Can't rename output file %s to %s: %s",
494                     slotfile, projfile, boincerror(retval)
495                 );
496             }
497         } else {
498             if (log_flags.slot_debug) {
499                 msg_printf(wup->project, MSG_INFO,
500                     "[slot] renamed %s to %s", slotfile, projfile
501                 );
502             }
503         }
504     }
505     return 0;
506 }
507 
get_priority(bool is_high_priority)508 static int get_priority(bool is_high_priority) {
509     int p = is_high_priority?cc_config.process_priority_special:cc_config.process_priority;
510 #ifdef _WIN32
511     switch (p) {
512     case 0: return IDLE_PRIORITY_CLASS;
513     case 1: return BELOW_NORMAL_PRIORITY_CLASS;
514     case 2: return NORMAL_PRIORITY_CLASS;
515     case 3: return ABOVE_NORMAL_PRIORITY_CLASS;
516     case 4: return HIGH_PRIORITY_CLASS;
517     case 5: return REALTIME_PRIORITY_CLASS;
518     }
519     return is_high_priority ? BELOW_NORMAL_PRIORITY_CLASS : IDLE_PRIORITY_CLASS;
520 #else
521     switch (p) {
522     case 0: return PROCESS_IDLE_PRIORITY;
523     case 1: return PROCESS_MEDIUM_PRIORITY;
524     case 2: return PROCESS_NORMAL_PRIORITY;
525     case 3: return PROCESS_ABOVE_NORMAL_PRIORITY;
526     case 4: return PROCESS_HIGH_PRIORITY;
527     case 5: return PROCESS_REALTIME_PRIORITY;
528     }
529     return is_high_priority ? PROCESS_MEDIUM_PRIORITY : PROCESS_IDLE_PRIORITY;
530 #endif
531 }
532 
533 // Start a task in a slot directory.
534 // This includes setting up soft links,
535 // passing preferences, and starting the process
536 //
537 // Current dir is top-level BOINC dir
538 //
539 // postcondition:
540 // If any error occurs
541 //   ACTIVE_TASK::task_state is PROCESS_COULDNT_START
542 //   report_result_error() is called
543 // else
544 //   ACTIVE_TASK::task_state is PROCESS_EXECUTING
545 //
546 // If "test" is set, we're doing the API test; just run "test_app".
547 //
start(bool test)548 int ACTIVE_TASK::start(bool test) {
549     char exec_name[256], file_path[MAXPATHLEN], buf[MAXPATHLEN], exec_path[MAXPATHLEN];
550     char cmdline[80000];    // 64KB plus some extra
551     unsigned int i;
552     FILE_REF fref;
553     FILE_INFO* fip;
554     int retval;
555     APP_INIT_DATA aid;
556 #ifdef _WIN32
557     bool success = false;
558     LPVOID environment_block=NULL;
559 #endif
560 
561     if (async_copy) {
562         if (log_flags.task_debug) {
563             msg_printf(wup->project, MSG_INFO,
564                 "[task_debug] ACTIVE_TASK::start(): async file copy already in progress"
565             );
566         }
567         return 0;
568     }
569 
570     // run it at above idle priority if it
571     // - uses coprocs
572     // - uses less than one CPU
573     // - is a wrapper
574     //
575     bool high_priority = false;
576     if (app_version->rsc_type()) high_priority = true;
577     if (app_version->avg_ncpus < 1) high_priority = true;
578     if (app_version->is_wrapper) high_priority = true;
579 
580     if (wup->project->verify_files_on_app_start) {
581         fip=0;
582         retval = gstate.input_files_available(result, true, &fip);
583         if (retval) {
584             if (fip) {
585                 snprintf(
586                     buf, sizeof(buf),
587                     "Input file %s missing or invalid: %s",
588                     fip->name, boincerror(retval)
589                 );
590             } else {
591                 safe_strcpy(buf, "Input file missing or invalid");
592             }
593             goto error;
594         }
595     }
596 
597     current_cpu_time = checkpoint_cpu_time;
598     elapsed_time = checkpoint_elapsed_time;
599 
600     graphics_request_queue.init(result->name);        // reset message queues
601     process_control_queue.init(result->name);
602 
603     bytes_sent_episode = 0;
604     bytes_received_episode = 0;
605 
606     if (!app_client_shm.shm) {
607         retval = get_shmem_seg_name();
608         if (retval) {
609             snprintf(buf, sizeof(buf),
610                 "Can't get shared memory segment name: %s",
611                 boincerror(retval)
612             );
613             goto error;
614         }
615     }
616 
617     // this must go AFTER creating shmem name,
618     // since the shmem name is part of the file
619     //
620     init_app_init_data(aid);
621     retval = write_app_init_file(aid);
622     if (retval) {
623         snprintf(buf, sizeof(buf), "Can't write init file: %s", boincerror(retval));
624         goto error;
625     }
626 
627     // set up applications files
628     //
629     if (test) {
630         safe_strcpy(exec_name, "test_app");
631         safe_strcpy(exec_path, "test_app");
632     } else {
633         safe_strcpy(exec_name, "");
634     }
635     for (i=0; i<app_version->app_files.size(); i++) {
636         fref = app_version->app_files[i];
637         fip = fref.file_info;
638         get_pathname(fip, file_path, sizeof(file_path));
639         if (fref.main_program) {
640             if (is_image_file(fip->name)) {
641                 snprintf(buf, sizeof(buf), "Main program %s is an image file", fip->name);
642                 retval = ERR_NO_SIGNATURE;
643                 goto error;
644             }
645             if (!fip->executable && !wup->project->anonymous_platform) {
646                 snprintf(buf, sizeof(buf), "Main program %s is not executable", fip->name);
647                 retval = ERR_NO_SIGNATURE;
648                 goto error;
649             }
650             safe_strcpy(exec_name, fip->name);
651             safe_strcpy(exec_path, file_path);
652         }
653         retval = setup_file(fip, fref, file_path, true, false);
654         if (retval == ERR_IN_PROGRESS) {
655             set_task_state(PROCESS_COPY_PENDING, "start");
656             return 0;
657         } else if (retval) {
658             safe_strcpy(buf, "Can't link app version file");
659             goto error;
660         }
661     }
662     if (!strlen(exec_name)) {
663         safe_strcpy(buf, "No main program specified");
664         retval = ERR_NOT_FOUND;
665         goto error;
666     }
667 
668     // set up input, output files
669     //
670     for (i=0; i<wup->input_files.size(); i++) {
671         fref = wup->input_files[i];
672         fip = fref.file_info;
673         get_pathname(fref.file_info, file_path, sizeof(file_path));
674         retval = setup_file(fip, fref, file_path, true, true);
675         if (retval == ERR_IN_PROGRESS) {
676             set_task_state(PROCESS_COPY_PENDING, "start");
677             return 0;
678         } else if (retval) {
679             safe_strcpy(buf, "Can't link input file");
680             goto error;
681         }
682     }
683     for (i=0; i<result->output_files.size(); i++) {
684         fref = result->output_files[i];
685         if (must_copy_file(fref, true)) continue;
686         fip = fref.file_info;
687         get_pathname(fref.file_info, file_path, sizeof(file_path));
688         retval = setup_file(fip, fref, file_path, false, true);
689         if (retval) {
690             safe_strcpy(buf, "Can't link output file");
691             goto error;
692         }
693     }
694 
695     link_user_files();
696         // don't check retval here
697 
698     // remove temporary exit file from last run
699     //
700     snprintf(file_path, sizeof(file_path), "%s/%s", slot_dir, TEMPORARY_EXIT_FILE);
701     delete_project_owned_file(file_path, true);
702 
703     if (cc_config.exit_before_start) {
704         msg_printf(0, MSG_INFO, "about to start a job; exiting");
705         exit(0);
706     }
707 
708 #ifdef _WIN32
709     PROCESS_INFORMATION process_info;
710     STARTUPINFO startup_info;
711     char slotdirpath[MAXPATHLEN];
712     char error_msg[1024];
713     char error_msg2[1024];
714     DWORD last_error = 0;
715 
716     memset(&process_info, 0, sizeof(process_info));
717     memset(&startup_info, 0, sizeof(startup_info));
718     startup_info.cb = sizeof(startup_info);
719 
720     // suppress 2-sec rotating hourglass cursor on startup
721     //
722     startup_info.dwFlags = STARTF_FORCEOFFFEEDBACK;
723 
724     app_client_shm.reset_msgs();
725 
726     if (cc_config.run_apps_manually) {
727         // fill in client's PID so we won't think app has exited
728         //
729         pid = GetCurrentProcessId();
730         process_handle = GetCurrentProcess();
731         set_task_state(PROCESS_EXECUTING, "start");
732         return 0;
733     }
734 
735     snprintf(cmdline, sizeof(cmdline),
736         "%s %s %s",
737         exec_path, wup->command_line.c_str(), app_version->cmdline
738     );
739     if (!app_version->api_version_at_least(7, 5)) {
740         int rt = app_version->gpu_usage.rsc_type;
741         if (rt) {
742             coproc_cmdline(rt, result, app_version->gpu_usage.usage, cmdline, sizeof(cmdline));
743         }
744     }
745 
746     relative_to_absolute(slot_dir, slotdirpath);
747     int prio_mask;
748     if (cc_config.no_priority_change) {
749         prio_mask = 0;
750     } else {
751         prio_mask = get_priority(high_priority);
752     }
753 
754     for (i=0; i<5; i++) {
755         last_error = 0;
756         if (sandbox_account_service_token != NULL) {
757 
758             if (!CreateEnvironmentBlock(&environment_block, sandbox_account_service_token, FALSE)) {
759                 if (log_flags.task) {
760                     windows_format_error_string(GetLastError(), error_msg, sizeof(error_msg));
761                     msg_printf(wup->project, MSG_INFO,
762                         "Process environment block creation failed: %s", error_msg
763                     );
764                 }
765             }
766 
767             if (CreateProcessAsUser(
768                 sandbox_account_service_token,
769                 exec_path,
770                 cmdline,
771                 NULL,
772                 NULL,
773                 FALSE,
774                 CREATE_NEW_PROCESS_GROUP|CREATE_NO_WINDOW|prio_mask|CREATE_UNICODE_ENVIRONMENT,
775                 environment_block,
776                 slotdirpath,
777                 &startup_info,
778                 &process_info
779             )) {
780                 success = true;
781                 break;
782             } else {
783                 last_error = GetLastError();
784                 windows_format_error_string(last_error, error_msg, sizeof(error_msg));
785                 msg_printf(wup->project, MSG_INTERNAL_ERROR,
786                     "Process creation failed: %s - error code %d (0x%x)",
787                     error_msg, last_error, last_error
788                 );
789             }
790 
791             if (!DestroyEnvironmentBlock(environment_block)) {
792                 if (log_flags.task) {
793                     windows_format_error_string(GetLastError(), error_msg, sizeof(error_msg2));
794                     msg_printf(wup->project, MSG_INFO,
795                         "Process environment block cleanup failed: %s",
796                         error_msg2
797                     );
798                 }
799             }
800 
801         } else {
802             if (CreateProcess(
803                 exec_path,
804                 cmdline,
805                 NULL,
806                 NULL,
807                 FALSE,
808                 CREATE_NEW_PROCESS_GROUP|CREATE_NO_WINDOW|prio_mask,
809                 NULL,
810                 slotdirpath,
811                 &startup_info,
812                 &process_info
813             )) {
814                 success = true;
815                 break;
816             } else {
817                 last_error = GetLastError();
818                 windows_format_error_string(last_error, error_msg, sizeof(error_msg));
819                 msg_printf(wup->project, MSG_INTERNAL_ERROR,
820                     "Process creation failed: %s - error code %d (0x%x)",
821                     error_msg, last_error, last_error
822                 );
823             }
824         }
825         boinc_sleep(drand());
826     }
827 
828     if (!success) {
829         snprintf(buf, sizeof(buf), "CreateProcess() failed - %s", error_msg);
830 
831         if (last_error == ERROR_NOT_ENOUGH_MEMORY) {
832             // if CreateProcess() failed because system is low on memory,
833             // treat this like a temporary exit;
834             // retry in 10 min, and give up after 100 times
835             //
836             bool will_restart;
837             handle_temporary_exit(will_restart, 600, "not enough memory", false);
838             if (will_restart) return 0;
839         }
840         retval = ERR_EXEC;
841         goto error;
842     }
843     pid = process_info.dwProcessId;
844     process_handle = process_info.hProcess;
845     CloseHandle(process_info.hThread);  // thread handle is not used
846 #elif defined(__EMX__)
847 
848     char* argv[100];
849     char current_dir[_MAX_PATH];
850 
851     // Set up client/app shared memory seg if needed
852     //
853     if (!app_client_shm.shm) {
854         retval = create_shmem(
855             shmem_seg_name, sizeof(SHARED_MEM), (void**)&app_client_shm.shm
856         );
857         if (retval) {
858             return retval;
859         }
860     }
861     app_client_shm.reset_msgs();
862 
863     // save current dir
864     getcwd(current_dir, sizeof(current_dir));
865 
866     // chdir() into the slot directory
867     //
868     retval = chdir(slot_dir);
869     if (retval) {
870         sprintf(buf, "Can't change directory to %s: %s", slot_dir, boincerror(retval));
871         goto error;
872     }
873 
874     // hook up stderr to a specially-named file
875     //
876     //freopen(STDERR_FILE, "a", stderr);
877 
878     argv[0] = exec_name;
879     safe_strcpy(cmdline, wup->command_line.c_str());
880     if (strlen(result->cmdline)) {
881         safe_strcat(cmdline, " ");
882         safe_strcat(cmdline, result->cmdline);
883     }
884     parse_command_line(cmdline, argv+1);
885     if (log_flags.task_debug) {
886         debug_print_argv(argv);
887     }
888     snprintf(buf, sizeof(buf), "../../%s", exec_path );
889     pid = spawnv(P_NOWAIT, buf, argv);
890     if (pid == -1) {
891         snprintf(buf, sizeof(buf), "Process creation failed: %s\n", boincerror(retval));
892         chdir(current_dir);
893         retval = ERR_EXEC;
894         goto error;
895     }
896 
897     // restore current dir
898     chdir(current_dir);
899 
900     if (log_flags.task_debug) {
901         msg_printf(wup->project, MSG_INFO,
902             "[task] ACTIVE_TASK::start(): forked process: pid %d\n", pid
903         );
904     }
905 
906     if (!cc_config.no_priority_change) {
907         int priority = get_priority(high_priority);
908         if (setpriority(PRIO_PROCESS, pid, priority)) {
909             perror("setpriority");
910         }
911     }
912 
913 #else
914     // Unix/Linux/Mac case
915 
916     char* argv[100];
917     char current_dir[1024];
918 
919     if (getcwd(current_dir, sizeof(current_dir)) == NULL) {
920         snprintf(buf, sizeof(buf), "Can't get cwd");
921         goto error;
922     }
923 
924     snprintf(cmdline, sizeof(cmdline),
925         "%s %s",
926         wup->command_line.c_str(), app_version->cmdline
927     );
928 
929     if (!app_version->api_version_at_least(7, 5)) {
930         int rt = app_version->gpu_usage.rsc_type;
931         if (rt) {
932             coproc_cmdline(rt, result, app_version->gpu_usage.usage, cmdline, sizeof(cmdline));
933         }
934     }
935 
936     // Set up client/app shared memory seg if needed
937     //
938     if (!app_client_shm.shm) {
939 #ifdef ANDROID
940         if (true) {
941 #else
942         if (app_version->api_version_at_least(6, 0)) {
943 #endif
944             // Use mmap() shared memory
945             snprintf(buf, sizeof(buf), "%s/%s", slot_dir, MMAPPED_FILE_NAME);
946             if (g_use_sandbox) {
947                 if (!boinc_file_exists(buf)) {
948                     int fd = open(buf, O_RDWR | O_CREAT, 0660);
949                     if (fd >= 0) {
950                         close (fd);
951                         if (g_use_sandbox) set_to_project_group(buf);
952                     }
953                 }
954             }
955             retval = create_shmem_mmap(
956                 buf, sizeof(SHARED_MEM), (void**)&app_client_shm.shm
957             );
958             if (retval) {
959                 msg_printf(wup->project, MSG_INTERNAL_ERROR,
960                     "ACTIVE_TASK::start(): can't create memory-mapped file: %s",
961                     boincerror(retval)
962                 );
963                 return retval;
964             }
965         } else {
966             // Use shmget() shared memory
967             retval = create_shmem(
968                 shmem_seg_name, sizeof(SHARED_MEM), gstate.boinc_project_gid,
969                 (void**)&app_client_shm.shm
970             );
971 
972             if (retval) {
973                 needs_shmem = true;
974                 destroy_shmem(shmem_seg_name);
975                 return retval;
976             }
977         }
978         needs_shmem = false;
979     }
980     app_client_shm.reset_msgs();
981 
982 #if (defined (__APPLE__) && (defined(__i386__) || defined(__x86_64__)))
983     // PowerPC apps emulated on i386 Macs crash if running graphics
984     powerpc_emulated_on_i386 = ! is_native_i386_app(exec_path);
985 #endif
986     if (cc_config.run_apps_manually) {
987         pid = getpid();     // use the client's PID
988         set_task_state(PROCESS_EXECUTING, "start");
989         return 0;
990     }
991     pid = fork();
992     if (pid == -1) {
993         snprintf(buf, sizeof(buf), "fork() failed: %s", strerror(errno));
994         retval = ERR_FORK;
995         goto error;
996     }
997     if (pid == 0) {
998         // from here on we're running in a new process.
999         // If an error happens,
1000         // exit nonzero so that the client knows there was a problem.
1001 
1002         // don't pass stdout to the app
1003         //
1004         int fd = open("/dev/null", O_RDWR);
1005         dup2(fd, STDOUT_FILENO);
1006         close(fd);
1007 
1008         // prepend to library path:
1009         // - the project dir (../../projects/X)
1010         // - the slot dir (.)
1011         // - the BOINC dir (../..)
1012         // (Mac) /usr/local/cuda/lib/
1013         // We use relative paths in case higher-level dirs
1014         // are not readable to the account under which app runs
1015         //
1016         char libpath[8192];
1017         char newlibs[256];
1018         snprintf(newlibs, sizeof(newlibs), "../../%s:.:../..", wup->project->project_dir());
1019 #ifdef __APPLE__
1020         safe_strcat(newlibs, ":/usr/local/cuda/lib/");
1021 #endif
1022         char* p = getenv("LD_LIBRARY_PATH");
1023         if (p) {
1024             snprintf(libpath, sizeof(libpath), "%s:%s", newlibs, p);
1025         } else {
1026             safe_strcpy(libpath, newlibs);
1027         }
1028         setenv("LD_LIBRARY_PATH", libpath, 1);
1029 
1030         // On the Mac, do the same for DYLD_LIBRARY_PATH
1031         //
1032 #ifdef __APPLE__
1033         p = getenv("DYLD_LIBRARY_PATH");
1034         if (p) {
1035             snprintf(libpath, sizeof(libpath), "%s:%s", newlibs, p);
1036         } else {
1037             safe_strcpy(libpath, newlibs);
1038         }
1039         setenv("DYLD_LIBRARY_PATH", libpath, 1);
1040 #endif
1041 
1042         retval = chdir(slot_dir);
1043         if (retval) {
1044             perror("chdir");
1045             fflush(NULL);
1046             _exit(errno);
1047         }
1048 
1049 #if 0
1050         // set stack size limit to the max.
1051         // Some BOINC apps have reported problems with exceeding
1052         // small stack limits (e.g. 8 MB)
1053         // and it seems like the best thing to raise it as high as possible
1054         //
1055         struct rlimit rlim;
1056 #define MIN_STACK_LIMIT 64000000
1057         getrlimit(RLIMIT_STACK, &rlim);
1058         if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur <= MIN_STACK_LIMIT) {
1059             if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max > MIN_STACK_LIMIT) {
1060                 rlim.rlim_cur = MIN_STACK_LIMIT;
1061             } else {
1062                 rlim.rlim_cur = rlim.rlim_max;
1063             }
1064             setrlimit(RLIMIT_STACK, &rlim);
1065         }
1066 #endif
1067 
1068         // hook up stderr to a specially-named file
1069         //
1070         (void) freopen(STDERR_FILE, "a", stderr);
1071 
1072         // lower our priority if needed
1073         //
1074         if (!cc_config.no_priority_change) {
1075 #if HAVE_SETPRIORITY
1076             int priority = get_priority(high_priority);
1077             if (setpriority(PRIO_PROCESS, 0, priority)) {
1078                 perror("setpriority");
1079             }
1080 #endif
1081 #ifdef ANDROID
1082             // Android has its own notion of background scheduling
1083             if (!high_priority) {
1084                 FILE* f = fopen("/dev/cpuctl/apps/bg_non_interactive/tasks", "w");
1085                 if (!f) {
1086                     msg_printf(NULL, MSG_INFO, "Can't open /dev/cpuctl/apps/bg_non_interactive/tasks");
1087                 } else {
1088                     fprintf(f, "%d", getpid());
1089                     fclose(f);
1090                 }
1091             }
1092 #endif
1093 #if HAVE_SCHED_SETSCHEDULER && defined(SCHED_BATCH) && defined (__linux__)
1094             if (!high_priority) {
1095                 struct sched_param sp;
1096                 sp.sched_priority = 0;
1097                 if (sched_setscheduler(0, SCHED_BATCH, &sp)) {
1098                     perror("sched_setscheduler");
1099                 }
1100             }
1101 #endif
1102         }
1103 
1104         // Run the application program.
1105         // If using account-based sandboxing, use a helper app
1106         // to do this, to set the right user ID
1107         //
1108         if (test) {
1109             strcpy(buf, exec_path);
1110         } else {
1111             snprintf(buf, sizeof(buf), "../../%s", exec_path);
1112         }
1113         if (g_use_sandbox) {
1114             char switcher_path[MAXPATHLEN];
1115             snprintf(switcher_path, sizeof(switcher_path),
1116                 "../../%s/%s",
1117                 SWITCHER_DIR, SWITCHER_FILE_NAME
1118             );
1119             argv[0] = const_cast<char*>(SWITCHER_FILE_NAME);
1120             argv[1] = buf;
1121             argv[2] = exec_name;
1122             parse_command_line(cmdline, argv+3);
1123             if (log_flags.task_debug) {
1124                 debug_print_argv(argv);
1125             }
1126             // Files written by projects have user boinc_project
1127             // and group boinc_project,
1128             // so they must be world-readable so BOINC CLient can read them
1129             //
1130             umask(2);
1131             retval = execv(switcher_path, argv);
1132         } else {
1133             argv[0] = buf;
1134             parse_command_line(cmdline, argv+1);
1135             retval = execv(buf, argv);
1136         }
1137         fprintf(stderr,
1138             "Process creation (%s) failed: %s, errno=%d\n",
1139             buf, boincerror(retval), errno
1140         );
1141         perror("execv");
1142         fflush(NULL);
1143         _exit(errno);
1144     }
1145 
1146     // parent process (client) continues here
1147     //
1148     if (log_flags.task_debug) {
1149         msg_printf(wup->project, MSG_INFO,
1150             "[task] ACTIVE_TASK::start(): forked process: pid %d\n", pid
1151         );
1152     }
1153 
1154 #endif
1155     set_task_state(PROCESS_EXECUTING, "start");
1156     return 0;
1157 
1158     // go here on error; "buf" contains error message, "retval" is nonzero
1159     //
1160 error:
1161     if (test) {
1162         return retval;
1163     }
1164 
1165     // if something failed, it's possible that the executable was munged.
1166     // Verify it to trigger another download.
1167     //
1168     gstate.input_files_available(result, true);
1169     char err_msg[4096];
1170     sprintf(err_msg, "couldn't start app: %s", buf);
1171     gstate.report_result_error(*result, err_msg);
1172     if (log_flags.task_debug) {
1173         msg_printf(wup->project, MSG_INFO,
1174             "[task] couldn't start app: %s", buf
1175         );
1176     }
1177     set_task_state(PROCESS_COULDNT_START, "start");
1178     return retval;
1179 }
1180 
1181 // Resume the task if it was previously running; otherwise start it
1182 // Postcondition: "state" is set correctly
1183 //
1184 int ACTIVE_TASK::resume_or_start(bool first_time) {
1185     const char* str = "??";
1186     int retval;
1187 
1188     switch (task_state()) {
1189     case PROCESS_UNINITIALIZED:
1190         str = (first_time)?"Starting":"Restarting";
1191         retval = start();
1192         if ((retval == ERR_SHMGET) || (retval == ERR_SHMAT)) {
1193             return retval;
1194         }
1195         if (retval) {
1196             set_task_state(PROCESS_COULDNT_START, "resume_or_start1");
1197             return retval;
1198         }
1199         break;
1200     case PROCESS_SUSPENDED:
1201         retval = unsuspend();
1202         if (retval) {
1203             msg_printf(wup->project, MSG_INTERNAL_ERROR,
1204                 "Couldn't resume task %s", result->name
1205             );
1206             set_task_state(PROCESS_COULDNT_START, "resume_or_start2");
1207             return retval;
1208         }
1209         str = "Resuming";
1210         break;
1211     default:
1212         msg_printf(result->project, MSG_INTERNAL_ERROR,
1213             "Unexpected state %d for task %s", task_state(), result->name
1214         );
1215         return 0;
1216     }
1217     if (log_flags.task && first_time) {
1218         msg_printf(result->project, MSG_INFO,
1219             "Starting task %s", result->name
1220         );
1221     }
1222     if (log_flags.cpu_sched) {
1223         char buf[256];
1224         safe_strcpy(buf, "");
1225         if (strlen(app_version->plan_class)) {
1226             snprintf(buf, sizeof(buf), " (%s)", app_version->plan_class);
1227         }
1228         msg_printf(result->project, MSG_INFO,
1229             "[cpu_sched] %s task %s using %s version %d%s in slot %d",
1230             str,
1231             result->name,
1232             app_version->app->name,
1233             app_version->version_num,
1234             buf,
1235             slot
1236         );
1237     }
1238     return 0;
1239 }
1240 
1241 #if (defined (__APPLE__) && (defined(__i386__) || defined(__x86_64__)))
1242 
1243 union headeru {
1244     fat_header fat;
1245     mach_header mach;
1246 };
1247 
1248 // Read the mach-o headers to determine the architectures
1249 // supported by executable file.
1250 // Returns 1 if application can run natively on i386 / x86_64 Macs,
1251 // else returns 0.
1252 //
1253 int ACTIVE_TASK::is_native_i386_app(char* exec_path) {
1254     FILE *f;
1255     int retval = 0;
1256 
1257     headeru myHeader;
1258     fat_arch fatHeader;
1259 
1260     uint32_t n, i, len;
1261     uint32_t theMagic;
1262     integer_t theType;
1263 
1264     f = boinc_fopen(exec_path, "rb");
1265     if (!f) {
1266         return retval;          // Should never happen
1267     }
1268 
1269     myHeader.fat.magic = 0;
1270     myHeader.fat.nfat_arch = 0;
1271 
1272     fread(&myHeader, 1, sizeof(fat_header), f);
1273     theMagic = myHeader.mach.magic;
1274     switch (theMagic) {
1275     case MH_CIGAM:
1276     case MH_MAGIC:
1277     case MH_MAGIC_64:
1278     case MH_CIGAM_64:
1279        theType = myHeader.mach.cputype;
1280         if ((theMagic == MH_CIGAM) || (theMagic == MH_CIGAM_64)) {
1281             theType = OSSwapInt32(theType);
1282         }
1283         if ((theType == CPU_TYPE_I386) || (theType == CPU_TYPE_X86_64)) {
1284             retval = 1;        // Single-architecture i386or x86_64 file
1285         }
1286         break;
1287     case FAT_MAGIC:
1288     case FAT_CIGAM:
1289         n = myHeader.fat.nfat_arch;
1290         if (theMagic == FAT_CIGAM) {
1291             n = OSSwapInt32(myHeader.fat.nfat_arch);
1292         }
1293            // Multiple architecture (fat) file
1294         for (i=0; i<n; i++) {
1295             len = fread(&fatHeader, 1, sizeof(fat_arch), f);
1296             if (len < sizeof(fat_arch)) {
1297                 break;          // Should never happen
1298             }
1299             theType = fatHeader.cputype;
1300             if (theMagic == FAT_CIGAM) {
1301                 theType = OSSwapInt32(theType);
1302             }
1303             if ((theType == CPU_TYPE_I386) || (theType == CPU_TYPE_X86_64)) {
1304                 retval = 1;
1305                 break;
1306             }
1307         }
1308         break;
1309     default:
1310         break;
1311     }
1312 
1313     fclose (f);
1314     return retval;
1315 }
1316 #endif
1317 
1318 // The following runs "test_app" and sends it various messages.
1319 // Used for testing the runtime system.
1320 //
1321 void run_test_app() {
1322     WORKUNIT wu;
1323     PROJECT project;
1324     APP app;
1325     APP_VERSION av;
1326     ACTIVE_TASK at;
1327     ACTIVE_TASK_SET ats;
1328     RESULT result;
1329     int retval;
1330 
1331     char buf[256];
1332     getcwd(buf, sizeof(buf));   // so we can see where we're running
1333 
1334     gstate.run_test_app = true;
1335 
1336     wu.project = &project;
1337     wu.app = &app;
1338     wu.command_line = string("--critical_section");
1339 
1340     safe_strcpy(app.name, "test app");
1341     av.init();
1342     av.avg_ncpus = 1;
1343 
1344     safe_strcpy(result.name, "test result");
1345     result.avp = &av;
1346     result.wup = &wu;
1347     result.project = &project;
1348     result.app = &app;
1349 
1350     at.result = &result;
1351     at.wup = &wu;
1352     at.app_version = &av;
1353     at.max_elapsed_time = 1e6;
1354     at.max_disk_usage = 1e14;
1355     at.max_mem_usage = 1e14;
1356     safe_strcpy(at.slot_dir, ".");
1357 
1358 #if 1
1359     // test file copy
1360     //
1361     ASYNC_COPY* ac = new ASYNC_COPY;
1362     FILE_INFO fi;
1363     retval = ac->init(&at, &fi, "big_file", "./big_file_copy");
1364     if (retval) {
1365         exit(1);
1366     }
1367     while (1) {
1368         do_async_file_op();
1369         if (at.async_copy == NULL) {
1370             break;
1371         }
1372     }
1373     fprintf(stderr, "done\n");
1374     exit(0);
1375 #endif
1376     ats.active_tasks.push_back(&at);
1377 
1378     unlink("boinc_finish_called");
1379     unlink("boinc_lockfile");
1380     unlink("boinc_temporary_exit");
1381     unlink("stderr.txt");
1382     retval = at.start(true);
1383     if (retval) {
1384         fprintf(stderr, "start() failed: %s\n", boincerror(retval));
1385     }
1386     while (1) {
1387         gstate.now = dtime();
1388         at.preempt(REMOVE_NEVER);
1389         ats.poll();
1390         boinc_sleep(.1);
1391         at.unsuspend();
1392         ats.poll();
1393         boinc_sleep(.2);
1394         //at.request_reread_prefs();
1395     }
1396 }
1397