1 #ifndef RUN_COMMAND_H
2 #define RUN_COMMAND_H
3
4 #include "thread-utils.h"
5
6 #include "strvec.h"
7
8 /**
9 * The run-command API offers a versatile tool to run sub-processes with
10 * redirected input and output as well as with a modified environment
11 * and an alternate current directory.
12 *
13 * A similar API offers the capability to run a function asynchronously,
14 * which is primarily used to capture the output that the function
15 * produces in the caller in order to process it.
16 */
17
18
19 /**
20 * This describes the arguments, redirections, and environment of a
21 * command to run in a sub-process.
22 *
23 * The caller:
24 *
25 * 1. allocates and clears (using child_process_init() or
26 * CHILD_PROCESS_INIT) a struct child_process variable;
27 * 2. initializes the members;
28 * 3. calls start_command();
29 * 4. processes the data;
30 * 5. closes file descriptors (if necessary; see below);
31 * 6. calls finish_command().
32 *
33 * Special forms of redirection are available by setting these members
34 * to 1:
35 *
36 * .no_stdin, .no_stdout, .no_stderr: The respective channel is
37 * redirected to /dev/null.
38 *
39 * .stdout_to_stderr: stdout of the child is redirected to its
40 * stderr. This happens after stderr is itself redirected.
41 * So stdout will follow stderr to wherever it is
42 * redirected.
43 */
44 struct child_process {
45
46 /**
47 * The .argv member is set up as an array of string pointers (NULL
48 * terminated), of which .argv[0] is the program name to run (usually
49 * without a path). If the command to run is a git command, set argv[0] to
50 * the command name without the 'git-' prefix and set .git_cmd = 1.
51 *
52 * Note that the ownership of the memory pointed to by .argv stays with the
53 * caller, but it should survive until `finish_command` completes. If the
54 * .argv member is NULL, `start_command` will point it at the .args
55 * `strvec` (so you may use one or the other, but you must use exactly
56 * one). The memory in .args will be cleaned up automatically during
57 * `finish_command` (or during `start_command` when it is unsuccessful).
58 *
59 */
60 const char **argv;
61
62 struct strvec args;
63 struct strvec env_array;
64 pid_t pid;
65
66 int trace2_child_id;
67 uint64_t trace2_child_us_start;
68 const char *trace2_child_class;
69 const char *trace2_hook_name;
70
71 /*
72 * Using .in, .out, .err:
73 * - Specify 0 for no redirections. No new file descriptor is allocated.
74 * (child inherits stdin, stdout, stderr from parent).
75 * - Specify -1 to have a pipe allocated as follows:
76 * .in: returns the writable pipe end; parent writes to it,
77 * the readable pipe end becomes child's stdin
78 * .out, .err: returns the readable pipe end; parent reads from
79 * it, the writable pipe end becomes child's stdout/stderr
80 * The caller of start_command() must close the returned FDs
81 * after it has completed reading from/writing to it!
82 * - Specify > 0 to set a channel to a particular FD as follows:
83 * .in: a readable FD, becomes child's stdin
84 * .out: a writable FD, becomes child's stdout/stderr
85 * .err: a writable FD, becomes child's stderr
86 * The specified FD is closed by start_command(), even in case
87 * of errors!
88 */
89 int in;
90 int out;
91 int err;
92
93 /**
94 * To specify a new initial working directory for the sub-process,
95 * specify it in the .dir member.
96 */
97 const char *dir;
98
99 /**
100 * To modify the environment of the sub-process, specify an array of
101 * string pointers (NULL terminated) in .env:
102 *
103 * - If the string is of the form "VAR=value", i.e. it contains '='
104 * the variable is added to the child process's environment.
105 *
106 * - If the string does not contain '=', it names an environment
107 * variable that will be removed from the child process's environment.
108 *
109 * If the .env member is NULL, `start_command` will point it at the
110 * .env_array `strvec` (so you may use one or the other, but not both).
111 * The memory in .env_array will be cleaned up automatically during
112 * `finish_command` (or during `start_command` when it is unsuccessful).
113 */
114 const char *const *env;
115
116 unsigned no_stdin:1;
117 unsigned no_stdout:1;
118 unsigned no_stderr:1;
119 unsigned git_cmd:1; /* if this is to be git sub-command */
120
121 /**
122 * If the program cannot be found, the functions return -1 and set
123 * errno to ENOENT. Normally, an error message is printed, but if
124 * .silent_exec_failure is set to 1, no message is printed for this
125 * special error condition.
126 */
127 unsigned silent_exec_failure:1;
128
129 /**
130 * Run the command from argv[0] using a shell (but note that we may
131 * still optimize out the shell call if the command contains no
132 * metacharacters). Note that further arguments to the command in
133 * argv[1], etc, do not need to be shell-quoted.
134 */
135 unsigned use_shell:1;
136
137 /**
138 * Release any open file handles to the object store before running
139 * the command; This is necessary e.g. when the spawned process may
140 * want to repack because that would delete `.pack` files (and on
141 * Windows, you cannot delete files that are still in use).
142 */
143 unsigned close_object_store:1;
144
145 unsigned stdout_to_stderr:1;
146 unsigned clean_on_exit:1;
147 unsigned wait_after_clean:1;
148 void (*clean_on_exit_handler)(struct child_process *process);
149 void *clean_on_exit_handler_cbdata;
150 };
151
152 #define CHILD_PROCESS_INIT { \
153 .args = STRVEC_INIT, \
154 .env_array = STRVEC_INIT, \
155 }
156
157 /**
158 * The functions: child_process_init, start_command, finish_command,
159 * run_command, run_command_v_opt, run_command_v_opt_cd_env, child_process_clear
160 * do the following:
161 *
162 * - If a system call failed, errno is set and -1 is returned. A diagnostic
163 * is printed.
164 *
165 * - If the program was not found, then -1 is returned and errno is set to
166 * ENOENT; a diagnostic is printed only if .silent_exec_failure is 0.
167 *
168 * - Otherwise, the program is run. If it terminates regularly, its exit
169 * code is returned. No diagnostic is printed, even if the exit code is
170 * non-zero.
171 *
172 * - If the program terminated due to a signal, then the return value is the
173 * signal number + 128, ie. the same value that a POSIX shell's $? would
174 * report. A diagnostic is printed.
175 *
176 */
177
178 /**
179 * Initialize a struct child_process variable.
180 */
181 void child_process_init(struct child_process *);
182
183 /**
184 * Release the memory associated with the struct child_process.
185 * Most users of the run-command API don't need to call this
186 * function explicitly because `start_command` invokes it on
187 * failure and `finish_command` calls it automatically already.
188 */
189 void child_process_clear(struct child_process *);
190
191 int is_executable(const char *name);
192
193 /**
194 * Check if the command exists on $PATH. This emulates the path search that
195 * execvp would perform, without actually executing the command so it
196 * can be used before fork() to prepare to run a command using
197 * execve() or after execvp() to diagnose why it failed.
198 *
199 * The caller should ensure that command contains no directory separators.
200 *
201 * Returns 1 if it is found in $PATH or 0 if the command could not be found.
202 */
203 int exists_in_PATH(const char *command);
204
205 /**
206 * Start a sub-process. Takes a pointer to a `struct child_process`
207 * that specifies the details and returns pipe FDs (if requested).
208 * See below for details.
209 */
210 int start_command(struct child_process *);
211
212 /**
213 * Wait for the completion of a sub-process that was started with
214 * start_command().
215 */
216 int finish_command(struct child_process *);
217
218 int finish_command_in_signal(struct child_process *);
219
220 /**
221 * A convenience function that encapsulates a sequence of
222 * start_command() followed by finish_command(). Takes a pointer
223 * to a `struct child_process` that specifies the details.
224 */
225 int run_command(struct child_process *);
226
227 /**
228 * Run a hook.
229 * The first argument is a pathname to an index file, or NULL
230 * if the hook uses the default index file or no index is needed.
231 * The second argument is the name of the hook.
232 * The further arguments correspond to the hook arguments.
233 * The last argument has to be NULL to terminate the arguments list.
234 * If the hook does not exist or is not executable, the return
235 * value will be zero.
236 * If it is executable, the hook will be executed and the exit
237 * status of the hook is returned.
238 * On execution, .stdout_to_stderr and .no_stdin will be set.
239 */
240 LAST_ARG_MUST_BE_NULL
241 int run_hook_le(const char *const *env, const char *name, ...);
242 int run_hook_ve(const char *const *env, const char *name, va_list args);
243
244 /*
245 * Trigger an auto-gc
246 */
247 int run_auto_maintenance(int quiet);
248
249 #define RUN_COMMAND_NO_STDIN (1<<0)
250 #define RUN_GIT_CMD (1<<1)
251 #define RUN_COMMAND_STDOUT_TO_STDERR (1<<2)
252 #define RUN_SILENT_EXEC_FAILURE (1<<3)
253 #define RUN_USING_SHELL (1<<4)
254 #define RUN_CLEAN_ON_EXIT (1<<5)
255 #define RUN_WAIT_AFTER_CLEAN (1<<6)
256 #define RUN_CLOSE_OBJECT_STORE (1<<7)
257
258 /**
259 * Convenience functions that encapsulate a sequence of
260 * start_command() followed by finish_command(). The argument argv
261 * specifies the program and its arguments. The argument opt is zero
262 * or more of the flags `RUN_COMMAND_NO_STDIN`, `RUN_GIT_CMD`,
263 * `RUN_COMMAND_STDOUT_TO_STDERR`, or `RUN_SILENT_EXEC_FAILURE`
264 * that correspond to the members .no_stdin, .git_cmd,
265 * .stdout_to_stderr, .silent_exec_failure of `struct child_process`.
266 * The argument dir corresponds the member .dir. The argument env
267 * corresponds to the member .env.
268 */
269 int run_command_v_opt(const char **argv, int opt);
270 int run_command_v_opt_tr2(const char **argv, int opt, const char *tr2_class);
271 /*
272 * env (the environment) is to be formatted like environ: "VAR=VALUE".
273 * To unset an environment variable use just "VAR".
274 */
275 int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env);
276 int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir,
277 const char *const *env, const char *tr2_class);
278
279 /**
280 * Execute the given command, sending "in" to its stdin, and capturing its
281 * stdout and stderr in the "out" and "err" strbufs. Any of the three may
282 * be NULL to skip processing.
283 *
284 * Returns -1 if starting the command fails or reading fails, and otherwise
285 * returns the exit code of the command. Any output collected in the
286 * buffers is kept even if the command returns a non-zero exit. The hint fields
287 * gives starting sizes for the strbuf allocations.
288 *
289 * The fields of "cmd" should be set up as they would for a normal run_command
290 * invocation. But note that there is no need to set the in, out, or err
291 * fields; pipe_command handles that automatically.
292 */
293 int pipe_command(struct child_process *cmd,
294 const char *in, size_t in_len,
295 struct strbuf *out, size_t out_hint,
296 struct strbuf *err, size_t err_hint);
297
298 /**
299 * Convenience wrapper around pipe_command for the common case
300 * of capturing only stdout.
301 */
capture_command(struct child_process * cmd,struct strbuf * out,size_t hint)302 static inline int capture_command(struct child_process *cmd,
303 struct strbuf *out,
304 size_t hint)
305 {
306 return pipe_command(cmd, NULL, 0, out, hint, NULL, 0);
307 }
308
309 /*
310 * The purpose of the following functions is to feed a pipe by running
311 * a function asynchronously and providing output that the caller reads.
312 *
313 * It is expected that no synchronization and mutual exclusion between
314 * the caller and the feed function is necessary so that the function
315 * can run in a thread without interfering with the caller.
316 *
317 * The caller:
318 *
319 * 1. allocates and clears (memset(&asy, 0, sizeof(asy));) a
320 * struct async variable;
321 * 2. initializes .proc and .data;
322 * 3. calls start_async();
323 * 4. processes communicates with proc through .in and .out;
324 * 5. closes .in and .out;
325 * 6. calls finish_async().
326 *
327 * There are serious restrictions on what the asynchronous function can do
328 * because this facility is implemented by a thread in the same address
329 * space on most platforms (when pthreads is available), but by a pipe to
330 * a forked process otherwise:
331 *
332 * - It cannot change the program's state (global variables, environment,
333 * etc.) in a way that the caller notices; in other words, .in and .out
334 * are the only communication channels to the caller.
335 *
336 * - It must not change the program's state that the caller of the
337 * facility also uses.
338 *
339 */
340 struct async {
341
342 /**
343 * The function pointer in .proc has the following signature:
344 *
345 * int proc(int in, int out, void *data);
346 *
347 * - in, out specifies a set of file descriptors to which the function
348 * must read/write the data that it needs/produces. The function
349 * *must* close these descriptors before it returns. A descriptor
350 * may be -1 if the caller did not configure a descriptor for that
351 * direction.
352 *
353 * - data is the value that the caller has specified in the .data member
354 * of struct async.
355 *
356 * - The return value of the function is 0 on success and non-zero
357 * on failure. If the function indicates failure, finish_async() will
358 * report failure as well.
359 *
360 */
361 int (*proc)(int in, int out, void *data);
362
363 void *data;
364
365 /**
366 * The members .in, .out are used to provide a set of fd's for
367 * communication between the caller and the callee as follows:
368 *
369 * - Specify 0 to have no file descriptor passed. The callee will
370 * receive -1 in the corresponding argument.
371 *
372 * - Specify < 0 to have a pipe allocated; start_async() replaces
373 * with the pipe FD in the following way:
374 *
375 * .in: Returns the writable pipe end into which the caller
376 * writes; the readable end of the pipe becomes the function's
377 * in argument.
378 *
379 * .out: Returns the readable pipe end from which the caller
380 * reads; the writable end of the pipe becomes the function's
381 * out argument.
382 *
383 * The caller of start_async() must close the returned FDs after it
384 * has completed reading from/writing from them.
385 *
386 * - Specify a file descriptor > 0 to be used by the function:
387 *
388 * .in: The FD must be readable; it becomes the function's in.
389 * .out: The FD must be writable; it becomes the function's out.
390 *
391 * The specified FD is closed by start_async(), even if it fails to
392 * run the function.
393 */
394 int in; /* caller writes here and closes it */
395 int out; /* caller reads from here and closes it */
396 #ifdef NO_PTHREADS
397 pid_t pid;
398 #else
399 pthread_t tid;
400 int proc_in;
401 int proc_out;
402 #endif
403 int isolate_sigpipe;
404 };
405
406 /**
407 * Run a function asynchronously. Takes a pointer to a `struct
408 * async` that specifies the details and returns a set of pipe FDs
409 * for communication with the function. See below for details.
410 */
411 int start_async(struct async *async);
412
413 /**
414 * Wait for the completion of an asynchronous function that was
415 * started with start_async().
416 */
417 int finish_async(struct async *async);
418
419 int in_async(void);
420 int async_with_fork(void);
421 void check_pipe(int err);
422
423 /**
424 * This callback should initialize the child process and preload the
425 * error channel if desired. The preloading of is useful if you want to
426 * have a message printed directly before the output of the child process.
427 * pp_cb is the callback cookie as passed to run_processes_parallel.
428 * You can store a child process specific callback cookie in pp_task_cb.
429 *
430 * Even after returning 0 to indicate that there are no more processes,
431 * this function will be called again until there are no more running
432 * child processes.
433 *
434 * Return 1 if the next child is ready to run.
435 * Return 0 if there are currently no more tasks to be processed.
436 * To send a signal to other child processes for abortion,
437 * return the negative signal number.
438 */
439 typedef int (*get_next_task_fn)(struct child_process *cp,
440 struct strbuf *out,
441 void *pp_cb,
442 void **pp_task_cb);
443
444 /**
445 * This callback is called whenever there are problems starting
446 * a new process.
447 *
448 * You must not write to stdout or stderr in this function. Add your
449 * message to the strbuf out instead, which will be printed without
450 * messing up the output of the other parallel processes.
451 *
452 * pp_cb is the callback cookie as passed into run_processes_parallel,
453 * pp_task_cb is the callback cookie as passed into get_next_task_fn.
454 *
455 * Return 0 to continue the parallel processing. To abort return non zero.
456 * To send a signal to other child processes for abortion, return
457 * the negative signal number.
458 */
459 typedef int (*start_failure_fn)(struct strbuf *out,
460 void *pp_cb,
461 void *pp_task_cb);
462
463 /**
464 * This callback is called on every child process that finished processing.
465 *
466 * You must not write to stdout or stderr in this function. Add your
467 * message to the strbuf out instead, which will be printed without
468 * messing up the output of the other parallel processes.
469 *
470 * pp_cb is the callback cookie as passed into run_processes_parallel,
471 * pp_task_cb is the callback cookie as passed into get_next_task_fn.
472 *
473 * Return 0 to continue the parallel processing. To abort return non zero.
474 * To send a signal to other child processes for abortion, return
475 * the negative signal number.
476 */
477 typedef int (*task_finished_fn)(int result,
478 struct strbuf *out,
479 void *pp_cb,
480 void *pp_task_cb);
481
482 /**
483 * Runs up to n processes at the same time. Whenever a process can be
484 * started, the callback get_next_task_fn is called to obtain the data
485 * required to start another child process.
486 *
487 * The children started via this function run in parallel. Their output
488 * (both stdout and stderr) is routed to stderr in a manner that output
489 * from different tasks does not interleave.
490 *
491 * start_failure_fn and task_finished_fn can be NULL to omit any
492 * special handling.
493 */
494 int run_processes_parallel(int n,
495 get_next_task_fn,
496 start_failure_fn,
497 task_finished_fn,
498 void *pp_cb);
499 int run_processes_parallel_tr2(int n, get_next_task_fn, start_failure_fn,
500 task_finished_fn, void *pp_cb,
501 const char *tr2_category, const char *tr2_label);
502
503 /**
504 * Convenience function which prepares env_array for a command to be run in a
505 * new repo. This adds all GIT_* environment variables to env_array with the
506 * exception of GIT_CONFIG_PARAMETERS and GIT_CONFIG_COUNT (which cause the
507 * corresponding environment variables to be unset in the subprocess) and adds
508 * an environment variable pointing to new_git_dir. See local_repo_env in
509 * cache.h for more information.
510 */
511 void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir);
512
513 /**
514 * Possible return values for start_bg_command().
515 */
516 enum start_bg_result {
517 /* child process is "ready" */
518 SBGR_READY = 0,
519
520 /* child process could not be started */
521 SBGR_ERROR,
522
523 /* callback error when testing for "ready" */
524 SBGR_CB_ERROR,
525
526 /* timeout expired waiting for child to become "ready" */
527 SBGR_TIMEOUT,
528
529 /* child process exited or was signalled before becomming "ready" */
530 SBGR_DIED,
531 };
532
533 /**
534 * Callback used by start_bg_command() to ask whether the
535 * child process is ready or needs more time to become "ready".
536 *
537 * The callback will receive the cmd and cb_data arguments given to
538 * start_bg_command().
539 *
540 * Returns 1 is child needs more time (subject to the requested timeout).
541 * Returns 0 if child is "ready".
542 * Returns -1 on any error and cause start_bg_command() to also error out.
543 */
544 typedef int(start_bg_wait_cb)(const struct child_process *cmd, void *cb_data);
545
546 /**
547 * Start a command in the background. Wait long enough for the child
548 * to become "ready" (as defined by the provided callback). Capture
549 * immediate errors (like failure to start) and any immediate exit
550 * status (such as a shutdown/signal before the child became "ready")
551 * and return this like start_command().
552 *
553 * We run a custom wait loop using the provided callback to wait for
554 * the child to start and become "ready". This is limited by the given
555 * timeout value.
556 *
557 * If the child does successfully start and become "ready", we orphan
558 * it into the background.
559 *
560 * The caller must not call finish_command().
561 *
562 * The opaque cb_data argument will be forwarded to the callback for
563 * any instance data that it might require. This may be NULL.
564 */
565 enum start_bg_result start_bg_command(struct child_process *cmd,
566 start_bg_wait_cb *wait_cb,
567 void *cb_data,
568 unsigned int timeout_sec);
569
570 #endif
571