1 // This is an open source non-commercial project. Dear PVS-Studio, please check
2 // it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
3
4 #include <assert.h>
5 #include <stdbool.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <uv.h>
9
10 #include "nvim/ascii.h"
11 #include "nvim/charset.h"
12 #include "nvim/event/libuv_process.h"
13 #include "nvim/event/loop.h"
14 #include "nvim/event/rstream.h"
15 #include "nvim/ex_cmds.h"
16 #include "nvim/fileio.h"
17 #include "nvim/lib/kvec.h"
18 #include "nvim/log.h"
19 #include "nvim/main.h"
20 #include "nvim/memline.h"
21 #include "nvim/memory.h"
22 #include "nvim/message.h"
23 #include "nvim/misc1.h"
24 #include "nvim/option_defs.h"
25 #include "nvim/os/shell.h"
26 #include "nvim/os/signal.h"
27 #include "nvim/path.h"
28 #include "nvim/screen.h"
29 #include "nvim/strings.h"
30 #include "nvim/types.h"
31 #include "nvim/ui.h"
32 #include "nvim/vim.h"
33
34 #define DYNAMIC_BUFFER_INIT { NULL, 0, 0 }
35 #define NS_1_SECOND 1000000000U // 1 second, in nanoseconds
36 #define OUT_DATA_THRESHOLD 1024 * 10U // 10KB, "a few screenfuls" of data.
37
38 #define SHELL_SPECIAL (char_u *)"\t \"&'$;<>()\\|"
39
40 typedef struct {
41 char *data;
42 size_t cap, len;
43 } DynamicBuffer;
44
45 #ifdef INCLUDE_GENERATED_DECLARATIONS
46 # include "os/shell.c.generated.h"
47 #endif
48
save_patterns(int num_pat,char_u ** pat,int * num_file,char_u *** file)49 static void save_patterns(int num_pat, char_u **pat, int *num_file, char_u ***file)
50 {
51 *file = xmalloc((size_t)num_pat * sizeof(char_u *));
52 for (int i = 0; i < num_pat; i++) {
53 char_u *s = vim_strsave(pat[i]);
54 // Be compatible with expand_filename(): halve the number of
55 // backslashes.
56 backslash_halve(s);
57 (*file)[i] = s;
58 }
59 *num_file = num_pat;
60 }
61
have_wildcard(int num,char_u ** file)62 static bool have_wildcard(int num, char_u **file)
63 {
64 for (int i = 0; i < num; i++) {
65 if (path_has_wildcard(file[i])) {
66 return true;
67 }
68 }
69 return false;
70 }
71
have_dollars(int num,char_u ** file)72 static bool have_dollars(int num, char_u **file)
73 {
74 for (int i = 0; i < num; i++) {
75 if (vim_strchr(file[i], '$') != NULL) {
76 return true;
77 }
78 }
79 return false;
80 }
81
82 /// Performs wildcard pattern matching using the shell.
83 ///
84 /// @param num_pat is the number of input patterns.
85 /// @param pat is an array of pointers to input patterns.
86 /// @param[out] num_file is pointer to number of matched file names.
87 /// Set to the number of pointers in *file.
88 /// @param[out] file is pointer to array of pointers to matched file names.
89 /// Memory pointed to by the initial value of *file will
90 /// not be freed.
91 /// Set to NULL if FAIL is returned. Otherwise points to
92 /// allocated memory.
93 /// @param flags is a combination of EW_* flags used in
94 /// expand_wildcards().
95 /// If matching fails but EW_NOTFOUND is set in flags or
96 /// there are no wildcards, the patterns from pat are
97 /// copied into *file.
98 ///
99 /// @returns OK for success or FAIL for error.
os_expand_wildcards(int num_pat,char_u ** pat,int * num_file,char_u *** file,int flags)100 int os_expand_wildcards(int num_pat, char_u **pat, int *num_file, char_u ***file, int flags)
101 FUNC_ATTR_NONNULL_ARG(3)
102 FUNC_ATTR_NONNULL_ARG(4)
103 {
104 int i;
105 size_t len;
106 char_u *p;
107 bool dir;
108 char_u *extra_shell_arg = NULL;
109 ShellOpts shellopts = kShellOptExpand | kShellOptSilent;
110 int j;
111 char_u *tempname;
112 char_u *command;
113 FILE *fd;
114 char_u *buffer;
115 #define STYLE_ECHO 0 // use "echo", the default
116 #define STYLE_GLOB 1 // use "glob", for csh
117 #define STYLE_VIMGLOB 2 // use "vimglob", for Posix sh
118 #define STYLE_PRINT 3 // use "print -N", for zsh
119 #define STYLE_BT 4 // `cmd` expansion, execute the pattern directly
120 int shell_style = STYLE_ECHO;
121 int check_spaces;
122 static bool did_find_nul = false;
123 bool ampersand = false;
124 // vimglob() function to define for Posix shell
125 static char *sh_vimglob_func =
126 "vimglob() { while [ $# -ge 1 ]; do echo \"$1\"; shift; done }; vimglob >";
127
128 bool is_fish_shell =
129 #if defined(UNIX)
130 STRNCMP(invocation_path_tail(p_sh, NULL), "fish", 4) == 0;
131 #else
132 false;
133 #endif
134
135 *num_file = 0; // default: no files found
136 *file = NULL;
137
138 // If there are no wildcards, just copy the names to allocated memory.
139 // Saves a lot of time, because we don't have to start a new shell.
140 if (!have_wildcard(num_pat, pat)) {
141 save_patterns(num_pat, pat, num_file, file);
142 return OK;
143 }
144
145 // Don't allow any shell command in the sandbox.
146 if (sandbox != 0 && check_secure()) {
147 return FAIL;
148 }
149
150 // Don't allow the use of backticks in secure.
151 if (secure) {
152 for (i = 0; i < num_pat; i++) {
153 if (vim_strchr(pat[i], '`') != NULL
154 && (check_secure())) {
155 return FAIL;
156 }
157 }
158 }
159
160 // get a name for the temp file
161 if ((tempname = vim_tempname()) == NULL) {
162 emsg(_(e_notmp));
163 return FAIL;
164 }
165
166 // Let the shell expand the patterns and write the result into the temp
167 // file.
168 // STYLE_BT: NL separated
169 // If expanding `cmd` execute it directly.
170 // STYLE_GLOB: NUL separated
171 // If we use *csh, "glob" will work better than "echo".
172 // STYLE_PRINT: NL or NUL separated
173 // If we use *zsh, "print -N" will work better than "glob".
174 // STYLE_VIMGLOB: NL separated
175 // If we use *sh*, we define "vimglob()".
176 // STYLE_ECHO: space separated.
177 // A shell we don't know, stay safe and use "echo".
178 if (num_pat == 1 && *pat[0] == '`'
179 && (len = STRLEN(pat[0])) > 2
180 && *(pat[0] + len - 1) == '`') {
181 shell_style = STYLE_BT;
182 } else if ((len = STRLEN(p_sh)) >= 3) {
183 if (STRCMP(p_sh + len - 3, "csh") == 0) {
184 shell_style = STYLE_GLOB;
185 } else if (STRCMP(p_sh + len - 3, "zsh") == 0) {
186 shell_style = STYLE_PRINT;
187 }
188 }
189 if (shell_style == STYLE_ECHO
190 && strstr((char *)path_tail(p_sh), "sh") != NULL) {
191 shell_style = STYLE_VIMGLOB;
192 }
193
194 // Compute the length of the command. We need 2 extra bytes: for the
195 // optional '&' and for the NUL.
196 // Worst case: "unset nonomatch; print -N >" plus two is 29
197 len = STRLEN(tempname) + 29;
198 if (shell_style == STYLE_VIMGLOB) {
199 len += STRLEN(sh_vimglob_func);
200 }
201
202 for (i = 0; i < num_pat; i++) {
203 // Count the length of the patterns in the same way as they are put in
204 // "command" below.
205 len++; // add space
206 for (j = 0; pat[i][j] != NUL; j++) {
207 if (vim_strchr(SHELL_SPECIAL, pat[i][j]) != NULL) {
208 len++; // may add a backslash
209 }
210 len++;
211 }
212 }
213
214 if (is_fish_shell) {
215 len += sizeof("egin;" " end") - 1;
216 }
217
218 command = xmalloc(len);
219
220 // Build the shell command:
221 // - Set $nonomatch depending on EW_NOTFOUND (hopefully the shell
222 // recognizes this).
223 // - Add the shell command to print the expanded names.
224 // - Add the temp file name.
225 // - Add the file name patterns.
226 if (shell_style == STYLE_BT) {
227 // change `command; command& ` to (command; command )
228 if (is_fish_shell) {
229 STRCPY(command, "begin; ");
230 } else {
231 STRCPY(command, "(");
232 }
233 STRCAT(command, pat[0] + 1); // exclude first backtick
234 p = command + STRLEN(command) - 1;
235 if (is_fish_shell) {
236 *p-- = ';';
237 STRCAT(command, " end");
238 } else {
239 *p-- = ')'; // remove last backtick
240 }
241 while (p > command && ascii_iswhite(*p)) {
242 p--;
243 }
244 if (*p == '&') { // remove trailing '&'
245 ampersand = true;
246 *p = ' ';
247 }
248 STRCAT(command, ">");
249 } else {
250 if (flags & EW_NOTFOUND) {
251 STRCPY(command, "set nonomatch; ");
252 } else {
253 STRCPY(command, "unset nonomatch; ");
254 }
255 if (shell_style == STYLE_GLOB) {
256 STRCAT(command, "glob >");
257 } else if (shell_style == STYLE_PRINT) {
258 STRCAT(command, "print -N >");
259 } else if (shell_style == STYLE_VIMGLOB) {
260 STRCAT(command, sh_vimglob_func);
261 } else {
262 STRCAT(command, "echo >");
263 }
264 }
265
266 STRCAT(command, tempname);
267
268 if (shell_style != STYLE_BT) {
269 for (i = 0; i < num_pat; i++) {
270 // Put a backslash before special
271 // characters, except inside ``.
272 bool intick = false;
273
274 p = command + STRLEN(command);
275 *p++ = ' ';
276 for (j = 0; pat[i][j] != NUL; j++) {
277 if (pat[i][j] == '`') {
278 intick = !intick;
279 } else if (pat[i][j] == '\\' && pat[i][j + 1] != NUL) {
280 // Remove a backslash, take char literally. But keep
281 // backslash inside backticks, before a special character
282 // and before a backtick.
283 if (intick
284 || vim_strchr(SHELL_SPECIAL, pat[i][j + 1]) != NULL
285 || pat[i][j + 1] == '`') {
286 *p++ = '\\';
287 }
288 j++;
289 } else if (!intick
290 && ((flags & EW_KEEPDOLLAR) == 0 || pat[i][j] != '$')
291 && vim_strchr(SHELL_SPECIAL, pat[i][j]) != NULL) {
292 // Put a backslash before a special character, but not
293 // when inside ``. And not for $var when EW_KEEPDOLLAR is
294 // set.
295 *p++ = '\\';
296 }
297
298 // Copy one character.
299 *p++ = pat[i][j];
300 }
301 *p = NUL;
302 }
303 }
304
305 if (flags & EW_SILENT) {
306 shellopts |= kShellOptHideMess;
307 }
308
309 if (ampersand) {
310 STRCAT(command, "&"); // put the '&' after the redirection
311 }
312
313 // Using zsh -G: If a pattern has no matches, it is just deleted from
314 // the argument list, otherwise zsh gives an error message and doesn't
315 // expand any other pattern.
316 if (shell_style == STYLE_PRINT) {
317 extra_shell_arg = (char_u *)"-G"; // Use zsh NULL_GLOB option
318
319 // If we use -f then shell variables set in .cshrc won't get expanded.
320 // vi can do it, so we will too, but it is only necessary if there is a "$"
321 // in one of the patterns, otherwise we can still use the fast option.
322 } else if (shell_style == STYLE_GLOB && !have_dollars(num_pat, pat)) {
323 extra_shell_arg = (char_u *)"-f"; // Use csh fast option
324 }
325
326 // execute the shell command
327 i = call_shell(command, shellopts, extra_shell_arg);
328
329 // When running in the background, give it some time to create the temp
330 // file, but don't wait for it to finish.
331 if (ampersand) {
332 os_delay(10L, true);
333 }
334
335 xfree(command);
336
337 if (i) { // os_call_shell() failed
338 os_remove((char *)tempname);
339 xfree(tempname);
340 // With interactive completion, the error message is not printed.
341 if (!(flags & EW_SILENT)) {
342 msg_putchar('\n'); // clear bottom line quickly
343 cmdline_row = Rows - 1; // continue on last line
344 msg(_(e_wildexpand));
345 msg_start(); // don't overwrite this message
346 }
347
348 // If a `cmd` expansion failed, don't list `cmd` as a match, even when
349 // EW_NOTFOUND is given
350 if (shell_style == STYLE_BT) {
351 return FAIL;
352 }
353 goto notfound;
354 }
355
356 // read the names from the file into memory
357 fd = fopen((char *)tempname, READBIN);
358 if (fd == NULL) {
359 // Something went wrong, perhaps a file name with a special char.
360 if (!(flags & EW_SILENT)) {
361 msg(_(e_wildexpand));
362 msg_start(); // don't overwrite this message
363 }
364 xfree(tempname);
365 goto notfound;
366 }
367 int fseek_res = fseek(fd, 0L, SEEK_END);
368 if (fseek_res < 0) {
369 xfree(tempname);
370 fclose(fd);
371 return FAIL;
372 }
373 int64_t templen = ftell(fd); // get size of temp file
374 if (templen < 0) {
375 xfree(tempname);
376 fclose(fd);
377 return FAIL;
378 }
379 #if SIZEOF_LONG_LONG > SIZEOF_SIZE_T
380 assert(templen <= (long long)SIZE_MAX); // NOLINT(runtime/int)
381 #endif
382 len = (size_t)templen;
383 fseek(fd, 0L, SEEK_SET);
384 buffer = xmalloc(len + 1);
385 // fread() doesn't terminate buffer with NUL;
386 // appropriate termination (not always NUL) is done below.
387 size_t readlen = fread((char *)buffer, 1, len, fd);
388 fclose(fd);
389 os_remove((char *)tempname);
390 if (readlen != len) {
391 // unexpected read error
392 semsg(_(e_notread), tempname);
393 xfree(tempname);
394 xfree(buffer);
395 return FAIL;
396 }
397 xfree(tempname);
398
399 // file names are separated with Space
400 if (shell_style == STYLE_ECHO) {
401 buffer[len] = '\n'; // make sure the buffer ends in NL
402 p = buffer;
403 for (i = 0; *p != '\n'; i++) { // count number of entries
404 while (*p != ' ' && *p != '\n') {
405 p++;
406 }
407 p = skipwhite(p); // skip to next entry
408 }
409 // file names are separated with NL
410 } else if (shell_style == STYLE_BT || shell_style == STYLE_VIMGLOB) {
411 buffer[len] = NUL; // make sure the buffer ends in NUL
412 p = buffer;
413 for (i = 0; *p != NUL; i++) { // count number of entries
414 while (*p != '\n' && *p != NUL) {
415 p++;
416 }
417 if (*p != NUL) {
418 p++;
419 }
420 p = skipwhite(p); // skip leading white space
421 }
422 // file names are separated with NUL
423 } else {
424 // Some versions of zsh use spaces instead of NULs to separate
425 // results. Only do this when there is no NUL before the end of the
426 // buffer, otherwise we would never be able to use file names with
427 // embedded spaces when zsh does use NULs.
428 // When we found a NUL once, we know zsh is OK, set did_find_nul and
429 // don't check for spaces again.
430 check_spaces = false;
431 if (shell_style == STYLE_PRINT && !did_find_nul) {
432 // If there is a NUL, set did_find_nul, else set check_spaces
433 buffer[len] = NUL;
434 if (len && (int)STRLEN(buffer) < (int)len) {
435 did_find_nul = true;
436 } else {
437 check_spaces = true;
438 }
439 }
440
441 // Make sure the buffer ends with a NUL. For STYLE_PRINT there
442 // already is one, for STYLE_GLOB it needs to be added.
443 if (len && buffer[len - 1] == NUL) {
444 len--;
445 } else {
446 buffer[len] = NUL;
447 }
448 for (p = buffer; p < buffer + len; p++) {
449 if (*p == NUL || (*p == ' ' && check_spaces)) { // count entry
450 i++;
451 *p = NUL;
452 }
453 }
454 if (len) {
455 i++; // count last entry
456 }
457 }
458 assert(buffer[len] == NUL || buffer[len] == '\n');
459
460 if (i == 0) {
461 // Can happen when using /bin/sh and typing ":e $NO_SUCH_VAR^I".
462 // /bin/sh will happily expand it to nothing rather than returning an
463 // error; and hey, it's good to check anyway -- webb.
464 xfree(buffer);
465 goto notfound;
466 }
467 *num_file = i;
468 *file = xmalloc(sizeof(char_u *) * (size_t)i);
469
470 // Isolate the individual file names.
471 p = buffer;
472 for (i = 0; i < *num_file; i++) {
473 (*file)[i] = p;
474 // Space or NL separates
475 if (shell_style == STYLE_ECHO || shell_style == STYLE_BT
476 || shell_style == STYLE_VIMGLOB) {
477 while (!(shell_style == STYLE_ECHO && *p == ' ')
478 && *p != '\n' && *p != NUL) {
479 p++;
480 }
481 if (p == buffer + len) { // last entry
482 *p = NUL;
483 } else {
484 *p++ = NUL;
485 p = skipwhite(p); // skip to next entry
486 }
487 } else { // NUL separates
488 while (*p && p < buffer + len) { // skip entry
489 p++;
490 }
491 p++; // skip NUL
492 }
493 }
494
495 // Move the file names to allocated memory.
496 for (j = 0, i = 0; i < *num_file; i++) {
497 // Require the files to exist. Helps when using /bin/sh
498 if (!(flags & EW_NOTFOUND) && !os_path_exists((*file)[i])) {
499 continue;
500 }
501
502 // check if this entry should be included
503 dir = (os_isdir((*file)[i]));
504 if ((dir && !(flags & EW_DIR)) || (!dir && !(flags & EW_FILE))) {
505 continue;
506 }
507
508 // Skip files that are not executable if we check for that.
509 if (!dir && (flags & EW_EXEC)
510 && !os_can_exe((char *)(*file)[i], NULL, !(flags & EW_SHELLCMD))) {
511 continue;
512 }
513
514 p = xmalloc(STRLEN((*file)[i]) + 1 + dir);
515 STRCPY(p, (*file)[i]);
516 if (dir) {
517 add_pathsep((char *)p); // add '/' to a directory name
518 }
519 (*file)[j++] = p;
520 }
521 xfree(buffer);
522 *num_file = j;
523
524 if (*num_file == 0) { // rejected all entries
525 XFREE_CLEAR(*file);
526 goto notfound;
527 }
528
529 return OK;
530
531 notfound:
532 if (flags & EW_NOTFOUND) {
533 save_patterns(num_pat, pat, num_file, file);
534 return OK;
535 }
536 return FAIL;
537 }
538
539 /// Builds the argument vector for running the user-configured 'shell' (p_sh)
540 /// with an optional command prefixed by 'shellcmdflag' (p_shcf). E.g.:
541 ///
542 /// ["shell", "-extra_args", "-shellcmdflag", "command with spaces"]
543 ///
544 /// @param cmd Command string, or NULL to run an interactive shell.
545 /// @param extra_args Extra arguments to the shell, or NULL.
546 /// @return Newly allocated argument vector. Must be freed with shell_free_argv.
shell_build_argv(const char * cmd,const char * extra_args)547 char **shell_build_argv(const char *cmd, const char *extra_args)
548 FUNC_ATTR_NONNULL_RET
549 {
550 size_t argc = tokenize(p_sh, NULL) + (cmd ? tokenize(p_shcf, NULL) : 0);
551 char **rv = xmalloc((argc + 4) * sizeof(*rv));
552
553 // Split 'shell'
554 size_t i = tokenize(p_sh, rv);
555
556 if (extra_args) {
557 rv[i++] = xstrdup(extra_args); // Push a copy of `extra_args`
558 }
559
560 if (cmd) {
561 i += tokenize(p_shcf, rv + i); // Split 'shellcmdflag'
562 rv[i++] = shell_xescape_xquote(cmd); // Copy (and escape) `cmd`.
563 }
564
565 rv[i] = NULL;
566
567 assert(rv[0]);
568
569 return rv;
570 }
571
572 /// Releases the memory allocated by `shell_build_argv`.
573 ///
574 /// @param argv The argument vector.
shell_free_argv(char ** argv)575 void shell_free_argv(char **argv)
576 {
577 char **p = argv;
578 if (p == NULL) {
579 // Nothing was allocated, return
580 return;
581 }
582 while (*p != NULL) {
583 // Free each argument
584 xfree(*p);
585 p++;
586 }
587 xfree(argv);
588 }
589
590 /// Joins shell arguments from `argv` into a new string.
591 /// If the result is too long it is truncated with ellipsis ("...").
592 ///
593 /// @returns[allocated] `argv` joined to a string.
shell_argv_to_str(char ** const argv)594 char *shell_argv_to_str(char **const argv)
595 FUNC_ATTR_NONNULL_ALL
596 {
597 size_t n = 0;
598 char **p = argv;
599 char *rv = xcalloc(256, sizeof(*rv));
600 const size_t maxsize = (256 * sizeof(*rv));
601 if (*p == NULL) {
602 return rv;
603 }
604 while (*p != NULL) {
605 xstrlcat(rv, "'", maxsize);
606 xstrlcat(rv, *p, maxsize);
607 n = xstrlcat(rv, "' ", maxsize);
608 if (n >= maxsize) {
609 break;
610 }
611 p++;
612 }
613 if (n < maxsize) {
614 rv[n - 1] = '\0';
615 } else {
616 // Command too long, show ellipsis: "/bin/bash 'foo' 'bar'..."
617 rv[maxsize - 4] = '.';
618 rv[maxsize - 3] = '.';
619 rv[maxsize - 2] = '.';
620 rv[maxsize - 1] = '\0';
621 }
622 return rv;
623 }
624
625 /// Calls the user-configured 'shell' (p_sh) for running a command or wildcard
626 /// expansion.
627 ///
628 /// @param cmd The command to execute, or NULL to run an interactive shell.
629 /// @param opts Options that control how the shell will work.
630 /// @param extra_args Extra arguments to the shell, or NULL.
631 ///
632 /// @return shell command exit code
os_call_shell(char_u * cmd,ShellOpts opts,char_u * extra_args)633 int os_call_shell(char_u *cmd, ShellOpts opts, char_u *extra_args)
634 {
635 DynamicBuffer input = DYNAMIC_BUFFER_INIT;
636 char *output = NULL, **output_ptr = NULL;
637 int current_state = State;
638 bool forward_output = true;
639
640 // While the child is running, ignore terminating signals
641 signal_reject_deadly();
642
643 if (opts & (kShellOptHideMess | kShellOptExpand)) {
644 forward_output = false;
645 } else {
646 State = EXTERNCMD;
647
648 if (opts & kShellOptWrite) {
649 read_input(&input);
650 }
651
652 if (opts & kShellOptRead) {
653 output_ptr = &output;
654 forward_output = false;
655 } else if (opts & kShellOptDoOut) {
656 // Caller has already redirected output
657 forward_output = false;
658 }
659 }
660
661 size_t nread;
662 int exitcode = do_os_system(shell_build_argv((char *)cmd, (char *)extra_args),
663 input.data, input.len, output_ptr, &nread,
664 emsg_silent, forward_output);
665 xfree(input.data);
666
667 if (output) {
668 (void)write_output(output, nread, true);
669 xfree(output);
670 }
671
672 if (!emsg_silent && exitcode != 0 && !(opts & kShellOptSilent)) {
673 msg_puts(_("\nshell returned "));
674 msg_outnum(exitcode);
675 msg_putchar('\n');
676 }
677
678 State = current_state;
679 signal_accept_deadly();
680
681 return exitcode;
682 }
683
684 /// os_system - synchronously execute a command in the shell
685 ///
686 /// example:
687 /// char *output = NULL;
688 /// size_t nread = 0;
689 /// char *argv[] = {"ls", "-la", NULL};
690 /// int exitcode = os_sytem(argv, NULL, 0, &output, &nread);
691 ///
692 /// @param argv The commandline arguments to be passed to the shell. `argv`
693 /// will be consumed.
694 /// @param input The input to the shell (NULL for no input), passed to the
695 /// stdin of the resulting process.
696 /// @param len The length of the input buffer (not used if `input` == NULL)
697 /// @param[out] output Pointer to a location where the output will be
698 /// allocated and stored. Will point to NULL if the shell
699 /// command did not output anything. If NULL is passed,
700 /// the shell output will be ignored.
701 /// @param[out] nread the number of bytes in the returned buffer (if the
702 /// returned buffer is not NULL)
703 /// @return the return code of the process, -1 if the process couldn't be
704 /// started properly
os_system(char ** argv,const char * input,size_t len,char ** output,size_t * nread)705 int os_system(char **argv, const char *input, size_t len, char **output,
706 size_t *nread) FUNC_ATTR_NONNULL_ARG(1)
707 {
708 return do_os_system(argv, input, len, output, nread, true, false);
709 }
710
do_os_system(char ** argv,const char * input,size_t len,char ** output,size_t * nread,bool silent,bool forward_output)711 static int do_os_system(char **argv, const char *input, size_t len, char **output, size_t *nread,
712 bool silent, bool forward_output)
713 {
714 out_data_decide_throttle(0); // Initialize throttle decider.
715 out_data_ring(NULL, 0); // Initialize output ring-buffer.
716 bool has_input = (input != NULL && input[0] != '\0');
717
718 // the output buffer
719 DynamicBuffer buf = DYNAMIC_BUFFER_INIT;
720 stream_read_cb data_cb = system_data_cb;
721 if (nread) {
722 *nread = 0;
723 }
724
725 if (forward_output) {
726 data_cb = out_data_cb;
727 } else if (!output) {
728 data_cb = NULL;
729 }
730
731 // Copy the program name in case we need to report an error.
732 char prog[MAXPATHL];
733 xstrlcpy(prog, argv[0], MAXPATHL);
734
735 LibuvProcess uvproc = libuv_process_init(&main_loop, &buf);
736 Process *proc = &uvproc.process;
737 MultiQueue *events = multiqueue_new_child(main_loop.events);
738 proc->events = events;
739 proc->argv = argv;
740 int status = process_spawn(proc, has_input, true, true);
741 if (status) {
742 loop_poll_events(&main_loop, 0);
743 // Failed, probably 'shell' is not executable.
744 if (!silent) {
745 msg_puts(_("\nshell failed to start: "));
746 msg_outtrans((char_u *)os_strerror(status));
747 msg_puts(": ");
748 msg_outtrans((char_u *)prog);
749 msg_putchar('\n');
750 }
751 multiqueue_free(events);
752 return -1;
753 }
754
755 // Note: unlike process events, stream events are not queued, as we want to
756 // deal with stream events as fast a possible. It prevents closing the
757 // streams while there's still data in the OS buffer (due to the process
758 // exiting before all data is read).
759 if (has_input) {
760 wstream_init(&proc->in, 0);
761 }
762 rstream_init(&proc->out, 0);
763 rstream_start(&proc->out, data_cb, &buf);
764 rstream_init(&proc->err, 0);
765 rstream_start(&proc->err, data_cb, &buf);
766
767 // write the input, if any
768 if (has_input) {
769 WBuffer *input_buffer = wstream_new_buffer((char *)input, len, 1, NULL);
770
771 if (!wstream_write(&proc->in, input_buffer)) {
772 // couldn't write, stop the process and tell the user about it
773 process_stop(proc);
774 return -1;
775 }
776 // close the input stream after everything is written
777 wstream_set_write_cb(&proc->in, shell_write_cb, NULL);
778 }
779
780 // Invoke busy_start here so LOOP_PROCESS_EVENTS_UNTIL will not change the
781 // busy state.
782 ui_busy_start();
783 ui_flush();
784 if (forward_output) {
785 msg_sb_eol();
786 msg_start();
787 msg_no_more = true;
788 lines_left = -1;
789 }
790 int exitcode = process_wait(proc, -1, NULL);
791 if (!got_int && out_data_decide_throttle(0)) {
792 // Last chunk of output was skipped; display it now.
793 out_data_ring(NULL, SIZE_MAX);
794 }
795 if (forward_output) {
796 // caller should decide if wait_return is invoked
797 no_wait_return++;
798 msg_end();
799 no_wait_return--;
800 msg_no_more = false;
801 }
802
803 ui_busy_stop();
804
805 // prepare the out parameters if requested
806 if (output) {
807 if (buf.len == 0) {
808 // no data received from the process, return NULL
809 *output = NULL;
810 xfree(buf.data);
811 } else {
812 // NUL-terminate to make the output directly usable as a C string
813 buf.data[buf.len] = NUL;
814 *output = buf.data;
815 }
816
817 if (nread) {
818 *nread = buf.len;
819 }
820 }
821
822 assert(multiqueue_empty(events));
823 multiqueue_free(events);
824
825 return exitcode;
826 }
827
828 /// - ensures at least `desired` bytes in buffer
829 ///
830 /// TODO(aktau): fold with kvec/garray
dynamic_buffer_ensure(DynamicBuffer * buf,size_t desired)831 static void dynamic_buffer_ensure(DynamicBuffer *buf, size_t desired)
832 {
833 if (buf->cap >= desired) {
834 assert(buf->data);
835 return;
836 }
837
838 buf->cap = desired;
839 kv_roundup32(buf->cap);
840 buf->data = xrealloc(buf->data, buf->cap);
841 }
842
system_data_cb(Stream * stream,RBuffer * buf,size_t count,void * data,bool eof)843 static void system_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data, bool eof)
844 {
845 DynamicBuffer *dbuf = data;
846
847 size_t nread = buf->size;
848 dynamic_buffer_ensure(dbuf, dbuf->len + nread + 1);
849 rbuffer_read(buf, dbuf->data + dbuf->len, nread);
850 dbuf->len += nread;
851 }
852
853 /// Tracks output received for the current executing shell command, and displays
854 /// a pulsing "..." when output should be skipped. Tracking depends on the
855 /// synchronous/blocking nature of ":!".
856 ///
857 /// Purpose:
858 /// 1. CTRL-C is more responsive. #1234 #5396
859 /// 2. Improves performance of :! (UI, esp. TUI, is the bottleneck).
860 /// 3. Avoids OOM during long-running, spammy :!.
861 ///
862 /// Vim does not need this hack because:
863 /// 1. :! in terminal-Vim runs in cooked mode, so CTRL-C is caught by the
864 /// terminal and raises SIGINT out-of-band.
865 /// 2. :! in terminal-Vim uses a tty (Nvim uses pipes), so commands
866 /// (e.g. `git grep`) may page themselves.
867 ///
868 /// @param size Length of data, used with internal state to decide whether
869 /// output should be skipped. size=0 resets the internal state and
870 /// returns the previous decision.
871 ///
872 /// @returns true if output should be skipped and pulse was displayed.
873 /// Returns the previous decision if size=0.
out_data_decide_throttle(size_t size)874 static bool out_data_decide_throttle(size_t size)
875 {
876 static uint64_t started = 0; // Start time of the current throttle.
877 static size_t received = 0; // Bytes observed since last throttle.
878 static size_t visit = 0; // "Pulse" count of the current throttle.
879 static char pulse_msg[] = { ' ', ' ', ' ', '\0' };
880
881 if (!size) {
882 bool previous_decision = (visit > 0);
883 started = received = visit = 0;
884 return previous_decision;
885 }
886
887 received += size;
888 if (received < OUT_DATA_THRESHOLD
889 // Display at least the first chunk of output even if it is big.
890 || (!started && received < size + 1000)) {
891 return false;
892 } else if (!visit) {
893 started = os_hrtime();
894 } else {
895 uint64_t since = os_hrtime() - started;
896 if (since < (visit * 0.1L * NS_1_SECOND)) {
897 return true;
898 }
899 if (since > (3 * NS_1_SECOND)) {
900 received = visit = 0;
901 return false;
902 }
903 }
904
905 visit++;
906 // Pulse "..." at the bottom of the screen.
907 size_t tick = visit % 4;
908 pulse_msg[0] = (tick > 0) ? '.' : ' ';
909 pulse_msg[1] = (tick > 1) ? '.' : ' ';
910 pulse_msg[2] = (tick > 2) ? '.' : ' ';
911 if (visit == 1) {
912 msg_puts("...\n");
913 }
914 msg_putchar('\r'); // put cursor at start of line
915 msg_puts(pulse_msg);
916 msg_putchar('\r');
917 ui_flush();
918 return true;
919 }
920
921 /// Saves output in a quasi-ringbuffer. Used to ensure the last ~page of
922 /// output for a shell-command is always displayed.
923 ///
924 /// Init mode: Resets the internal state.
925 /// output = NULL
926 /// size = 0
927 /// Print mode: Displays the current saved data.
928 /// output = NULL
929 /// size = SIZE_MAX
930 ///
931 /// @param output Data to save, or NULL to invoke a special mode.
932 /// @param size Length of `output`.
out_data_ring(char * output,size_t size)933 static void out_data_ring(char *output, size_t size)
934 {
935 #define MAX_CHUNK_SIZE (OUT_DATA_THRESHOLD / 2)
936 static char last_skipped[MAX_CHUNK_SIZE]; // Saved output.
937 static size_t last_skipped_len = 0;
938
939 assert(output != NULL || (size == 0 || size == SIZE_MAX));
940
941 if (output == NULL && size == 0) { // Init mode
942 last_skipped_len = 0;
943 return;
944 }
945
946 if (output == NULL && size == SIZE_MAX) { // Print mode
947 out_data_append_to_screen(last_skipped, &last_skipped_len, true);
948 return;
949 }
950
951 // This is basically a ring-buffer...
952 if (size >= MAX_CHUNK_SIZE) { // Save mode
953 size_t start = size - MAX_CHUNK_SIZE;
954 memcpy(last_skipped, output + start, MAX_CHUNK_SIZE);
955 last_skipped_len = MAX_CHUNK_SIZE;
956 } else if (size > 0) {
957 // Length of the old data that can be kept.
958 size_t keep_len = MIN(last_skipped_len, MAX_CHUNK_SIZE - size);
959 size_t keep_start = last_skipped_len - keep_len;
960 // Shift the kept part of the old data to the start.
961 if (keep_start) {
962 memmove(last_skipped, last_skipped + keep_start, keep_len);
963 }
964 // Copy the entire new data to the remaining space.
965 memcpy(last_skipped + keep_len, output, size);
966 last_skipped_len = keep_len + size;
967 }
968 }
969
970 /// Continue to append data to last screen line.
971 ///
972 /// @param output Data to append to screen lines.
973 /// @param remaining Size of data.
974 /// @param new_line If true, next data output will be on a new line.
out_data_append_to_screen(char * output,size_t * count,bool eof)975 static void out_data_append_to_screen(char *output, size_t *count, bool eof)
976 FUNC_ATTR_NONNULL_ALL
977 {
978 char *p = output, *end = output + *count;
979 while (p < end) {
980 if (*p == '\n' || *p == '\r' || *p == TAB || *p == BELL) {
981 msg_putchar_attr((uint8_t)(*p), 0);
982 p++;
983 } else {
984 // Note: this is not 100% precise:
985 // 1. we don't check if received continuation bytes are already invalid
986 // and we thus do some buffering that could be avoided
987 // 2. we don't compose chars over buffer boundaries, even if we see an
988 // incomplete UTF-8 sequence that could be composing with the last
989 // complete sequence.
990 // This will be corrected when we switch to vterm based implementation
991 int i = *p ? utfc_ptr2len_len((char_u *)p, (int)(end-p)) : 1;
992 if (!eof && i == 1 && utf8len_tab_zero[*(uint8_t *)p] > (end-p)) {
993 *count = (size_t)(p - output);
994 goto end;
995 }
996
997 (void)msg_outtrans_len_attr((char_u *)p, i, 0);
998 p += i;
999 }
1000 }
1001
1002 end:
1003 ui_flush();
1004 }
1005
out_data_cb(Stream * stream,RBuffer * buf,size_t count,void * data,bool eof)1006 static void out_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data, bool eof)
1007 {
1008 size_t cnt;
1009 char *ptr = rbuffer_read_ptr(buf, &cnt);
1010
1011 if (ptr != NULL && cnt > 0
1012 && out_data_decide_throttle(cnt)) { // Skip output above a threshold.
1013 // Save the skipped output. If it is the final chunk, we display it later.
1014 out_data_ring(ptr, cnt);
1015 } else if (ptr != NULL) {
1016 out_data_append_to_screen(ptr, &cnt, eof);
1017 }
1018
1019 if (cnt) {
1020 rbuffer_consumed(buf, cnt);
1021 }
1022
1023 // Move remaining data to start of buffer, so the buffer can never
1024 // wrap around.
1025 rbuffer_reset(buf);
1026 }
1027
1028 /// Parses a command string into a sequence of words, taking quotes into
1029 /// consideration.
1030 ///
1031 /// @param str The command string to be parsed
1032 /// @param argv The vector that will be filled with copies of the parsed
1033 /// words. It can be NULL if the caller only needs to count words.
1034 /// @return The number of words parsed.
tokenize(const char_u * const str,char ** const argv)1035 static size_t tokenize(const char_u *const str, char **const argv)
1036 FUNC_ATTR_NONNULL_ARG(1)
1037 {
1038 size_t argc = 0;
1039 const char *p = (const char *)str;
1040
1041 while (*p != NUL) {
1042 const size_t len = word_length((const char_u *)p);
1043
1044 if (argv != NULL) {
1045 // Fill the slot
1046 argv[argc] = vim_strnsave_unquoted(p, len);
1047 }
1048
1049 argc++;
1050 p = (const char *)skipwhite((char_u *)(p + len));
1051 }
1052
1053 return argc;
1054 }
1055
1056 /// Calculates the length of a shell word.
1057 ///
1058 /// @param str A pointer to the first character of the word
1059 /// @return The offset from `str` at which the word ends.
word_length(const char_u * str)1060 static size_t word_length(const char_u *str)
1061 {
1062 const char_u *p = str;
1063 bool inquote = false;
1064 size_t length = 0;
1065
1066 // Move `p` to the end of shell word by advancing the pointer while it's
1067 // inside a quote or it's a non-whitespace character
1068 while (*p && (inquote || (*p != ' ' && *p != TAB))) {
1069 if (*p == '"') {
1070 // Found a quote character, switch the `inquote` flag
1071 inquote = !inquote;
1072 } else if (*p == '\\' && inquote) {
1073 p++;
1074 length++;
1075 }
1076
1077 p++;
1078 length++;
1079 }
1080
1081 return length;
1082 }
1083
1084 /// To remain compatible with the old implementation (which forked a process
1085 /// for writing) the entire text is copied to a temporary buffer before the
1086 /// event loop starts. If we don't (by writing in chunks returned by `ml_get`)
1087 /// the buffer being modified might get modified by reading from the process
1088 /// before we finish writing.
read_input(DynamicBuffer * buf)1089 static void read_input(DynamicBuffer *buf)
1090 {
1091 size_t written = 0, l = 0, len = 0;
1092 linenr_T lnum = curbuf->b_op_start.lnum;
1093 char_u *lp = ml_get(lnum);
1094
1095 for (;;) {
1096 l = strlen((char *)lp + written);
1097 if (l == 0) {
1098 len = 0;
1099 } else if (lp[written] == NL) {
1100 // NL -> NUL translation
1101 len = 1;
1102 dynamic_buffer_ensure(buf, buf->len + len);
1103 buf->data[buf->len++] = NUL;
1104 } else {
1105 char_u *s = vim_strchr(lp + written, NL);
1106 len = s == NULL ? l : (size_t)(s - (lp + written));
1107 dynamic_buffer_ensure(buf, buf->len + len);
1108 memcpy(buf->data + buf->len, lp + written, len);
1109 buf->len += len;
1110 }
1111
1112 if (len == l) {
1113 // Finished a line, add a NL, unless this line should not have one.
1114 if (lnum != curbuf->b_op_end.lnum
1115 || (!curbuf->b_p_bin && curbuf->b_p_fixeol)
1116 || (lnum != curbuf->b_no_eol_lnum
1117 && (lnum != curbuf->b_ml.ml_line_count || curbuf->b_p_eol))) {
1118 dynamic_buffer_ensure(buf, buf->len + 1);
1119 buf->data[buf->len++] = NL;
1120 }
1121 ++lnum;
1122 if (lnum > curbuf->b_op_end.lnum) {
1123 break;
1124 }
1125 lp = ml_get(lnum);
1126 written = 0;
1127 } else if (len > 0) {
1128 written += len;
1129 }
1130 }
1131 }
1132
write_output(char * output,size_t remaining,bool eof)1133 static size_t write_output(char *output, size_t remaining, bool eof)
1134 {
1135 if (!output) {
1136 return 0;
1137 }
1138
1139 char *start = output;
1140 size_t off = 0;
1141 while (off < remaining) {
1142 if (output[off] == NL) {
1143 // Insert the line
1144 output[off] = NUL;
1145 ml_append(curwin->w_cursor.lnum++, (char_u *)output, (int)off + 1,
1146 false);
1147 size_t skip = off + 1;
1148 output += skip;
1149 remaining -= skip;
1150 off = 0;
1151 continue;
1152 }
1153
1154 if (output[off] == NUL) {
1155 // Translate NUL to NL
1156 output[off] = NL;
1157 }
1158 off++;
1159 }
1160
1161 if (eof) {
1162 if (remaining) {
1163 // append unfinished line
1164 ml_append(curwin->w_cursor.lnum++, (char_u *)output, 0, false);
1165 // remember that the NL was missing
1166 curbuf->b_no_eol_lnum = curwin->w_cursor.lnum;
1167 output += remaining;
1168 } else {
1169 curbuf->b_no_eol_lnum = 0;
1170 }
1171 }
1172
1173 ui_flush();
1174
1175 return (size_t)(output - start);
1176 }
1177
shell_write_cb(Stream * stream,void * data,int status)1178 static void shell_write_cb(Stream *stream, void *data, int status)
1179 {
1180 if (status) {
1181 // Can happen if system() tries to send input to a shell command that was
1182 // backgrounded (:call system("cat - &", "foo")). #3529 #5241
1183 msg_schedule_semsg(_("E5677: Error writing input to shell-command: %s"),
1184 uv_err_name(status));
1185 }
1186 stream_close(stream, NULL, NULL);
1187 }
1188
1189 /// Applies 'shellxescape' (p_sxe) and 'shellxquote' (p_sxq) to a command.
1190 ///
1191 /// @param cmd Command string
1192 /// @return Escaped/quoted command string (allocated).
shell_xescape_xquote(const char * cmd)1193 static char *shell_xescape_xquote(const char *cmd)
1194 FUNC_ATTR_NONNULL_ALL FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT
1195 {
1196 if (*p_sxq == NUL) {
1197 return xstrdup(cmd);
1198 }
1199
1200 const char *ecmd = cmd;
1201 if (*p_sxe != NUL && STRCMP(p_sxq, "(") == 0) {
1202 ecmd = (char *)vim_strsave_escaped_ext((char_u *)cmd, p_sxe, '^', false);
1203 }
1204 size_t ncmd_size = strlen(ecmd) + STRLEN(p_sxq) * 2 + 1;
1205 char *ncmd = xmalloc(ncmd_size);
1206
1207 // When 'shellxquote' is ( append ).
1208 // When 'shellxquote' is "( append )".
1209 if (STRCMP(p_sxq, "(") == 0) {
1210 vim_snprintf(ncmd, ncmd_size, "(%s)", ecmd);
1211 } else if (STRCMP(p_sxq, "\"(") == 0) {
1212 vim_snprintf(ncmd, ncmd_size, "\"(%s)\"", ecmd);
1213 } else {
1214 vim_snprintf(ncmd, ncmd_size, "%s%s%s", p_sxq, ecmd, p_sxq);
1215 }
1216
1217 if (ecmd != cmd) {
1218 xfree((void *)ecmd);
1219 }
1220
1221 return ncmd;
1222 }
1223
1224