1#!/usr/bin/env ksh 2 3# This file must be sourced in mksh: 4# 5# source `which env_parallel.mksh` 6# 7# after which 'env_parallel' works 8# 9# 10# Copyright (C) 2016-2021 Ole Tange, http://ole.tange.dk and Free 11# Software Foundation, Inc. 12# 13# This program is free software; you can redistribute it and/or modify 14# it under the terms of the GNU General Public License as published by 15# the Free Software Foundation; either version 3 of the License, or 16# (at your option) any later version. 17# 18# This program is distributed in the hope that it will be useful, but 19# WITHOUT ANY WARRANTY; without even the implied warranty of 20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21# General Public License for more details. 22# 23# You should have received a copy of the GNU General Public License 24# along with this program; if not, see <http://www.gnu.org/licenses/> 25# or write to the Free Software Foundation, Inc., 51 Franklin St, 26# Fifth Floor, Boston, MA 02110-1301 USA 27# 28# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. 29# SPDX-License-Identifier: GPL-3.0-or-later 30 31env_parallel() { 32 # env_parallel.mksh 33 34 _names_of_ALIASES() { 35 alias | perl -pe 's/=.*//' 36 } 37 _bodies_of_ALIASES() { 38 alias "$@" | perl -pe 's/^/alias /; 39 sub warning { print STDERR "env_parallel: Warning: @_\n"; } 40 if(/^alias (\S+)=\$.*\\n/) { 41 warning("Alias \"$1\" contains newline."); 42 warning("Make sure the command has at least one newline after \"$1\"."); 43 warning("See BUGS in \"man env_parallel\"."); 44 }' 45 46 } 47 _names_of_maybe_FUNCTIONS() { 48 true not used 49 } 50 _names_of_FUNCTIONS() { 51 typeset +f 52 } 53 _bodies_of_FUNCTIONS() { 54 typeset -f "$@" 55 } 56 _names_of_VARIABLES() { 57 typeset +p | 58 perl -pe 's/^(type)?set( [-+][a-zA-Z0-9]*)* //; s/(\[\d+\])?=.*//' | 59 uniq 60 } 61 _bodies_of_VARIABLES() { 62 typeset -p "$@" 63 } 64 _ignore_HARDCODED() { 65 # These names cannot be detected 66 echo '(_)' 67 } 68 _ignore_READONLY() { 69 readonly | perl -e '@r = map { 70 chomp; 71 # sh on UnixWare: readonly TIMEOUT 72 # ash: readonly var='val' 73 # ksh: var='val' 74 # mksh: PIPESTATUS[0] 75 s/^(readonly )?([^=\[ ]*?)(\[\d+\])?(=.*|)$/$2/ or 76 # bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4") 77 # zsh: typeset -r var='val' 78 s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/; 79 $_ } <>; 80 $vars = join "|",map { quotemeta $_ } @r; 81 print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; 82 ' 83 } 84 _remove_bad_NAMES() { 85 # Do not transfer vars and funcs from env_parallel 86 _ignore_RO="`_ignore_READONLY`" 87 _ignore_HARD="`_ignore_HARDCODED`" 88 # Macos-grep does not like long patterns 89 # Old Solaris grep does not support -E 90 # Perl Version of: 91 # grep -Ev '^(...)$' | 92 perl -ne '/^( 93 PARALLEL_ENV| 94 PARALLEL_TMP| 95 _alias_NAMES| 96 _bodies_of_ALIASES| 97 _bodies_of_FUNCTIONS| 98 _bodies_of_VARIABLES| 99 _error_PAR| 100 _function_NAMES| 101 _get_ignored_VARS| 102 _grep_REGEXP| 103 _ignore_HARD| 104 _ignore_HARDCODED| 105 _ignore_READONLY| 106 _ignore_RO| 107 _ignore_UNDERSCORE| 108 _list_alias_BODIES| 109 _list_function_BODIES| 110 _list_variable_VALUES| 111 _make_grep_REGEXP| 112 _names_of_ALIASES| 113 _names_of_FUNCTIONS| 114 _names_of_VARIABLES| 115 _names_of_maybe_FUNCTIONS| 116 _parallel_exit_CODE| 117 _prefix_PARALLEL_ENV| 118 _prefix_PARALLEL_ENV| 119 _remove_bad_NAMES| 120 _remove_readonly| 121 _variable_NAMES| 122 _warning_PAR| 123 _which_PAR)$/x and next; 124 # Filter names matching --env 125 /^'"$_grep_REGEXP"'$/ or next; 126 /^'"$_ignore_UNDERSCORE"'$/ and next; 127 # Remove readonly variables 128 /^'"$_ignore_RO"'$/ and next; 129 /^'"$_ignore_HARD"'$/ and next; 130 print;' 131 } 132 _get_ignored_VARS() { 133 perl -e ' 134 for(@ARGV){ 135 $next_is_env and push @envvar, split/,/, $_; 136 $next_is_env=/^--env$/; 137 } 138 if(grep { /^_$/ } @envvar) { 139 if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { 140 print STDERR "parallel: Error: ", 141 "Run \"parallel --record-env\" in a clean environment first.\n"; 142 } else { 143 chomp(@ignored_vars = <IN>); 144 } 145 } 146 if($ENV{PARALLEL_IGNORED_NAMES}) { 147 push @ignored_vars, split/\s+/, $ENV{PARALLEL_IGNORED_NAMES}; 148 chomp @ignored_vars; 149 } 150 $vars = join "|",map { quotemeta $_ } @ignored_vars; 151 print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; 152 ' -- "$@" 153 } 154 155 # Get the --env variables if set 156 # --env _ should be ignored 157 # and convert a b c to (a|b|c) 158 # If --env not set: Match everything (.*) 159 _make_grep_REGEXP() { 160 perl -e ' 161 for(@ARGV){ 162 /^_$/ and $next_is_env = 0; 163 $next_is_env and push @envvar, split/,/, $_; 164 $next_is_env = /^--env$/; 165 } 166 $vars = join "|",map { quotemeta $_ } @envvar; 167 print $vars ? "($vars)" : "(.*)"; 168 ' -- "$@" 169 } 170 _which_PAR() { 171 # type returns: 172 # ll is an alias for ls -l (in ash) 173 # bash is a tracked alias for /bin/bash 174 # true is a shell builtin (in bash) 175 # myfunc is a function (in bash) 176 # myfunc is a shell function (in zsh) 177 # which is /usr/bin/which (in sh, bash) 178 # which is hashed (/usr/bin/which) 179 # gi is aliased to `grep -i' (in bash) 180 # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' 181 # Return 0 if found, 1 otherwise 182 LANG=C type "$@" | 183 perl -pe '$exit += (s/ is an alias for .*// || 184 s/ is aliased to .*// || 185 s/ is a function// || 186 s/ is a shell function// || 187 s/ is a shell builtin// || 188 s/.* is hashed .(\S+).$/$1/ || 189 s/.* is (a tracked alias for )?//); 190 END { exit not $exit }' 191 } 192 _warning_PAR() { 193 echo "env_parallel: Warning: $@" >&2 194 } 195 _error_PAR() { 196 echo "env_parallel: Error: $@" >&2 197 } 198 199 if _which_PAR parallel >/dev/null; then 200 true parallel found in path 201 else 202 _error_PAR 'parallel must be in $PATH.' 203 return 255 204 fi 205 206 # Grep regexp for vars given by --env 207 _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" 208 unset _make_grep_REGEXP 209 210 # Deal with --env _ 211 _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" 212 unset _get_ignored_VARS 213 214 # --record-env 215 if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then 216 true skip 217 else 218 (_names_of_ALIASES; 219 _names_of_FUNCTIONS; 220 _names_of_VARIABLES) | 221 cat > "$HOME"/.parallel/ignored_vars 222 return 0 223 fi 224 225 # --session 226 if perl -e 'exit grep { /^--session$/ } @ARGV' -- "$@"; then 227 true skip 228 else 229 # Insert ::: between each level of session 230 # so you can pop off the last ::: at --end-session 231 PARALLEL_IGNORED_NAMES="`echo \"$PARALLEL_IGNORED_NAMES\"; 232 echo :::; 233 (_names_of_ALIASES; 234 _names_of_FUNCTIONS; 235 _names_of_VARIABLES) | perl -ne ' 236 BEGIN{ 237 map { $ignored_vars{$_}++ } 238 split/\s+/, $ENV{PARALLEL_IGNORED_NAMES}; 239 } 240 chomp; 241 for(split/\s+/) { 242 if(not $ignored_vars{$_}) { 243 print $_,\"\\n\"; 244 } 245 } 246 '`" 247 export PARALLEL_IGNORED_NAMES 248 return 0 249 fi 250 if perl -e 'exit grep { /^--end.?session$/ } @ARGV' -- "$@"; then 251 true skip 252 else 253 # Pop off last ::: from PARALLEL_IGNORED_NAMES 254 PARALLEL_IGNORED_NAMES="`perl -e ' 255 $ENV{PARALLEL_IGNORED_NAMES} =~ s/(.*):::.*?$/$1/s; 256 print $ENV{PARALLEL_IGNORED_NAMES} 257 '`" 258 return 0 259 fi 260 # Grep alias names 261 _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`" 262 _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" 263 if [ "$_alias_NAMES" = "" ] ; then 264 # no aliases selected 265 _list_alias_BODIES="true" 266 fi 267 unset _alias_NAMES 268 269 # Grep function names 270 _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`" 271 _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" 272 if [ "$_function_NAMES" = "" ] ; then 273 # no functions selected 274 _list_function_BODIES="true" 275 fi 276 unset _function_NAMES 277 278 # Grep variable names 279 _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`" 280 _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" 281 if [ "$_variable_NAMES" = "" ] ; then 282 # no variables selected 283 _list_variable_VALUES="true" 284 fi 285 unset _variable_NAMES 286 287 PARALLEL_ENV="` 288 $_list_alias_BODIES; 289 $_list_function_BODIES; 290 $_list_variable_VALUES; 291 `" 292 export PARALLEL_ENV 293 unset _list_alias_BODIES _list_variable_VALUES _list_function_BODIES 294 unset _bodies_of_ALIASES _bodies_of_VARIABLES _bodies_of_FUNCTIONS 295 unset _names_of_ALIASES _names_of_VARIABLES _names_of_FUNCTIONS 296 unset _ignore_HARDCODED _ignore_READONLY _ignore_UNDERSCORE 297 unset _remove_bad_NAMES _grep_REGEXP 298 unset _prefix_PARALLEL_ENV 299 # Test if environment is too big 300 if `_which_PAR true` >/dev/null 2>/dev/null ; then 301 parallel "$@" 302 _parallel_exit_CODE=$? 303 # Clean up variables/functions 304 unset PARALLEL_ENV 305 unset _which_PAR _which_TRUE 306 unset _warning_PAR _error_PAR 307 # Unset _parallel_exit_CODE before return 308 eval "unset _parallel_exit_CODE; return $_parallel_exit_CODE" 309 else 310 unset PARALLEL_ENV; 311 _error_PAR "Your environment is too big." 312 _error_PAR "You can try 3 different approaches:" 313 _error_PAR "1. Run 'env_parallel --session' before you set" 314 _error_PAR " variables or define functions." 315 _error_PAR "2. Use --env and only mention the names to copy." 316 _error_PAR "3. Try running this in a clean environment once:" 317 _error_PAR " env_parallel --record-env" 318 _error_PAR " And then use '--env _'" 319 _error_PAR "For details see: man env_parallel" 320 return 255 321 fi 322} 323 324parset() { 325 _parset_PARALLEL_PRG=parallel 326 _parset_main "$@" 327} 328 329env_parset() { 330 _parset_PARALLEL_PRG=env_parallel 331 _parset_main "$@" 332} 333 334_parset_main() { 335 # If $1 contains ',' or space: 336 # Split on , to get the destination variable names 337 # If $1 is a single destination variable name: 338 # Treat it as the name of an array 339 # 340 # # Create array named myvar 341 # parset myvar echo ::: {1..10} 342 # echo ${myvar[5]} 343 # 344 # # Put output into $var_a $var_b $var_c 345 # varnames=(var_a var_b var_c) 346 # parset "${varnames[*]}" echo ::: {1..3} 347 # echo $var_c 348 # 349 # # Put output into $var_a4 $var_b4 $var_c4 350 # parset "var_a4 var_b4 var_c4" echo ::: {1..3} 351 # echo $var_c4 352 353 _parset_NAME="$1" 354 if [ "$_parset_NAME" = "" ] ; then 355 echo parset: Error: No destination variable given. >&2 356 echo parset: Error: Try: >&2 357 echo parset: Error: ' ' parset myarray echo ::: foo bar >&2 358 return 255 359 fi 360 if [ "$_parset_NAME" = "--help" ] ; then 361 echo parset: Error: Usage: >&2 362 echo parset: Error: ' ' parset varname GNU Parallel options and command >&2 363 echo 364 parallel --help 365 return 255 366 fi 367 if [ "$_parset_NAME" = "--version" ] ; then 368 echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" 369 echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" 370 echo "Foundation, Inc." 371 echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" 372 echo "This is free software: you are free to change and redistribute it." 373 echo "GNU parallel comes with no warranty." 374 echo 375 echo "Web site: https://www.gnu.org/software/parallel" 376 echo 377 echo "When using programs that use GNU Parallel to process data for publication" 378 echo "please cite as described in 'parallel --citation'." 379 echo 380 return 255 381 fi 382 shift 383 384 # Bash: declare -A myassoc=( ) 385 # Zsh: typeset -A myassoc=( ) 386 # Ksh: typeset -A myassoc=( ) 387 if (typeset -p "$_parset_NAME" 2>/dev/null; echo) | 388 perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then 389 # This is an associative array 390 eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`" 391 # The eval returns the function! 392 else 393 # This is a normal array or a list of variable names 394 eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`" 395 # The eval returns the function! 396 fi 397} 398