3 # Note: GNU Parallel obviates the need for this script. The main
4 # motivation to writing this script was to manage the number of CPU
5 # threads being spawned by a list of files being feed to a while loop.
8 yell
() { echo "$0: $*" >&2; } # print script path and all args to stderr
9 die
() { yell
"$*"; exit 111; } # same as yell() but non-zero exit status
10 try
() { "$@" || die
"cannot $*"; } # runs args as command, reports args if command fails
13 job_count
="$(jobs -r | wc -l | tr -d ' ' )";
14 #yell "DEBUG:job_count:$job_count";
15 if [[ -z $job_count ]]; then job_count
="0"; fi;
17 }; # Return number of background jobs
19 #yell "DEBUG:starting test_job() with:$1";
21 if [[ -f $1 ]]; then cat "$1" 1>/dev
/random
2>&1; fi; # read file
22 sleep "$(shuf -i1-10 -n1)"; # debug
24 count_jobs_display_update
() {
25 # Depends: various variables
26 if [[ $
(( SECONDS
% jobs_update_interval
)) -eq $jobs_update_init_delay ]] && \
27 [[ $permit_update == "true" ]]; then
29 jobs_run
="$(count_jobs)";
30 jobs_end
="$((jobs_beg - jobs_run))";
31 yell
"STATUS:$jobs_beg jobs begun. $jobs_end jobs ended. $jobs_run jobs running.";
32 permit_update
="false";
34 if [[ $
(( SECONDS
% jobs_update_interval
)) -eq $
((jobs_update_init_delay
+ 1)) ]]; then
37 }; # periodically display updates
41 path_target
="/tmp"; # path to dir with files to run test_job() on
44 list_paths
="$(find "$path_target" -type f 2>/dev/null)";
46 ## Perform test_job() on each file
47 jobs_max
=100; # adjust me (e.g. "4" on 4-core CPU)
48 jobs_update_interval
=5; # seconds.
49 jobs_update_init_delay
=1; # seconds.
50 permit_update
="true"; # flag to make updates happen periodically
51 jobs_n
=0; # loop counter
52 jobs_check_delay
=0.001; # seconds between each job count check if running jobs > jobs_max
53 while read -r line
; do
55 #yell "DEBUG:jobs_n:$jobs_n"
57 ## Wait until job count falls below $jobs_max
58 while [[ "$(count_jobs)" -gt $jobs_max ]]; do
59 #yell "DEBUG:sleeping since $(count_jobs) > $jobs_max";
60 sleep "$jobs_check_delay";
61 count_jobs_display_update
;
64 ## Start new parallel job on file $line
65 test_job
"$line" 1>/dev
/null
2>&1 &
67 ## Get job status updates every $jobs_update_interval seconds
68 count_jobs_display_update
;
71 done < <( shuf
< <(echo -n "$list_paths") );
72 yell
"STATUS:All jobs ($jobs_n) started. $(count_jobs) jobs running.";
74 # Detect when no outstanding jobs
75 while [[ "$(count_jobs)" -gt 0 ]]; do
77 count_jobs_display_update
;
79 yell
"STATUS:No more jobs visible.";
81 # Author: Steven Baltakatei Sandoval