Commit | Line | Data |
---|---|---|
868a58fb SBS |
1 | #!/usr/bin/env bash |
2 | ||
3 | # Note: GNU Parallel obviates the need for this script. The main | |
4 | # motivation to writing this script was to manage the number of CPU | |
5 | # threads being spawned by a list of files being feed to a while loop. | |
6 | ||
7 | # Functions | |
8 | yell() { echo "$0: $*" >&2; } # print script path and all args to stderr | |
9 | die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status | |
10 | try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails | |
11 | count_jobs() { | |
07b20692 SBS |
12 | # Desc: Count and return total number of jobs |
13 | # Usage: count_jobs | |
14 | # Input: None. | |
15 | # Output: stdout integer number of jobs | |
16 | # Depends: Bash 5.1.16 | |
17 | # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; | |
18 | # Version: 0.0.1 | |
19 | ||
868a58fb SBS |
20 | local job_count; |
21 | job_count="$(jobs -r | wc -l | tr -d ' ' )"; | |
22 | #yell "DEBUG:job_count:$job_count"; | |
23 | if [[ -z $job_count ]]; then job_count="0"; fi; | |
24 | echo "$job_count"; | |
25 | }; # Return number of background jobs | |
26 | test_job() { | |
27 | #yell "DEBUG:starting test_job() with:$1"; | |
28 | # Do work on $1 file | |
29 | if [[ -f $1 ]]; then cat "$1" 1>/dev/random 2>&1; fi; # read file | |
30 | sleep "$(shuf -i1-10 -n1)"; # debug | |
31 | }; # Test job | |
32 | count_jobs_display_update() { | |
33 | # Depends: various variables | |
34 | if [[ $(( SECONDS % jobs_update_interval )) -eq $jobs_update_init_delay ]] && \ | |
35 | [[ $permit_update == "true" ]]; then | |
36 | jobs_beg="$jobs_n"; | |
37 | jobs_run="$(count_jobs)"; | |
38 | jobs_end="$((jobs_beg - jobs_run))"; | |
39 | yell "STATUS:$jobs_beg jobs begun. $jobs_end jobs ended. $jobs_run jobs running."; | |
40 | permit_update="false"; | |
41 | fi; | |
42 | if [[ $(( SECONDS % jobs_update_interval )) -eq $((jobs_update_init_delay + 1)) ]]; then | |
43 | permit_update="true"; | |
44 | fi; | |
45 | }; # periodically display updates | |
46 | ||
47 | ||
48 | # Test Code | |
49 | path_target="/tmp"; # path to dir with files to run test_job() on | |
50 | ||
51 | ## Setup file list | |
52 | list_paths="$(find "$path_target" -type f 2>/dev/null)"; | |
53 | ||
54 | ## Perform test_job() on each file | |
55 | jobs_max=100; # adjust me (e.g. "4" on 4-core CPU) | |
56 | jobs_update_interval=5; # seconds. | |
57 | jobs_update_init_delay=1; # seconds. | |
58 | permit_update="true"; # flag to make updates happen periodically | |
59 | jobs_n=0; # loop counter | |
60 | jobs_check_delay=0.001; # seconds between each job count check if running jobs > jobs_max | |
61 | while read -r line; do | |
62 | ((jobs_n++)); | |
63 | #yell "DEBUG:jobs_n:$jobs_n" | |
64 | ||
65 | ## Wait until job count falls below $jobs_max | |
66 | while [[ "$(count_jobs)" -gt $jobs_max ]]; do | |
67 | #yell "DEBUG:sleeping since $(count_jobs) > $jobs_max"; | |
68 | sleep "$jobs_check_delay"; | |
69 | count_jobs_display_update; | |
70 | done; | |
71 | ||
72 | ## Start new parallel job on file $line | |
73 | test_job "$line" 1>/dev/null 2>&1 & | |
74 | ||
75 | ## Get job status updates every $jobs_update_interval seconds | |
76 | count_jobs_display_update; | |
77 | ||
78 | #sleep 1; # debug | |
79 | done < <( shuf < <(echo -n "$list_paths") ); | |
80 | yell "STATUS:All jobs ($jobs_n) started. $(count_jobs) jobs running."; | |
81 | ||
82 | # Detect when no outstanding jobs | |
83 | while [[ "$(count_jobs)" -gt 0 ]]; do | |
84 | sleep 1; | |
85 | count_jobs_display_update; | |
86 | done; | |
87 | yell "STATUS:No more jobs visible."; | |
88 | ||
89 | # Author: Steven Baltakatei Sandoval | |
90 | # License: GPLv3+ |