| 1 | #!/usr/bin/env bash |
| 2 | |
| 3 | # Note: GNU Parallel obviates the need for this script. The main |
| 4 | # motivation to writing this script was to manage the number of CPU |
| 5 | # threads being spawned by a list of files being feed to a while loop. |
| 6 | |
| 7 | # Functions |
| 8 | yell() { echo "$0: $*" >&2; } # print script path and all args to stderr |
| 9 | die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status |
| 10 | try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails |
| 11 | count_jobs() { |
| 12 | # Desc: Count and return total number of jobs |
| 13 | # Usage: count_jobs |
| 14 | # Input: None. |
| 15 | # Output: stdout integer number of jobs |
| 16 | # Depends: Bash 5.1.16 |
| 17 | # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; |
| 18 | # Version: 0.0.1 |
| 19 | |
| 20 | local job_count; |
| 21 | job_count="$(jobs -r | wc -l | tr -d ' ' )"; |
| 22 | #yell "DEBUG:job_count:$job_count"; |
| 23 | if [[ -z $job_count ]]; then job_count="0"; fi; |
| 24 | echo "$job_count"; |
| 25 | }; # Return number of background jobs |
| 26 | test_job() { |
| 27 | #yell "DEBUG:starting test_job() with:$1"; |
| 28 | # Do work on $1 file |
| 29 | if [[ -f $1 ]]; then cat "$1" 1>/dev/random 2>&1; fi; # read file |
| 30 | sleep "$(shuf -i1-10 -n1)"; # debug |
| 31 | }; # Test job |
| 32 | count_jobs_display_update() { |
| 33 | # Depends: various variables |
| 34 | if [[ $(( SECONDS % jobs_update_interval )) -eq $jobs_update_init_delay ]] && \ |
| 35 | [[ $permit_update == "true" ]]; then |
| 36 | jobs_beg="$jobs_n"; |
| 37 | jobs_run="$(count_jobs)"; |
| 38 | jobs_end="$((jobs_beg - jobs_run))"; |
| 39 | yell "STATUS:$jobs_beg jobs begun. $jobs_end jobs ended. $jobs_run jobs running."; |
| 40 | permit_update="false"; |
| 41 | fi; |
| 42 | if [[ $(( SECONDS % jobs_update_interval )) -eq $((jobs_update_init_delay + 1)) ]]; then |
| 43 | permit_update="true"; |
| 44 | fi; |
| 45 | }; # periodically display updates |
| 46 | |
| 47 | |
| 48 | # Test Code |
| 49 | path_target="/tmp"; # path to dir with files to run test_job() on |
| 50 | |
| 51 | ## Setup file list |
| 52 | list_paths="$(find "$path_target" -type f 2>/dev/null)"; |
| 53 | |
| 54 | ## Perform test_job() on each file |
| 55 | jobs_max=100; # adjust me (e.g. "4" on 4-core CPU) |
| 56 | jobs_update_interval=5; # seconds. |
| 57 | jobs_update_init_delay=1; # seconds. |
| 58 | permit_update="true"; # flag to make updates happen periodically |
| 59 | jobs_n=0; # loop counter |
| 60 | jobs_check_delay=0.001; # seconds between each job count check if running jobs > jobs_max |
| 61 | while read -r line; do |
| 62 | ((jobs_n++)); |
| 63 | #yell "DEBUG:jobs_n:$jobs_n" |
| 64 | |
| 65 | ## Wait until job count falls below $jobs_max |
| 66 | while [[ "$(count_jobs)" -gt $jobs_max ]]; do |
| 67 | #yell "DEBUG:sleeping since $(count_jobs) > $jobs_max"; |
| 68 | sleep "$jobs_check_delay"; |
| 69 | count_jobs_display_update; |
| 70 | done; |
| 71 | |
| 72 | ## Start new parallel job on file $line |
| 73 | test_job "$line" 1>/dev/null 2>&1 & |
| 74 | |
| 75 | ## Get job status updates every $jobs_update_interval seconds |
| 76 | count_jobs_display_update; |
| 77 | |
| 78 | #sleep 1; # debug |
| 79 | done < <( shuf < <(echo -n "$list_paths") ); |
| 80 | yell "STATUS:All jobs ($jobs_n) started. $(count_jobs) jobs running."; |
| 81 | |
| 82 | # Detect when no outstanding jobs |
| 83 | while [[ "$(count_jobs)" -gt 0 ]]; do |
| 84 | sleep 1; |
| 85 | count_jobs_display_update; |
| 86 | done; |
| 87 | yell "STATUS:No more jobs visible."; |
| 88 | |
| 89 | # Author: Steven Baltakatei Sandoval |
| 90 | # License: GPLv3+ |