unitproc/bkt-count_jobs

   1 #!/usr/bin/env bash
   2
   3 # Note: GNU Parallel obviates the need for this script. The main
   4 # motivation to writing this script was to manage the number of CPU
   5 # threads being spawned by a list of files being feed to a while loop.
   6
   7 # Functions
   8 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
   9 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
  10 try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
  11 count_jobs() {
  12     # Desc: Count and return total number of jobs
  13     # Usage: count_jobs
  14     # Input: None.
  15     # Output: stdout   integer number of jobs
  16     # Depends: Bash 5.1.16
  17     # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
  18     # Version: 0.0.1
  19
  20     local job_count;
  21     job_count="$(jobs -r | wc -l | tr -d ' ' )";
  22     #yell "DEBUG:job_count:$job_count";
  23     if [[ -z $job_count ]]; then job_count="0"; fi;
  24     echo "$job_count";
  25 }; # Return number of background jobs
  26 test_job() {
  27     #yell "DEBUG:starting test_job() with:$1";
  28     # Do work on $1 file
  29     if [[ -f $1 ]]; then cat "$1" 1>/dev/random 2>&1; fi; # read file
  30     sleep "$(shuf -i1-10 -n1)"; # debug
  31 }; # Test job
  32 count_jobs_display_update() {
  33     # Depends: various variables
  34     if [[ $(( SECONDS % jobs_update_interval )) -eq $jobs_update_init_delay ]] && \
  35        [[ $permit_update == "true" ]]; then
  36     jobs_beg="$jobs_n";
  37     jobs_run="$(count_jobs)";
  38     jobs_end="$((jobs_beg - jobs_run))";
  39     yell "STATUS:$jobs_beg jobs begun. $jobs_end jobs ended. $jobs_run jobs running.";
  40     permit_update="false";
  41     fi;
  42     if [[ $(( SECONDS % jobs_update_interval )) -eq $((jobs_update_init_delay + 1)) ]]; then
  43         permit_update="true";
  44     fi;
  45 }; # periodically display updates
  46
  47
  48 # Test Code
  49 path_target="/tmp"; # path to dir with files to run test_job() on
  50
  51 ## Setup file list
  52 list_paths="$(find "$path_target" -type f 2>/dev/null)";
  53
  54 ## Perform test_job() on each file
  55 jobs_max=100; # adjust me (e.g. "4" on 4-core CPU)
  56 jobs_update_interval=5; # seconds.
  57 jobs_update_init_delay=1; # seconds.
  58 permit_update="true"; # flag to make updates happen periodically
  59 jobs_n=0; # loop counter
  60 jobs_check_delay=0.001; # seconds between each job count check if running jobs > jobs_max
  61 while read -r line; do
  62     ((jobs_n++));
  63     #yell "DEBUG:jobs_n:$jobs_n"
  64
  65     ## Wait until job count falls below $jobs_max
  66     while [[ "$(count_jobs)" -gt $jobs_max ]]; do
  67         #yell "DEBUG:sleeping since $(count_jobs) > $jobs_max";
  68         sleep "$jobs_check_delay";
  69         count_jobs_display_update;
  70     done;
  71
  72     ## Start new parallel job on file $line
  73     test_job "$line" 1>/dev/null 2>&1 &
  74
  75     ## Get job status updates every $jobs_update_interval seconds
  76     count_jobs_display_update;
  77
  78     #sleep 1; # debug
  79 done < <( shuf < <(echo -n "$list_paths") );
  80 yell "STATUS:All jobs ($jobs_n) started. $(count_jobs) jobs running.";
  81
  82 # Detect when no outstanding jobs
  83 while [[ "$(count_jobs)" -gt 0 ]]; do
  84     sleep 1;
  85     count_jobs_display_update;
  86 done;
  87 yell "STATUS:No more jobs visible.";
  88
  89 # Author: Steven Baltakatei Sandoval
  90 # License: GPLv3+