feat(user/bkots):Optimize with GNU Parallel for executing commands
[BK-2020-03.git] / unitproc / bktemp-count_jobs
... / ...
CommitLineData
1#!/usr/bin/env bash
2
3# Note: GNU Parallel obviates the need for this script. The main
4# motivation to writing this script was to manage the number of CPU
5# threads being spawned by a list of files being feed to a while loop.
6
7# Functions
8yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
9die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
10try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
11count_jobs() {
12 local job_count;
13 job_count="$(jobs -r | wc -l | tr -d ' ' )";
14 #yell "DEBUG:job_count:$job_count";
15 if [[ -z $job_count ]]; then job_count="0"; fi;
16 echo "$job_count";
17}; # Return number of background jobs
18test_job() {
19 #yell "DEBUG:starting test_job() with:$1";
20 # Do work on $1 file
21 if [[ -f $1 ]]; then cat "$1" 1>/dev/random 2>&1; fi; # read file
22 sleep "$(shuf -i1-10 -n1)"; # debug
23}; # Test job
24count_jobs_display_update() {
25 # Depends: various variables
26 if [[ $(( SECONDS % jobs_update_interval )) -eq $jobs_update_init_delay ]] && \
27 [[ $permit_update == "true" ]]; then
28 jobs_beg="$jobs_n";
29 jobs_run="$(count_jobs)";
30 jobs_end="$((jobs_beg - jobs_run))";
31 yell "STATUS:$jobs_beg jobs begun. $jobs_end jobs ended. $jobs_run jobs running.";
32 permit_update="false";
33 fi;
34 if [[ $(( SECONDS % jobs_update_interval )) -eq $((jobs_update_init_delay + 1)) ]]; then
35 permit_update="true";
36 fi;
37}; # periodically display updates
38
39
40# Test Code
41path_target="/tmp"; # path to dir with files to run test_job() on
42
43## Setup file list
44list_paths="$(find "$path_target" -type f 2>/dev/null)";
45
46## Perform test_job() on each file
47jobs_max=100; # adjust me (e.g. "4" on 4-core CPU)
48jobs_update_interval=5; # seconds.
49jobs_update_init_delay=1; # seconds.
50permit_update="true"; # flag to make updates happen periodically
51jobs_n=0; # loop counter
52jobs_check_delay=0.001; # seconds between each job count check if running jobs > jobs_max
53while read -r line; do
54 ((jobs_n++));
55 #yell "DEBUG:jobs_n:$jobs_n"
56
57 ## Wait until job count falls below $jobs_max
58 while [[ "$(count_jobs)" -gt $jobs_max ]]; do
59 #yell "DEBUG:sleeping since $(count_jobs) > $jobs_max";
60 sleep "$jobs_check_delay";
61 count_jobs_display_update;
62 done;
63
64 ## Start new parallel job on file $line
65 test_job "$line" 1>/dev/null 2>&1 &
66
67 ## Get job status updates every $jobs_update_interval seconds
68 count_jobs_display_update;
69
70 #sleep 1; # debug
71done < <( shuf < <(echo -n "$list_paths") );
72yell "STATUS:All jobs ($jobs_n) started. $(count_jobs) jobs running.";
73
74# Detect when no outstanding jobs
75while [[ "$(count_jobs)" -gt 0 ]]; do
76 sleep 1;
77 count_jobs_display_update;
78done;
79yell "STATUS:No more jobs visible.";
80
81# Author: Steven Baltakatei Sandoval
82# License: GPLv3+