feat(user/rand_media_pl.sh):Make robust and parallelize

[BK-2020-03.git] / user / bkots

diff --git a/user/bkots b/user/bkots
index baffb0886d3dc152fffc81b1aec729d31a57b952..3c9705ca2cb58a20ee3ebdf2d1f9e05605d178c3 100755 (executable)
--- a/user/bkots
+++ b/user/bkots
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
 # Define variables
 #!/usr/bin/env bash
 
 # Define variables

+declare -g max_job_count="2"; # default max job count
+declare -g age_threshold="60"; # min age to add file; seconds;

 declare -Ag appRollCall # Associative array for storing app status
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status
 declare -Ag appRollCall # Associative array for storing app status
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status


@@ -10,13 +12,11 @@ calendars+=("https://finney.calendar.eternitywall.com");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");

-declare -a commands # array for storing assembled commands
-age_threshold="60"; # min age to add file; seconds;

 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status

-try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
+must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails

 checkapp() {
     # Desc: If arg is a command, save result in assoc array 'appRollCall'
     # Usage: checkapp arg1 arg2 arg3 ...
 checkapp() {
     # Desc: If arg is a command, save result in assoc array 'appRollCall'
     # Usage: checkapp arg1 arg2 arg3 ...


@@ -200,7 +200,7 @@ showVersion() {
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'

-bkots 1.0.3
+bkots 2.1.1

 Copyright (C) 2022 Steven Baltakatei Sandoval
 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.
 Copyright (C) 2022 Steven Baltakatei Sandoval
 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.


@@ -239,6 +239,8 @@ showUsage() {
         --include-dotfiles
                 Include files and directories starting with '.' (not
                 included by default).
         --include-dotfiles
                 Include files and directories starting with '.' (not
                 included by default).

+        -j, --jobs
+                Specify simultaneous job count (default: 2)

         -r, --recursive
                 Consider files in dirs recursively.
         --version
         -r, --recursive
                 Consider files in dirs recursively.
         --version


@@ -268,6 +270,32 @@ showUsage() {
       bkots foo.txt bar.pdf /home/username/Pictures/
 EOF
 } # Display information on how to use this script.
       bkots foo.txt bar.pdf /home/username/Pictures/
 EOF
 } # Display information on how to use this script.

+count_jobs() {
+    # Desc: Count and return total number of jobs
+    # Usage: count_jobs
+    # Input: None.
+    # Output: stdout   integer number of jobs
+    # Depends: Bash 5.1.16
+    # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+    # Version: 0.0.1
+
+    local job_count;
+    job_count="$(jobs -r | wc -l | tr -d ' ' )";
+    #yell "DEBUG:job_count:$job_count";
+    if [[ -z $job_count ]]; then job_count="0"; fi;
+    echo "$job_count";
+}; # Return number of background jobs
+wait_for_jobslot() {
+    # Desc: Does not return until count_jobs() falls below $max_job_count
+    # Input: var max_job_count
+    # Output: return code 0
+    # Depends: count_jobs(), yell();
+    while [[ $(count_jobs) -ge $max_job_count ]]; do
+        printf "\r%d:Working..." "$SECONDS";
+        sleep 1;
+    done;
+    return 0;
+};

 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"


@@ -302,6 +330,15 @@ processArgs() {
             --include-dotfiles) # Include dotfiles
                 option_include_dotfiles="true";
                 vbm "DEBUG:Option enabled:include dotfiles";;
             --include-dotfiles) # Include dotfiles
                 option_include_dotfiles="true";
                 vbm "DEBUG:Option enabled:include dotfiles";;

+            -j | --jobs) # Specify max_job_count
+                if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+                    max_job_count="$2";
+                    vbm "STATUS:Max job count set to:$max_job_count";
+                    shift;
+                else
+                    showUsage;
+                    die "FATAL:Invalid job count:$2";
+                fi;;

             -r | --recursive) # Specify recursive option
                 option_recursive="true";
                 vbm "DEBUG:option enabled:include files in dirs recursively";;
             -r | --recursive) # Specify recursive option
                 option_recursive="true";
                 vbm "DEBUG:option enabled:include files in dirs recursively";;


@@ -336,7 +373,7 @@ get_parent_dirnames() {
     # Input: arg1  input  path
     # Output: stdout   newline-delimited list of parent dirs
     # Version: 0.0.1
     # Input: arg1  input  path
     # Output: stdout   newline-delimited list of parent dirs
     # Version: 0.0.1

-    # Depends: yell(), die(), try()
+    # Depends: yell(), die(), must()

     local path
 
     # Check input
     local path
 
     # Check input


@@ -354,14 +391,6 @@ get_parent_dirnames() {
         echo "$name_base";
     done;    
 }; # Output parent dirnames to stdout
         echo "$name_base";
     done;    
 }; # Output parent dirnames to stdout

-cmdwrap() {
-    # print command to stderr
-    echo "$@" 1>&2;
-
-    # execute command
-    "$@";
-}; # print and execute string together
-export -f cmdwrap; # export cmdwrap for use in other functions

 main() {
     # Desc: Creates `.ots` file:
     #     - for each file specified in arrayPosArgs array
 main() {
     # Desc: Creates `.ots` file:
     #     - for each file specified in arrayPosArgs array


@@ -377,7 +406,6 @@ main() {
     # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
     #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
     #             [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377
     # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
     #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
     #             [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377

-    #             [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001

     local -a file_list file_list_pruned;
     local -a files_to_verify files_to_upgrade files_to_stamp
     local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned
     local -a file_list file_list_pruned;
     local -a files_to_verify files_to_upgrade files_to_stamp
     local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned


@@ -638,86 +666,108 @@ main() {
         vbm "DEBUG:files_to_stamp_pruned:";
         printf "%s\n" "${files_to_stamp_pruned[@]}";
     fi;
         vbm "DEBUG:files_to_stamp_pruned:";
         printf "%s\n" "${files_to_stamp_pruned[@]}";
     fi;

-    
+

     # Act on files
     # Act on files

-    ## Assemble upgrade file commands
+    ## Assemble and execute upgrade file commands

     for item in "${files_to_upgrade_pruned[@]}"; do
     for item in "${files_to_upgrade_pruned[@]}"; do

-        path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
-        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].
-        if [[ -z "$path_prf" ]]; then
-            yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
-            continue;
-        fi;
-        vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
-        if [[ ! $option_dry_run == "true" ]]; then
-            ### Try upgrade with known calendars in random order
-            while read -r url; do
-                vbm "DEBUG:Upgrading with calendar:url:$url";
-                if [[ "$opVerbose" = "true" ]]; then
-                    commands+=("cmdwrap ots -v -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break;
-                else
-                    commands+=("cmdwrap ots -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break;
-                fi;
-                #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
-            done < <(printf "%s\n" "${calendars[@]}" | shuf);
-        else
-            yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
-        fi;
+        wait_for_jobslot && {
+            path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
+            if [[ -z "$path_prf" ]]; then
+                yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
+                return 1; # would have been `continue` were it not in a subshell
+            fi;
+            vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
+            if [[ ! $option_dry_run == "true" ]]; then
+                ### Try upgrade with known calendars in random order
+                while read -r url; do
+                    vbm "DEBUG:Upgrading with calendar:url:$url";
+
+                    #### assemble command
+                    local -a cmd_temp;
+                    cmd_temp=("ots");
+                    if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                    cmd_temp+=("-l" "$url" "--no-default-whitelist");
+                    cmd_temp+=("upgrade" "$path_prf");
+                    if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                    #### execute command
+                    "${cmd_temp[@]}";
+                    unset cmd_temp;
+                    break;
+                    #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
+                done < <(printf "%s\n" "${calendars[@]}" | shuf);
+            else
+                yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
+            fi;
+        } &

     done;
 
     done;
 

-    ## Assemble verify file commands
+    ## Assemble and execute verify file commands

     for item in "${files_to_verify_pruned[@]}"; do
     for item in "${files_to_verify_pruned[@]}"; do

-        path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
-        path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
-        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].
-        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].
-        if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
-            yell "ERROR:blank verify item encountered. Skipping:item:$item";
-            continue;
-        fi;
-        vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
-        vbm "DEBUG:    against proof file:          path_prf:$path_prf";
-        if [[ ! $option_dry_run == "true" ]]; then
-            ### Try verify with known calendars in random order
-            while read -r url; do
-                vbm "DEBUG:Verifying with calendar:url:$url";
-                if [[ "$opVerbose" = "true" ]]; then
-                    commands+=("cmdwrap ots -v -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break;
-                else
-                    commands+=("cmdwrap ots -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break;
-                fi;
-                #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
-            done < <(printf "%s\n" "${calendars[@]}" | shuf);
-        else
-            yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
-        fi;
+        wait_for_jobslot && {
+            path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+            path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
+            if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
+                yell "ERROR:blank verify item encountered. Skipping:item:$item";
+                return 1; # would have been `continue` were it not in a subshell
+            fi;
+            vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
+            vbm "DEBUG:    against proof file:          path_prf:$path_prf";
+            if [[ ! $option_dry_run == "true" ]]; then
+                ### Try verify with known calendars in random order
+                while read -r url; do
+                    vbm "DEBUG:Verifying with calendar:url:$url";
+
+                    #### assemble command
+                    local -a cmd_temp;
+                    cmd_temp=("ots");
+                    if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                    cmd_temp+=("-l" "$url" "--no-default-whitelist");
+                    cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
+                    if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                    #### execute command
+                    "${cmd_temp[@]}";
+                    unset cmd_temp;
+                    break;
+                    #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
+                done < <(printf "%s\n" "${calendars[@]}" | shuf);
+            else
+                yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
+            fi;
+        } &

     done;
     done;

-    
-    ## Assemble stamp file commands
+
+    ## Assemble and execute stamp file commands

     for item in "${files_to_stamp_pruned[@]}"; do
     for item in "${files_to_stamp_pruned[@]}"; do

-        path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
-        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].
-        if [[ -z "$path_src" ]]; then
-            yell "ERROR:blank stamp item encountered. Skipping:item:$item";
-            continue;
-        fi;
-        vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
-        if [[ ! $option_dry_run == "true" ]]; then
-            if [[ "$opVerbose" = "true" ]]; then
-                commands+=("cmdwrap ots -v stamp \"$path_src_sesc\"");
+        wait_for_jobslot && {
+            path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+            if [[ -z "$path_src" ]]; then
+                yell "ERROR:blank stamp item encountered. Skipping:item:$item";
+                return 1; # would have been `continue` were it not in a subshell
+            fi;
+            vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
+            if [[ ! $option_dry_run == "true" ]]; then
+
+                #### assemble command
+                local -a cmd_temp;
+                cmd_temp=("ots");
+                if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                cmd_temp+=("stamp" "$path_src");
+                if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                #### execute command
+                "${cmd_temp[@]}";
+                unset cmd_temp;
+                #ots stamp "$path_src";

             else
             else

-                commands+=("cmdwrap ots stamp \"$path_src_sesc\"");
+                yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";

             fi;
             fi;

-            #ots stamp "$path_src";
-        else
-            yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";
-        fi;
+        } &

     done;
 
     done;
 

-    ## Run commands
-    #yell "DEBUG:commands:$(printf "%s\n" "${commands[@]}")";
-    printf "%s\n" "${commands[@]}" | parallel --group --jobs 25%;
-
+    ## Wait for jobs to finish.
+    wait;

 }; # main program
 
 # Run program
 }; # main program
 
 # Run program