update(user/bkots):Remove unnecessary must()

[BK-2020-03.git] / user / bkots

diff --git a/user/bkots b/user/bkots
old mode 100644 (file)
new mode 100755 (executable)
index d4df2ec..b38f0ac
--- a/user/bkots
+++ b/user/bkots
@@ -5,17 +5,18 @@ declare -Ag appRollCall # Associative array for storing app status
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status
 declare -ag arrayPosArgs # Associative array for processArgs() function
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status
 declare -ag arrayPosArgs # Associative array for processArgs() function

-declare -g ots_delay; ots_delay=1 # minimum time in seconds between ots operations

 declare -ag calendars;
 calendars+=("https://finney.calendar.eternitywall.com");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");
 declare -ag calendars;
 calendars+=("https://finney.calendar.eternitywall.com");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");

+age_threshold="60"; # min age to add file; seconds;
+max_job_count="2"; # default max job count

 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status

-try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
+must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails

 checkapp() {
     # Desc: If arg is a command, save result in assoc array 'appRollCall'
     # Usage: checkapp arg1 arg2 arg3 ...
 checkapp() {
     # Desc: If arg is a command, save result in assoc array 'appRollCall'
     # Usage: checkapp arg1 arg2 arg3 ...


@@ -199,7 +200,7 @@ showVersion() {
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'

-bkots 0.0.7
+bkots 2.0.1

 Copyright (C) 2022 Steven Baltakatei Sandoval
 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.
 Copyright (C) 2022 Steven Baltakatei Sandoval
 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.


@@ -238,6 +239,8 @@ showUsage() {
         --include-dotfiles
                 Include files and directories starting with '.' (not
                 included by default).
         --include-dotfiles
                 Include files and directories starting with '.' (not
                 included by default).

+        -j, --jobs
+                Specify simultaneous job count (default: 2)

         -r, --recursive
                 Consider files in dirs recursively.
         --version
         -r, --recursive
                 Consider files in dirs recursively.
         --version


@@ -260,11 +263,39 @@ showUsage() {
         themselves are also ignored by default
         (e.g. '/home/user/.gitconfig').
 
         themselves are also ignored by default
         (e.g. '/home/user/.gitconfig').
 

+        Files modified less than 1 minute ago are ignored.
+

     EXAMPLES:
       bkots -v foo.txt
       bkots foo.txt bar.pdf /home/username/Pictures/
 EOF
 } # Display information on how to use this script.
     EXAMPLES:
       bkots -v foo.txt
       bkots foo.txt bar.pdf /home/username/Pictures/
 EOF
 } # Display information on how to use this script.

+count_jobs() {
+    # Desc: Count and return total number of jobs
+    # Usage: count_jobs
+    # Input: None.
+    # Output: stdout   integer number of jobs
+    # Depends: Bash 5.1.16
+    # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+    # Version: 0.0.1
+
+    local job_count;
+    job_count="$(jobs -r | wc -l | tr -d ' ' )";
+    #yell "DEBUG:job_count:$job_count";
+    if [[ -z $job_count ]]; then job_count="0"; fi;
+    echo "$job_count";
+}; # Return number of background jobs
+wait_for_jobslot() {
+    # Desc: Does not return until count_jobs() falls below $max_job_count
+    # Input: var max_job_count
+    # Output: return code 0
+    # Depends: count_jobs(), yell();
+    while [[ $(count_jobs) -ge $max_job_count ]]; do
+        printf "\r%d:Working..." "$SECONDS";
+        sleep 1;
+    done;
+    return 0;
+};

 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"


@@ -299,6 +330,15 @@ processArgs() {
             --include-dotfiles) # Include dotfiles
                 option_include_dotfiles="true";
                 vbm "DEBUG:Option enabled:include dotfiles";;
             --include-dotfiles) # Include dotfiles
                 option_include_dotfiles="true";
                 vbm "DEBUG:Option enabled:include dotfiles";;

+            -j | --jobs) # Specify max_job_count
+                if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+                    max_job_count="$2";
+                    vbm "STATUS:Max job count set to:$max_job_count";
+                    shift;
+                else
+                    showUsage;
+                    die "FATAL:Invalid job count:$2";
+                fi;;

             -r | --recursive) # Specify recursive option
                 option_recursive="true";
                 vbm "DEBUG:option enabled:include files in dirs recursively";;
             -r | --recursive) # Specify recursive option
                 option_recursive="true";
                 vbm "DEBUG:option enabled:include files in dirs recursively";;


@@ -333,7 +373,7 @@ get_parent_dirnames() {
     # Input: arg1  input  path
     # Output: stdout   newline-delimited list of parent dirs
     # Version: 0.0.1
     # Input: arg1  input  path
     # Output: stdout   newline-delimited list of parent dirs
     # Version: 0.0.1

-    # Depends: yell(), die(), try()
+    # Depends: yell(), die(), must()

     local path
 
     # Check input
     local path
 
     # Check input


@@ -357,12 +397,16 @@ main() {
     #     - for each file in each dir specified in arrayPosArgs array
     #   Output file created alongside each file or in output directory specified by pathDirIn1
     # Usage: main "$@";
     #     - for each file in each dir specified in arrayPosArgs array
     #   Output file created alongside each file or in output directory specified by pathDirIn1
     # Usage: main "$@";

-    # Input: arrayPosArgs  array with positional arguments
-    #        pathDirOut1   path for output `.ots` files (if pathDirOut1 is specified and is a path)
+    # Input: arrayPosArgs   array with positional arguments
+    #        pathDirOut1    path for output `.ots` files (if pathDirOut1 is specified and is a path)
+    #        age_threshold  var: mininum age in seconds to timestamp file
+    

     # Output: file(s) creates `.ots` file alongside specified files
     # Output: file(s) creates `.ots` file alongside specified files

-    # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort)
+    # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822

     # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
     #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
     # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
     #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028

+    #             [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377
+    #             [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001

     local -a file_list file_list_pruned;
     local -a files_to_verify files_to_upgrade files_to_stamp
     local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned
     local -a file_list file_list_pruned;
     local -a files_to_verify files_to_upgrade files_to_stamp
     local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned


@@ -371,22 +415,18 @@ main() {
     processArgs "$@";
     
     # Check dependencies
     processArgs "$@";
     
     # Check dependencies

-    if ! checkapp ots find; then
+    if ! checkapp ots find parallel; then

         displayMissing;
         die "FATAL:Missing dependencies.";
     fi;
     
         displayMissing;
         die "FATAL:Missing dependencies.";
     fi;
     

-    # Check arguments    
-    ## Mark if output dir option specified
-    if [[ -v pathDirOut1 ]]; then
-        vbm "DEBUG:output directory specified:pathDirOut1:$pathDirOut1";
-        if [[ -d $pathDirOut1 ]]; then
-            vbm "DEBUG:pathDirOut1:$pathDirOut1";
-            config_output_dir="true";
-        else
-            die "ERROR:Not a dir:$pathDirOut1";
+    # Check arguments
+    for arg in "${arrayPosArgs[@]}"; do
+        arg="$(readlink -f "$arg")";
+        if ! { [[ -d $arg ]] || [[ -f $arg ]]; }; then
+            die "FATAL:Not a file or dir:arg:$arg";

         fi;
         fi;

-    fi;
+    done;

 
     # Display ots details
     vbm "$(type ots)"; # show how 'ots' is defined
 
     # Display ots details
     vbm "$(type ots)"; # show how 'ots' is defined


@@ -425,7 +465,7 @@ main() {
                 done < <(find "$item" -maxdepth 1 -type f);
             fi;
         else
                 done < <(find "$item" -maxdepth 1 -type f);
             fi;
         else

-            die "ERROR:Not a file or dir:item:$item";
+            die "FATAL:Not a file or dir:item:$item";

         fi;
     done;
     if [[ $opVerbose == "true" ]]; then
         fi;
     done;
     if [[ $opVerbose == "true" ]]; then


@@ -435,50 +475,65 @@ main() {
 
     # Prune file_list
     for item in "${file_list[@]}"; do
 
     # Prune file_list
     for item in "${file_list[@]}"; do

-        if ! [[ $option_include_dotfiles == "true" ]]; then
-            ## Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
-            unset flag_contains_dotfile_parent;
-            while read -r line; do
-                ### Check line from output of get_parent_dirnames
-                pattern="^\.";
-                if [[ $line =~ $pattern ]]; then
-                    #### line starts with '.'
-                    vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
-                    vbm "DEBUG:Dotfile in path:item:$item";
-                    vbm "DEBUG:Dotfile parent:line:$line";
-                    flag_contains_dotfile_parent="true";
-                    break
-                fi;
-            done < <(get_parent_dirnames "$item");
-            if [[ $flag_contains_dotfile_parent == "true" ]]; then
-                unset flag_contains_dotfile_parent;
-                continue; # skip to next item (i.e. don't add to file_list_pruned)
-            fi;
+        ## Ignore files that end in '.ots.bak'.
+        if [[ $item =~ '.ots.bak'$ ]]; then
+            vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item";
+            continue; # skip to next item
+        fi;

 
 

-            ## Ignore dotfiles themselves
-            item_basename="$(basename "$item")";
-            pattern="^\.";
-            if [[ $item_basename =~ $pattern ]]; then
-                vbm "INFO :Skipping dotfile:item:$item";
-                continue; # skip to next item
+        ## Ignore dotfiles
+        if ! [[ $option_include_dotfiles == "true" ]]; then
+            ### Ignore files if '/.' contained within canonical path
+            pattern="/\."; # a dot after a forward slash
+            if [[ $item =~ $pattern ]]; then
+                continue;

             fi;
             fi;

+            
+            # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
+            # unset flag_contains_dotfile_parent;
+            # while read -r line; do
+            #     #### Check line from output of get_parent_dirnames
+            #     pattern="^\.";
+            #     if [[ $line =~ $pattern ]]; then
+            #         ##### line starts with '.'
+            #         vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
+            #         vbm "DEBUG:Dotfile in path:item:$item";
+            #         vbm "DEBUG:Dotfile parent:line:$line";
+            #         flag_contains_dotfile_parent="true";
+            #         break
+            #     fi;
+            # done < <(get_parent_dirnames "$item");
+            # if [[ $flag_contains_dotfile_parent == "true" ]]; then
+            #     unset flag_contains_dotfile_parent;
+            #     continue; # skip to next item (i.e. don't add to file_list_pruned)
+            # fi;
+
+            # ### Ignore dotfiles themselves
+            # item_basename="$(basename "$item")";
+            # pattern="^\.";
+            # if [[ $item_basename =~ $pattern ]]; then
+            #     vbm "INFO :Skipping dotfile:item:$item";
+            #     continue; # skip to next item
+            # fi;

         fi;
         fi;

-
+        

         ## Ignore files with newlines present in filename. See [2].
         if [[ $item =~ $'\n' ]]; then
         ## Ignore files with newlines present in filename. See [2].
         if [[ $item =~ $'\n' ]]; then

-            yell "INFO :Skipping file name with newline:$item";
+            vbm "DEBUG:Skipping file name with newline:$item";

             continue; # skip to next item
         fi;
 
         ## Ignore files that end in '~'.
         if [[ $item =~ ~$ ]]; then
             continue; # skip to next item
         fi;
 
         ## Ignore files that end in '~'.
         if [[ $item =~ ~$ ]]; then

-            yell "INFO :Skipping file ending in tilde:$item";
+            vbm "DEBUG:Skipping file ending in tilde:$item";

             continue; # skip to next item
         fi;
 
             continue; # skip to next item
         fi;
 

-        ## Ignore files that end in '.ots.bak'.
-        if [[ $item =~ '.ots.bak'$ ]]; then
-            yell "INFO :Skipping file ending in '.ots.bak':item:$item";
+        ## Ignore files modified less than age_threshold. See [3].
+        time_now="$(date +%s)"; # epoch seconds
+        item_age="$(stat -c %Y "$item")"; # age in seconds
+        if [[ $(( time_now - item_age )) -lt "$age_threshold" ]]; then
+            yell "INFO :Skipping file modified less than $age_threshold seconds ago:item:$item";

             continue; # skip to next item
         fi;
         
             continue; # skip to next item
         fi;
         


@@ -614,9 +669,10 @@ main() {
     fi;
     
     # Act on files
     fi;
     
     # Act on files

-    ## Upgrade files
+    ## Assemble and execute upgrade file commands

     for item in "${files_to_upgrade_pruned[@]}"; do
         path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
     for item in "${files_to_upgrade_pruned[@]}"; do
         path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";

+        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].

         if [[ -z "$path_prf" ]]; then
             yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
             continue;
         if [[ -z "$path_prf" ]]; then
             yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
             continue;


@@ -626,18 +682,32 @@ main() {
             ### Try upgrade with known calendars in random order
             while read -r url; do
                 vbm "DEBUG:Upgrading with calendar:url:$url";
             ### Try upgrade with known calendars in random order
             while read -r url; do
                 vbm "DEBUG:Upgrading with calendar:url:$url";

-                ots -l "$url" --no-default-whitelist upgrade "\"$path_prf\"" && break;
+
+                #### assemble command
+                local -a cmd_temp;
+                cmd_temp+=("ots");
+                if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                cmd_temp+=("-l" "$url" "--no-default-whitelist");
+                cmd_temp+=("upgrade" "$path_prf_sesc");
+                if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                #### execute command
+                wait_for_jobslot && "${cmd_temp[@]}" &
+                unset cmd_temp;
+                break;
+                #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;

             done < <(printf "%s\n" "${calendars[@]}" | shuf);
         else
             done < <(printf "%s\n" "${calendars[@]}" | shuf);
         else

-            yell "DEBUG:DRY RUN:Not running:\"ots upgrade \"$path_prf\"\"";
+            yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";

         fi;
         fi;

-        #sleep "$ots_delay";

     done;
 
     done;
 

-    ## Verify files
+    ## Assemble and execute verify file commands

     for item in "${files_to_verify_pruned[@]}"; do
         path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
         path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
     for item in "${files_to_verify_pruned[@]}"; do
         path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
         path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";

+        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].
+        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].

         if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
             yell "ERROR:blank verify item encountered. Skipping:item:$item";
             continue;
         if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
             yell "ERROR:blank verify item encountered. Skipping:item:$item";
             continue;


@@ -648,31 +718,54 @@ main() {
             ### Try verify with known calendars in random order
             while read -r url; do
                 vbm "DEBUG:Verifying with calendar:url:$url";
             ### Try verify with known calendars in random order
             while read -r url; do
                 vbm "DEBUG:Verifying with calendar:url:$url";

-                ots -l "$url" --no-default-whitelist verify -f "\"$path_src\"" "\"$path_prf\"" && break;
+
+                #### assemble command
+                local -a cmd_temp;
+                cmd_temp+=("ots");
+                if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                cmd_temp+=("-l" "$url" "--no-default-whitelist");
+                cmd_temp+=("verify" "-f" "$path_src_sesc" "$path_prf_sesc");
+                if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                #### execute command
+                wait_for_jobslot && "${cmd_temp[@]}" &
+                unset cmd_temp;
+                break;
+                #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;

             done < <(printf "%s\n" "${calendars[@]}" | shuf);
         else
             done < <(printf "%s\n" "${calendars[@]}" | shuf);
         else

-            yell "DEBUG:DRY RUN:Not running:\"ots verify -f \"$path_src\" \"$path_prf\"\"";
+            yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";

         fi;
         fi;

-        #sleep "$ots_delay";

     done;
     
     done;
     

-    ## Stamp files
+    ## Assemble and execute stamp file commands

     for item in "${files_to_stamp_pruned[@]}"; do
         path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
     for item in "${files_to_stamp_pruned[@]}"; do
         path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";

+        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].

         if [[ -z "$path_src" ]]; then
             yell "ERROR:blank stamp item encountered. Skipping:item:$item";
             continue;
         fi;
         vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
         if [[ ! $option_dry_run == "true" ]]; then
         if [[ -z "$path_src" ]]; then
             yell "ERROR:blank stamp item encountered. Skipping:item:$item";
             continue;
         fi;
         vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
         if [[ ! $option_dry_run == "true" ]]; then

-            ots stamp "\"$path_src\"";
+
+            #### assemble command
+            local -a cmd_temp;
+            cmd_temp+=("ots");
+            if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+            cmd_temp+=("stamp" "$path_src_sesc");
+            if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+            #### execute command
+            wait_for_jobslot && "${cmd_temp[@]}" &
+            unset cmd_temp;
+            #ots stamp "$path_src";

         else
         else

-            yell "DEBUG:DRY RUN:Not running:\"ots stamp \"$path_src\"\"";
+            yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";

         fi;
         fi;

-        sleep "$ots_delay";

     done;
     done;

-

 }; # main program
 
 # Run program
 main "$@";
 }; # main program
 
 # Run program
 main "$@";

+exit 0;