feat(user/ping_offline.sh):Add timestamps before each ping line

[BK-2020-03.git] / user / bkots

diff --git a/user/bkots b/user/bkots
index 8d6ecde4471a668c3c9ecbf8189bed654022e86a..acea7f51e20b8daf00aaeec64fd904bf1475551b 100755 (executable)
--- a/user/bkots
+++ b/user/bkots
@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 # Desc: Recursively timestamp files with OpenTimestamp
 # Usage: bkots [PATH]
 #!/usr/bin/env bash
 # Desc: Recursively timestamp files with OpenTimestamp
 # Usage: bkots [PATH]

-# Version: 0.2.0
+# Version: 3.0.0

 
 # Define variables
 declare -g max_job_count="2"; # default max job count
 
 # Define variables
 declare -g max_job_count="2"; # default max job count

+declare -g max_batch_size="500"; # default xargs batch size for upgrading and stamping

 declare -g age_threshold="60"; # min age to add file; seconds;
 declare -g stamp_throttle="0.1"; # seconds delay between stamps
 declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors
 declare -g age_threshold="60"; # min age to add file; seconds;
 declare -g stamp_throttle="0.1"; # seconds delay between stamps
 declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors


@@ -17,6 +18,7 @@ calendars+=("https://finney.calendar.eternitywall.com");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");
 calendars+=("https://btc.calendar.catallaxy.com");
 calendars+=("https://alice.btc.calendar.opentimestamps.org");
 calendars+=("https://bob.btc.calendar.opentimestamps.org");

+export OTS_CALS; OTS_CALS="$(printf '%s\n' "${calendars[@]}")";  # convert calendars array into string list

 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 
 # Declare functions
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr


@@ -205,8 +207,8 @@ showVersion() {
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'
     vbm "DEBUG:showVersion function called."
 
     cat <<'EOF'

-bkots 2.1.1
-Copyright (C) 2022 Steven Baltakatei Sandoval
+bkots 3.0.0
+Copyright (C) 2026 Steven Baltakatei Sandoval

 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.
 There is NO WARRANTY, to the extent permitted by law.
 License GPLv3: GNU GPL version 3
 This is free software; you are free to change and redistribute it.
 There is NO WARRANTY, to the extent permitted by law.


@@ -246,8 +248,12 @@ showUsage() {
                 included by default).
         -j, --jobs
                 Specify simultaneous job count (default: 2)
                 included by default).
         -j, --jobs
                 Specify simultaneous job count (default: 2)

+        -n, --batch-size
+                Specify file batch sizes for upgrade and stamp operations (default:500)

         -r, --recursive
                 Consider files in dirs recursively.
         -r, --recursive
                 Consider files in dirs recursively.

+        --verify
+                Verify OTS files.

         --version
                 Display script version.
         -v, --verbose
         --version
                 Display script version.
         -v, --verbose


@@ -344,9 +350,21 @@ processArgs() {
                     showUsage;
                     die "FATAL:Invalid job count:$2";
                 fi;;
                     showUsage;
                     die "FATAL:Invalid job count:$2";
                 fi;;

+            -n | --batch-size)
+                if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+                    max_batch_size="$2";
+                    vbm "STATUS:Max batch size set to:$max_batch_size";
+                    shift;
+                else
+                    showUsage;
+                    die "FATAL:Invalid xargs file batch size:$2";
+                fi;;

             -r | --recursive) # Specify recursive option
                 option_recursive="true";
             -r | --recursive) # Specify recursive option
                 option_recursive="true";

-                vbm "DEBUG:option enabled:include files in dirs recursively";;
+                vbm "DEBUG:Option enabled:include files in dirs recursively";;
+            --verify)
+                option_verify="true";
+                vbm "DEBUG:Option enabled:verify OTS files";;

            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
             --) # End of all options. See [2].
            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
             --) # End of all options. See [2].


@@ -644,6 +662,7 @@ main() {
     ## Sort and prune file action arrays
     ### files to upgrade
     while read -r -d $'\0' line; do
     ## Sort and prune file action arrays
     ### files to upgrade
     while read -r -d $'\0' line; do

+        if [[ -z $line ]]; then continue; fi; # Skip empty lines

         vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line";
         files_to_upgrade_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1]
         vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line";
         files_to_upgrade_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1]


@@ -654,6 +673,7 @@ main() {
 
     ### files to verify
     while read -r -d $'\0' line; do
 
     ### files to verify
     while read -r -d $'\0' line; do

+        if [[ -z $line ]]; then continue; fi; # Skip empty lines

         vbm "DEBUG:adding to files_to_verify_pruned:line:$line";
         files_to_verify_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1]
         vbm "DEBUG:adding to files_to_verify_pruned:line:$line";
         files_to_verify_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1]


@@ -664,6 +684,7 @@ main() {
 
     ### files to stamp
     while read -r -d $'\0' line; do
 
     ### files to stamp
     while read -r -d $'\0' line; do

+        if [[ -z $line ]]; then continue; fi; # Skip empty lines

         vbm "DEBUG:adding to files_to_stamp_pruned:line:$line";
         files_to_stamp_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1]
         vbm "DEBUG:adding to files_to_stamp_pruned:line:$line";
         files_to_stamp_pruned+=("$line");
     done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1]


@@ -674,111 +695,90 @@ main() {
 
     # Act on files
     ## Assemble and execute upgrade file commands
 
     # Act on files
     ## Assemble and execute upgrade file commands

-    for item in "${files_to_upgrade_pruned[@]}"; do
-        wait_for_jobslot && {
-            path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
-            if [[ -z "$path_prf" ]]; then
-                yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
-                return 1; # would have been `continue` were it not in a subshell
-            fi;
-            vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
-            if [[ ! $option_dry_run == "true" ]]; then
-                ### Try upgrade with known calendars in random order
-                while read -r url; do
-                    vbm "DEBUG:Upgrading with calendar:url:$url";
-
-                    #### assemble command
-                    local -a cmd_temp;
-                    cmd_temp=("ots");
-                    if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
-                    cmd_temp+=("-l" "$url" "--no-default-whitelist");
-                    cmd_temp+=("upgrade" "$path_prf");
-                    if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
-                    #### execute command
-                    "${cmd_temp[@]}";
-                    unset cmd_temp;
-                    break;
-                    #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
-                done < <(printf "%s\n" "${calendars[@]}" | shuf);
-            else
-                yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
-            fi;
-        } &
-    done;
-
-    ## Assemble and execute verify file commands
-    for item in "${files_to_verify_pruned[@]}"; do
-        wait_for_jobslot && {
-            path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
-            path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
-            if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
-                yell "ERROR:blank verify item encountered. Skipping:item:$item";
-                return 1; # would have been `continue` were it not in a subshell
-            fi;
-            vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
-            vbm "DEBUG:    against proof file:          path_prf:$path_prf";
-            if [[ ! $option_dry_run == "true" ]]; then
-                ### Try verify with known calendars in random order
-                while read -r url; do
-                    vbm "DEBUG:Verifying with calendar:url:$url";
-
-                    #### assemble command
-                    local -a cmd_temp;
-                    cmd_temp=("ots");
-                    if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
-                    cmd_temp+=("-l" "$url" "--no-default-whitelist");
-                    cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
-                    if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
-                    #### execute command
-                    "${cmd_temp[@]}";
-                    unset cmd_temp;
-                    break;
-                    #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
-                done < <(printf "%s\n" "${calendars[@]}" | shuf);
-            else
-                yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
-            fi;
-        } &
-    done;
+    if [[ ${#files_to_upgrade_pruned[@]} -gt 0 ]]; then
+        if [[ $option_dry_run == "true" ]]; then
+            yell "DEBUG:DRY RUN:Not running:\"ots upgrade\"";
+        else
+            ### Pick random calendar
+            function get_calurl() {
+                # Desc: Returns random calendar url
+                # Input:  string  OTS_CALS   newline-delimited list of calendars
+                # Output: stdout             single calendar url
+                printf "%s" "$OTS_CALS" | shuf -n1;
+            }; # print random calendar url to stdout
+            export -f get_calurl;
+
+            ### Perform upgrade command in xargs batches
+            printf '%s\0' "${files_to_upgrade_pruned[@]}" \
+                | shuf -z \
+                | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots --no-default-whitelist -l "$(get_calurl)" upgrade "$@";' _ ; 
+        fi;
+    fi;

 
 

-    ## Assemble and execute stamp file commands
-    for item in "${files_to_stamp_pruned[@]}"; do
-        wait_for_jobslot && {
-            path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
-            if [[ -z "$path_src" ]]; then
-                yell "ERROR:blank stamp item encountered. Skipping:item:$item";
-                return 1; # would have been `continue` were it not in a subshell
-            fi;
-            vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
-            if [[ ! $option_dry_run == "true" ]]; then
-
-                #### assemble command
-                local -a cmd_temp;
-                cmd_temp=("ots");
-                if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
-                cmd_temp+=("stamp" "$path_src");
-                if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
-                #### execute command
-                if "${cmd_temp[@]}"; then
-                    sleep "$stamp_throttle" || die "FATAL:Invalid stamp throttle.";
+    # ## Assemble and execute verify file commands
+    if [[ $option_verify == "true" ]]; then
+        for item in "${files_to_verify_pruned[@]}"; do
+            wait_for_jobslot && {
+                path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+                path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
+                if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
+                    yell "ERROR:blank verify item encountered. Skipping:item:$item";
+                    return 1; # would have been `continue` were it not in a subshell
+                fi;
+                vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
+                vbm "DEBUG:    against proof file:          path_prf:$path_prf";
+                if [[ ! $option_dry_run == "true" ]]; then
+                    ### Try verify with known calendars in random order
+                    while read -r url; do
+                        vbm "DEBUG:Verifying with calendar:url:$url";
+
+                        #### assemble command
+                        local -a cmd_temp;
+                        cmd_temp=("ots");
+                        if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+                        cmd_temp+=("-l" "$url" "--no-default-whitelist");
+                        cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
+                        if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+                        #### execute command
+                        "${cmd_temp[@]}";
+                        unset cmd_temp;
+                        break;
+                        #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
+                    done < <(printf "%s\n" "${calendars[@]}" | shuf);

                 else
                 else

-                    yell "ERROR:Could not stamp file with command:$(declare -p cmd_temp)";
-                    sleep "$stamp_throttle_fail" || die "FATAL:Invalid stamp throttle.";
+                    yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";

                 fi;
                 fi;

-                
-                unset cmd_temp;
-                #ots stamp "$path_src";
-            else
-                yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";
-            fi;
-        } &
-    done;
+            } &
+        done;
+
+        ## Wait for jobs to finish.
+        wait;
+    else
+        vbm "DEBUG:Skipping operation:ots verify.";
+    fi;
+    
+    ## Assemble and execute stamp file commands
+    if [[ ${#files_to_stamp_pruned[@]} -gt 0 ]]; then
+        if [[ $option_dry_run == "true" ]]; then
+            yell "DEBUG:DRY RUN:Not running:\"ots stamp\"";
+        else
+            function get_calurl() {
+                # Desc: Returns random calendar url
+                # Input:  string  OTS_CALS   newline-delimited list of calendars
+                # Output: stdout             single calendar url
+                printf "%s" "$OTS_CALS" | shuf -n1;
+            }; # print random calendar url to stdout
+            export -f get_calurl;
+
+            ### Perform stamp command in xargs batches
+            printf '%s\0' "${files_to_stamp_pruned[@]}" \
+                | shuf -z \
+                | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots stamp "$@";' _ ;
+        fi;
+    fi;
+

 
 

-    ## Wait for jobs to finish.
-    wait;

 }; # main program
 
 # Run program
 }; # main program
 
 # Run program