X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/blobdiff_plain/7eb0a0908207b32b896df11cb4cc9a5410dae9ab..105fa8b43670a1e5d7e21054f25cd7c318f0551e:/user/bkots?ds=sidebyside diff --git a/user/bkots b/user/bkots index 8d6ecde..acea7f5 100755 --- a/user/bkots +++ b/user/bkots @@ -1,10 +1,11 @@ #!/usr/bin/env bash # Desc: Recursively timestamp files with OpenTimestamp # Usage: bkots [PATH] -# Version: 0.2.0 +# Version: 3.0.0 # Define variables declare -g max_job_count="2"; # default max job count +declare -g max_batch_size="500"; # default xargs batch size for upgrading and stamping declare -g age_threshold="60"; # min age to add file; seconds; declare -g stamp_throttle="0.1"; # seconds delay between stamps declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors @@ -17,6 +18,7 @@ calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); +export OTS_CALS; OTS_CALS="$(printf '%s\n' "${calendars[@]}")"; # convert calendars array into string list # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr @@ -205,8 +207,8 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.1.1 -Copyright (C) 2022 Steven Baltakatei Sandoval +bkots 3.0.0 +Copyright (C) 2026 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. @@ -246,8 +248,12 @@ showUsage() { included by default). -j, --jobs Specify simultaneous job count (default: 2) + -n, --batch-size + Specify file batch sizes for upgrade and stamp operations (default:500) -r, --recursive Consider files in dirs recursively. + --verify + Verify OTS files. --version Display script version. -v, --verbose @@ -344,9 +350,21 @@ processArgs() { showUsage; die "FATAL:Invalid job count:$2"; fi;; + -n | --batch-size) + if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then + max_batch_size="$2"; + vbm "STATUS:Max batch size set to:$max_batch_size"; + shift; + else + showUsage; + die "FATAL:Invalid xargs file batch size:$2"; + fi;; -r | --recursive) # Specify recursive option option_recursive="true"; - vbm "DEBUG:option enabled:include files in dirs recursively";; + vbm "DEBUG:Option enabled:include files in dirs recursively";; + --verify) + option_verify="true"; + vbm "DEBUG:Option enabled:verify OTS files";; --version) showVersion; exit 1;; # Show version -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1]. --) # End of all options. See [2]. @@ -644,6 +662,7 @@ main() { ## Sort and prune file action arrays ### files to upgrade while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line"; files_to_upgrade_pruned+=("$line"); done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1] @@ -654,6 +673,7 @@ main() { ### files to verify while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_verify_pruned:line:$line"; files_to_verify_pruned+=("$line"); done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1] @@ -664,6 +684,7 @@ main() { ### files to stamp while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_stamp_pruned:line:$line"; files_to_stamp_pruned+=("$line"); done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1] @@ -674,111 +695,90 @@ main() { # Act on files ## Assemble and execute upgrade file commands - for item in "${files_to_upgrade_pruned[@]}"; do - wait_for_jobslot && { - path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_prf" ]]; then - yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - return 1; # would have been `continue` were it not in a subshell - fi; - vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try upgrade with known calendars in random order - while read -r url; do - vbm "DEBUG:Upgrading with calendar:url:$url"; - - #### assemble command - local -a cmd_temp; - cmd_temp=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("upgrade" "$path_prf"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - "${cmd_temp[@]}"; - unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; - fi; - } & - done; - - ## Assemble and execute verify file commands - for item in "${files_to_verify_pruned[@]}"; do - wait_for_jobslot && { - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then - yell "ERROR:blank verify item encountered. Skipping:item:$item"; - return 1; # would have been `continue` were it not in a subshell - fi; - vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; - vbm "DEBUG: against proof file: path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try verify with known calendars in random order - while read -r url; do - vbm "DEBUG:Verifying with calendar:url:$url"; - - #### assemble command - local -a cmd_temp; - cmd_temp=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - "${cmd_temp[@]}"; - unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; - fi; - } & - done; + if [[ ${#files_to_upgrade_pruned[@]} -gt 0 ]]; then + if [[ $option_dry_run == "true" ]]; then + yell "DEBUG:DRY RUN:Not running:\"ots upgrade\""; + else + ### Pick random calendar + function get_calurl() { + # Desc: Returns random calendar url + # Input: string OTS_CALS newline-delimited list of calendars + # Output: stdout single calendar url + printf "%s" "$OTS_CALS" | shuf -n1; + }; # print random calendar url to stdout + export -f get_calurl; + + ### Perform upgrade command in xargs batches + printf '%s\0' "${files_to_upgrade_pruned[@]}" \ + | shuf -z \ + | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots --no-default-whitelist -l "$(get_calurl)" upgrade "$@";' _ ; + fi; + fi; - ## Assemble and execute stamp file commands - for item in "${files_to_stamp_pruned[@]}"; do - wait_for_jobslot && { - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_src" ]]; then - yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - return 1; # would have been `continue` were it not in a subshell - fi; - vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; - if [[ ! $option_dry_run == "true" ]]; then - - #### assemble command - local -a cmd_temp; - cmd_temp=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("stamp" "$path_src"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - if "${cmd_temp[@]}"; then - sleep "$stamp_throttle" || die "FATAL:Invalid stamp throttle."; + # ## Assemble and execute verify file commands + if [[ $option_verify == "true" ]]; then + for item in "${files_to_verify_pruned[@]}"; do + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then + yell "ERROR:blank verify item encountered. Skipping:item:$item"; + return 1; # would have been `continue` were it not in a subshell + fi; + vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; + vbm "DEBUG: against proof file: path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try verify with known calendars in random order + while read -r url; do + vbm "DEBUG:Verifying with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); else - yell "ERROR:Could not stamp file with command:$(declare -p cmd_temp)"; - sleep "$stamp_throttle_fail" || die "FATAL:Invalid stamp throttle."; + yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; fi; - - unset cmd_temp; - #ots stamp "$path_src"; - else - yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; - fi; - } & - done; + } & + done; + + ## Wait for jobs to finish. + wait; + else + vbm "DEBUG:Skipping operation:ots verify."; + fi; + + ## Assemble and execute stamp file commands + if [[ ${#files_to_stamp_pruned[@]} -gt 0 ]]; then + if [[ $option_dry_run == "true" ]]; then + yell "DEBUG:DRY RUN:Not running:\"ots stamp\""; + else + function get_calurl() { + # Desc: Returns random calendar url + # Input: string OTS_CALS newline-delimited list of calendars + # Output: stdout single calendar url + printf "%s" "$OTS_CALS" | shuf -n1; + }; # print random calendar url to stdout + export -f get_calurl; + + ### Perform stamp command in xargs batches + printf '%s\0' "${files_to_stamp_pruned[@]}" \ + | shuf -z \ + | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots stamp "$@";' _ ; + fi; + fi; + - ## Wait for jobs to finish. - wait; }; # main program # Run program