#!/usr/bin/env bash
+# Desc: Recursively timestamp files with OpenTimestamp
+# Usage: bkots [PATH]
+# Version: 3.0.0
# Define variables
+declare -g max_job_count="2"; # default max job count
+declare -g max_batch_size="500"; # default xargs batch size for upgrading and stamping
+declare -g age_threshold="60"; # min age to add file; seconds;
+declare -g stamp_throttle="0.1"; # seconds delay between stamps
+declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors
declare -Ag appRollCall # Associative array for storing app status
declare -Ag fileRollCall # Associative array for storing file status
declare -Ag dirRollCall # Associative array for storing dir status
calendars+=("https://btc.calendar.catallaxy.com");
calendars+=("https://alice.btc.calendar.opentimestamps.org");
calendars+=("https://bob.btc.calendar.opentimestamps.org");
-age_threshold="60"; # min age to add file; seconds;
-max_job_count="2"; # default max job count
+export OTS_CALS; OTS_CALS="$(printf '%s\n' "${calendars[@]}")"; # convert calendars array into string list
# Declare functions
yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
vbm "DEBUG:showVersion function called."
cat <<'EOF'
-bkots 2.0.2
-Copyright (C) 2022 Steven Baltakatei Sandoval
+bkots 3.0.0
+Copyright (C) 2026 Steven Baltakatei Sandoval
License GPLv3: GNU GPL version 3
This is free software; you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
included by default).
-j, --jobs
Specify simultaneous job count (default: 2)
+ -n, --batch-size
+ Specify file batch sizes for upgrade and stamp operations (default:500)
-r, --recursive
Consider files in dirs recursively.
+ --verify
+ Verify OTS files.
--version
Display script version.
-v, --verbose
showUsage;
die "FATAL:Invalid job count:$2";
fi;;
+ -n | --batch-size)
+ if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+ max_batch_size="$2";
+ vbm "STATUS:Max batch size set to:$max_batch_size";
+ shift;
+ else
+ showUsage;
+ die "FATAL:Invalid xargs file batch size:$2";
+ fi;;
-r | --recursive) # Specify recursive option
option_recursive="true";
- vbm "DEBUG:option enabled:include files in dirs recursively";;
+ vbm "DEBUG:Option enabled:include files in dirs recursively";;
+ --verify)
+ option_verify="true";
+ vbm "DEBUG:Option enabled:verify OTS files";;
--version) showVersion; exit 1;; # Show version
-v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
--) # End of all options. See [2].
## Sort and prune file action arrays
### files to upgrade
while read -r -d $'\0' line; do
+ if [[ -z $line ]]; then continue; fi; # Skip empty lines
vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line";
files_to_upgrade_pruned+=("$line");
done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1]
### files to verify
while read -r -d $'\0' line; do
+ if [[ -z $line ]]; then continue; fi; # Skip empty lines
vbm "DEBUG:adding to files_to_verify_pruned:line:$line";
files_to_verify_pruned+=("$line");
done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1]
### files to stamp
while read -r -d $'\0' line; do
+ if [[ -z $line ]]; then continue; fi; # Skip empty lines
vbm "DEBUG:adding to files_to_stamp_pruned:line:$line";
files_to_stamp_pruned+=("$line");
done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1]
vbm "DEBUG:files_to_stamp_pruned:";
printf "%s\n" "${files_to_stamp_pruned[@]}";
fi;
-
+
# Act on files
## Assemble and execute upgrade file commands
- for item in "${files_to_upgrade_pruned[@]}"; do
- path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
- if [[ -z "$path_prf" ]]; then
- yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
- if [[ ! $option_dry_run == "true" ]]; then
- ### Try upgrade with known calendars in random order
- while read -r url; do
- vbm "DEBUG:Upgrading with calendar:url:$url";
-
- #### assemble command
- local -a cmd_temp;
- cmd_temp+=("ots");
- if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
- cmd_temp+=("-l" "$url" "--no-default-whitelist");
- cmd_temp+=("upgrade" "$path_prf");
- if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
- #### execute command
- wait_for_jobslot && "${cmd_temp[@]}" &
- unset cmd_temp;
- break;
- #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
- done < <(printf "%s\n" "${calendars[@]}" | shuf);
+ if [[ ${#files_to_upgrade_pruned[@]} -gt 0 ]]; then
+ if [[ $option_dry_run == "true" ]]; then
+ yell "DEBUG:DRY RUN:Not running:\"ots upgrade\"";
else
- yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
+ ### Pick random calendar
+ function get_calurl() {
+ # Desc: Returns random calendar url
+ # Input: string OTS_CALS newline-delimited list of calendars
+ # Output: stdout single calendar url
+ printf "%s" "$OTS_CALS" | shuf -n1;
+ }; # print random calendar url to stdout
+ export -f get_calurl;
+
+ ### Perform upgrade command in xargs batches
+ printf '%s\0' "${files_to_upgrade_pruned[@]}" \
+ | shuf -z \
+ | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots --no-default-whitelist -l "$(get_calurl)" upgrade "$@";' _ ;
fi;
- done;
+ fi;
- ## Assemble and execute verify file commands
- for item in "${files_to_verify_pruned[@]}"; do
- path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
- path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
- if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
- yell "ERROR:blank verify item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
- vbm "DEBUG: against proof file: path_prf:$path_prf";
- if [[ ! $option_dry_run == "true" ]]; then
- ### Try verify with known calendars in random order
- while read -r url; do
- vbm "DEBUG:Verifying with calendar:url:$url";
-
- #### assemble command
- local -a cmd_temp;
- cmd_temp+=("ots");
- if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
- cmd_temp+=("-l" "$url" "--no-default-whitelist");
- cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
- if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
- #### execute command
- wait_for_jobslot && "${cmd_temp[@]}" &
- unset cmd_temp;
- break;
- #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
- done < <(printf "%s\n" "${calendars[@]}" | shuf);
- else
- yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
- fi;
- done;
+ # ## Assemble and execute verify file commands
+ if [[ $option_verify == "true" ]]; then
+ for item in "${files_to_verify_pruned[@]}"; do
+ wait_for_jobslot && {
+ path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+ path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
+ if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
+ yell "ERROR:blank verify item encountered. Skipping:item:$item";
+ return 1; # would have been `continue` were it not in a subshell
+ fi;
+ vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
+ vbm "DEBUG: against proof file: path_prf:$path_prf";
+ if [[ ! $option_dry_run == "true" ]]; then
+ ### Try verify with known calendars in random order
+ while read -r url; do
+ vbm "DEBUG:Verifying with calendar:url:$url";
+
+ #### assemble command
+ local -a cmd_temp;
+ cmd_temp=("ots");
+ if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+ cmd_temp+=("-l" "$url" "--no-default-whitelist");
+ cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
+ if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+ #### execute command
+ "${cmd_temp[@]}";
+ unset cmd_temp;
+ break;
+ #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
+ done < <(printf "%s\n" "${calendars[@]}" | shuf);
+ else
+ yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
+ fi;
+ } &
+ done;
+
+ ## Wait for jobs to finish.
+ wait;
+ else
+ vbm "DEBUG:Skipping operation:ots verify.";
+ fi;
## Assemble and execute stamp file commands
- for item in "${files_to_stamp_pruned[@]}"; do
- path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
- if [[ -z "$path_src" ]]; then
- yell "ERROR:blank stamp item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
- if [[ ! $option_dry_run == "true" ]]; then
-
- #### assemble command
- local -a cmd_temp;
- cmd_temp+=("ots");
- if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
- cmd_temp+=("stamp" "$path_src");
- if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
-
- #### execute command
- wait_for_jobslot && "${cmd_temp[@]}" &
- unset cmd_temp;
- #ots stamp "$path_src";
+ if [[ ${#files_to_stamp_pruned[@]} -gt 0 ]]; then
+ if [[ $option_dry_run == "true" ]]; then
+ yell "DEBUG:DRY RUN:Not running:\"ots stamp\"";
else
- yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";
+ function get_calurl() {
+ # Desc: Returns random calendar url
+ # Input: string OTS_CALS newline-delimited list of calendars
+ # Output: stdout single calendar url
+ printf "%s" "$OTS_CALS" | shuf -n1;
+ }; # print random calendar url to stdout
+ export -f get_calurl;
+
+ ### Perform stamp command in xargs batches
+ printf '%s\0' "${files_to_stamp_pruned[@]}" \
+ | shuf -z \
+ | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots stamp "$@";' _ ;
fi;
- done;
+ fi;
+
+
}; # main program
# Run program
projects / BK-2020-03.git / blobdiff