user/bkots

   1 #!/usr/bin/env bash
   2 # Desc: Recursively timestamp files with OpenTimestamp
   3 # Usage: bkots [PATH]
   4 # Version: 3.0.0
   5
   6 # Define variables
   7 declare -g max_job_count="2"; # default max job count
   8 declare -g max_batch_size="500"; # default xargs batch size for upgrading and stamping
   9 declare -g age_threshold="60"; # min age to add file; seconds;
  10 declare -g stamp_throttle="0.1"; # seconds delay between stamps
  11 declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors
  12 declare -Ag appRollCall # Associative array for storing app status
  13 declare -Ag fileRollCall # Associative array for storing file status
  14 declare -Ag dirRollCall # Associative array for storing dir status
  15 declare -ag arrayPosArgs # Associative array for processArgs() function
  16 declare -ag calendars;
  17 calendars+=("https://finney.calendar.eternitywall.com");
  18 calendars+=("https://btc.calendar.catallaxy.com");
  19 calendars+=("https://alice.btc.calendar.opentimestamps.org");
  20 calendars+=("https://bob.btc.calendar.opentimestamps.org");
  21 export OTS_CALS; OTS_CALS="$(printf '%s\n' "${calendars[@]}")";  # convert calendars array into string list
  22
  23 # Declare functions
  24 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
  25 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
  26 must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
  27 checkapp() {
  28     # Desc: If arg is a command, save result in assoc array 'appRollCall'
  29     # Usage: checkapp arg1 arg2 arg3 ...
  30     # Version: 0.1.1
  31     # Input: global assoc. array 'appRollCall'
  32     # Output: adds/updates key(value) to global assoc array 'appRollCall'
  33     # Depends: bash 5.0.3
  34     local returnState
  35
  36     #===Process Args===
  37     for arg in "$@"; do
  38         if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
  39             appRollCall[$arg]="true";
  40             if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
  41         else
  42             appRollCall[$arg]="false"; returnState="false";
  43         fi;
  44     done;
  45
  46     #===Determine function return code===
  47     if [ "$returnState" = "true" ]; then
  48         return 0;
  49     else
  50         return 1;
  51     fi;
  52 } # Check that app exists
  53 checkfile() {
  54     # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
  55     # Usage: checkfile arg1 arg2 arg3 ...
  56     # Version: 0.1.1
  57     # Input: global assoc. array 'fileRollCall'
  58     # Output: adds/updates key(value) to global assoc array 'fileRollCall';
  59     # Output: returns 0 if app found, 1 otherwise
  60     # Depends: bash 5.0.3
  61     local returnState
  62
  63     #===Process Args===
  64     for arg in "$@"; do
  65         if [ -f "$arg" ]; then
  66             fileRollCall["$arg"]="true";
  67             if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
  68         else
  69             fileRollCall["$arg"]="false"; returnState="false";
  70         fi;
  71     done;
  72
  73     #===Determine function return code===
  74     if [ "$returnState" = "true" ]; then
  75         return 0;
  76     else
  77         return 1;
  78     fi;
  79 } # Check that file exists
  80 checkdir() {
  81     # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
  82     # Usage: checkdir arg1 arg2 arg3 ...
  83     # Version 0.1.2
  84     # Input: global assoc. array 'dirRollCall'
  85     # Output: adds/updates key(value) to global assoc array 'dirRollCall';
  86     # Output: returns 0 if all args are dirs; 1 otherwise
  87     # Depends: Bash 5.0.3
  88     local returnState
  89
  90     #===Process Args===
  91     for arg in "$@"; do
  92         if [ -z "$arg" ]; then
  93             dirRollCall["(Unspecified Dirname(s))"]="false"; returnState="false";
  94         elif [ -d "$arg" ]; then
  95             dirRollCall["$arg"]="true";
  96             if ! [ "$returnState" = "false" ]; then returnState="true"; fi
  97         else
  98             dirRollCall["$arg"]="false"; returnState="false";
  99         fi
 100     done
 101
 102     #===Determine function return code===
 103     if [ "$returnState" = "true" ]; then
 104         return 0;
 105     else
 106         return 1;
 107     fi
 108 } # Check that dir exists
 109 displayMissing() {
 110     # Desc: Displays missing apps, files, and dirs
 111     # Usage: displayMissing
 112     # Version 1.0.0
 113     # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
 114     # Output: stderr: messages indicating missing apps, file, or dirs
 115     # Output: returns exit code 0 if nothing missing; 1 otherwise
 116     # Depends: bash 5, checkAppFileDir()
 117     local missingApps value appMissing missingFiles fileMissing
 118     local missingDirs dirMissing
 119
 120     #==BEGIN Display errors==
 121     #===BEGIN Display Missing Apps===
 122     missingApps="Missing apps  :";
 123     #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
 124     for key in "${!appRollCall[@]}"; do
 125         value="${appRollCall[$key]}";
 126         if [ "$value" = "false" ]; then
 127             #echo "DEBUG:Missing apps: $key => $value";
 128             missingApps="$missingApps""$key ";
 129             appMissing="true";
 130         fi;
 131     done;
 132     if [ "$appMissing" = "true" ]; then  # Only indicate if an app is missing.
 133         echo "$missingApps" 1>&2;
 134     fi;
 135     unset value;
 136     #===END Display Missing Apps===
 137
 138     #===BEGIN Display Missing Files===
 139     missingFiles="Missing files:";
 140     #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
 141     for key in "${!fileRollCall[@]}"; do
 142         value="${fileRollCall[$key]}";
 143         if [ "$value" = "false" ]; then
 144             #echo "DEBUG:Missing files: $key => $value";
 145             missingFiles="$missingFiles""$key ";
 146             fileMissing="true";
 147         fi;
 148     done;
 149     if [ "$fileMissing" = "true" ]; then  # Only indicate if an app is missing.
 150         echo "$missingFiles" 1>&2;
 151     fi;
 152     unset value;
 153     #===END Display Missing Files===
 154
 155     #===BEGIN Display Missing Directories===
 156     missingDirs="Missing dirs:";
 157     #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
 158     for key in "${!dirRollCall[@]}"; do
 159         value="${dirRollCall[$key]}";
 160         if [ "$value" = "false" ]; then
 161             #echo "DEBUG:Missing dirs: $key => $value";
 162             missingDirs="$missingDirs""$key ";
 163             dirMissing="true";
 164         fi;
 165     done;
 166     if [ "$dirMissing" = "true" ]; then  # Only indicate if an dir is missing.
 167         echo "$missingDirs" 1>&2;
 168     fi;
 169     unset value;
 170     #===END Display Missing Directories===
 171
 172     #==END Display errors==
 173     #==BEGIN Determine function return code===
 174     if [ "$appMissing" == "true" ] || [ "$fileMissing" == "true" ] || [ "$dirMissing" == "true" ]; then
 175         return 1;
 176     else
 177         return 0;
 178     fi
 179     #==END Determine function return code===
 180 } # Display missing apps, files, dirs
 181 vbm() {
 182     # Description: Prints verbose message ("vbm") to stderr if opVerbose is set to "true".
 183     # Usage: vbm "DEBUG :verbose message here"
 184     # Version 0.2.0
 185     # Input: arg1: string
 186     #        vars: opVerbose
 187     # Output: stderr
 188     # Depends: bash 5.0.3, GNU-coreutils 8.30 (echo, date)
 189
 190     if [ "$opVerbose" = "true" ]; then
 191         functionTime="$(date --iso-8601=ns)"; # Save current time in nano seconds.
 192         echo "[$functionTime]:$0:""$*" 1>&2;  # Display argument text.
 193     fi
 194
 195     # End function
 196     return 0; # Function finished.
 197 } # Displays message if opVerbose true
 198 showVersion() {
 199     # Desc: Displays script version and license information.
 200     # Usage: showVersion
 201     # Version: 0.0.1
 202     # Input: scriptVersion   var containing version string
 203     # Output: stdout
 204     # Depends: vbm(), yell, GNU-coreutils 8.30
 205
 206     # Initialize function
 207     vbm "DEBUG:showVersion function called."
 208
 209     cat <<'EOF'
 210 bkots 3.0.0
 211 Copyright (C) 2026 Steven Baltakatei Sandoval
 212 License GPLv3: GNU GPL version 3
 213 This is free software; you are free to change and redistribute it.
 214 There is NO WARRANTY, to the extent permitted by law.
 215
 216     GNU Coreutils 8.32
 217     Copyright (C) 2020 Free Software Foundation, Inc.
 218     License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
 219     This is free software: you are free to change and redistribute it.
 220     There is NO WARRANTY, to the extent permitted by law.
 221 EOF
 222
 223     # End function
 224     vbm "DEBUG:showVersion function ended."
 225     return 0; # Function finished.
 226 } # Display script version.
 227 showUsage() {
 228     # Desc: Display script usage information
 229     # Usage: showUsage
 230     # Version 0.0.1
 231     # Input: none
 232     # Output: stdout
 233     # Depends: GNU-coreutils 8.30 (cat)
 234     cat <<'EOF'
 235     USAGE:
 236         bkots [ options ] [PATH...]
 237
 238     POSITIONAL ARGUMENTS:
 239         PATH    Path(s) of file(s) or directory(ies)
 240
 241     OPTIONS:
 242         --dry-run
 243                 Do everything except run 'ots' commands.
 244         -h, --help
 245                 Display help information.
 246         --include-dotfiles
 247                 Include files and directories starting with '.' (not
 248                 included by default).
 249         -j, --jobs
 250                 Specify simultaneous job count (default: 2)
 251         -n, --batch-size
 252                 Specify file batch sizes for upgrade and stamp operations (default:500)
 253         -r, --recursive
 254                 Consider files in dirs recursively.
 255         --verify
 256                 Verify OTS files.
 257         --version
 258                 Display script version.
 259         -v, --verbose
 260                 Display debugging info.
 261         --
 262                 Mark end of options. Interpret remaining arguments as
 263                 positional arguments.
 264
 265     DESCRIPTION:
 266         Scans files by file paths or directory paths provided by
 267         positional arguments to see if Open Timestamps '.ots' file
 268         exists. If so, attempt to upgrade and verify the '.ots'
 269         file. If no '.ots' file exists, attempt to create one.
 270
 271         Files with a dotfile parent directory located anywhere in the
 272         file path are ignored by default. (e.g. 'HEAD' in
 273         '/home/user/diary/.git/logs/HEAD' because of '.git'). Dotfiles
 274         themselves are also ignored by default
 275         (e.g. '/home/user/.gitconfig').
 276
 277         Files modified less than 1 minute ago are ignored.
 278
 279     EXAMPLES:
 280       bkots -v foo.txt
 281       bkots foo.txt bar.pdf /home/username/Pictures/
 282 EOF
 283 } # Display information on how to use this script.
 284 count_jobs() {
 285     # Desc: Count and return total number of jobs
 286     # Usage: count_jobs
 287     # Input: None.
 288     # Output: stdout   integer number of jobs
 289     # Depends: Bash 5.1.16
 290     # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
 291     # Version: 0.0.1
 292
 293     local job_count;
 294     job_count="$(jobs -r | wc -l | tr -d ' ' )";
 295     #yell "DEBUG:job_count:$job_count";
 296     if [[ -z $job_count ]]; then job_count="0"; fi;
 297     echo "$job_count";
 298 }; # Return number of background jobs
 299 wait_for_jobslot() {
 300     # Desc: Does not return until count_jobs() falls below $max_job_count
 301     # Input: var max_job_count
 302     # Output: return code 0
 303     # Depends: count_jobs(), yell();
 304     while [[ $(count_jobs) -ge $max_job_count ]]; do
 305         printf "\r%d:Working..." "$SECONDS";
 306         sleep 1;
 307     done;
 308     return 0;
 309 };
 310 processArgs() {
 311     # Desc: Processes arguments provided to script.
 312     # Usage: processArgs "$@"
 313     # Version: 1.0.0
 314     # Input: "$@"          (list of arguments provided to the function)
 315     # Output: Sets following variables used by other functions:
 316     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
 317     #   arrayPosArgs         Array of remaining positional argments
 318     # Depends:
 319     #   yell()           Displays messages to stderr.
 320     #   vbm()            Displays messsages to stderr if opVerbose set to "true".
 321     #   showUsage()      Displays usage information about parent script.
 322     #   showVersion()    Displays version about parent script.
 323     #   arrayPosArgs     Global array for storing non-option positional arguments (i.e. arguments following the `--` option).
 324     # External dependencies: bash (5.1.16), echo
 325     # Ref./Attrib.:
 326     #  [1]: Marco Aurelio (2014-05-08). "echo that outputs to stderr". https://stackoverflow.com/a/23550347
 327     #  [2]: "Handling positional parameters" (2018-05-12). https://wiki.bash-hackers.org/scripting/posparams
 328
 329     # Initialize function
 330     vbm "DEBUG:processArgs function called."
 331
 332     # Perform work
 333     while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
 334         #yell "DEBUG:Starting processArgs while loop." # Debug stderr message. See [1].
 335         #yell "DEBUG:Provided arguments are:""$*"      # Debug stderr message. See [1].
 336         case "$1" in
 337             --dry-run) # Do not run ots commands
 338                 option_dry_run="true";
 339                 vbm "DEBUG:Option enabled:dry run";;
 340             -h | --help) showUsage; exit 1;; # Display usage.
 341             --include-dotfiles) # Include dotfiles
 342                 option_include_dotfiles="true";
 343                 vbm "DEBUG:Option enabled:include dotfiles";;
 344             -j | --jobs) # Specify max_job_count
 345                 if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
 346                     max_job_count="$2";
 347                     vbm "STATUS:Max job count set to:$max_job_count";
 348                     shift;
 349                 else
 350                     showUsage;
 351                     die "FATAL:Invalid job count:$2";
 352                 fi;;
 353             -n | --batch-size)
 354                 if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
 355                     max_batch_size="$2";
 356                     vbm "STATUS:Max batch size set to:$max_batch_size";
 357                     shift;
 358                 else
 359                     showUsage;
 360                     die "FATAL:Invalid xargs file batch size:$2";
 361                 fi;;
 362             -r | --recursive) # Specify recursive option
 363                 option_recursive="true";
 364                 vbm "DEBUG:Option enabled:include files in dirs recursively";;
 365             --verify)
 366                 option_verify="true";
 367                 vbm "DEBUG:Option enabled:verify OTS files";;
 368             --version) showVersion; exit 1;; # Show version
 369             -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
 370             --) # End of all options. See [2].
 371                 shift;
 372                 for arg in "$@"; do
 373                     vbm "DEBUG:adding to arrayPosArgs:$arg";
 374                     arrayPosArgs+=("$arg");
 375                 done;
 376                 break;;
 377             -*) showUsage; yell "ERROR: Unrecognized option."; exit 1;; # Display usage
 378             *) # Assume remaining arguments are positional arguments
 379                 for arg in "$@"; do
 380                     vbm "DEBUG:adding to arrayPosArgs:$arg";
 381                     arrayPosArgs+=("$arg");
 382                 done;
 383                 break;;
 384             #*) showUsage; yell "ERROR: Unrecognized argument."; exit 1;; # Handle unrecognized options. See [1].
 385         esac
 386         shift
 387     done
 388
 389     # End function
 390     vbm "DEBUG:processArgs function ended."
 391     return  0; # Function finished.
 392 }; # Evaluate script options from positional arguments (ex: $1, $2, $3, etc.).
 393 get_parent_dirnames() {
 394     # Desc: Provides newline-delimited list of each parent dir of a file or dir
 395     # Usage: get_parent_dirnames arg1
 396     # Input: arg1  input  path
 397     # Output: stdout   newline-delimited list of parent dirs
 398     # Version: 0.0.1
 399     # Depends: yell(), die(), must()
 400     local path
 401
 402     # Check input
 403     if [[ $# -ne 1 ]]; then die "FATAL:Incorrect number of arguments:$#"; fi;
 404     if ! { [[ -f $1 ]] || [[ -d $1 ]]; }; then die "FATAL:Not a file or dir:$1"; fi;
 405
 406     # Process path
 407     path="$1";
 408     while [[ -f $path ]] || [[ -d $path ]]; do
 409         path="$(dirname "$path")";
 410         name_base_previous="$name_base";
 411         name_base="$(basename "$path")";
 412         ## Check for stop condition (dirname returns same result as previous iteration)
 413         if [[ $name_base == "$name_base_previous" ]]; then break; fi;
 414         echo "$name_base";
 415     done;
 416 }; # Output parent dirnames to stdout
 417 main() {
 418     # Desc: Creates `.ots` file:
 419     #     - for each file specified in arrayPosArgs array
 420     #     - for each file in each dir specified in arrayPosArgs array
 421     #   Output file created alongside each file or in output directory specified by pathDirIn1
 422     # Usage: main "$@";
 423     # Input: arrayPosArgs   array with positional arguments
 424     #        pathDirOut1    path for output `.ots` files (if pathDirOut1 is specified and is a path)
 425     #        age_threshold  var: mininum age in seconds to timestamp file
 426
 427     # Output: file(s) creates `.ots` file alongside specified files
 428     # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822
 429     # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
 430     #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
 431     #             [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377
 432     local -a file_list file_list_pruned;
 433     local -a files_to_verify files_to_upgrade files_to_stamp
 434     local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned
 435
 436     # Process args
 437     processArgs "$@";
 438
 439     # Check dependencies
 440     if ! checkapp ots find parallel; then
 441         displayMissing;
 442         die "FATAL:Missing dependencies.";
 443     fi;
 444
 445     # Check arguments
 446     for arg in "${arrayPosArgs[@]}"; do
 447         arg="$(readlink -f "$arg")";
 448         if ! { [[ -d $arg ]] || [[ -f $arg ]]; }; then
 449             die "FATAL:Not a file or dir:arg:$arg";
 450         fi;
 451     done;
 452
 453     # Display ots details
 454     vbm "$(type ots)"; # show how 'ots' is defined
 455         #TODO: add option to define 'ots' as a bash function that
 456         #populates the ots option '--bitcoin-node FILE' with a
 457         #user-specified FILE.
 458
 459     # Populate file_list
 460     vbm "DEBUG:begin populate file_list array";
 461     for item in "${arrayPosArgs[@]}"; do
 462         vbm "DEBUG:adding to file list:item:$item";
 463
 464         ## Get full canonicalized path (follow symlinks)
 465         item="$(readlink -f "$item")";
 466         vbm "DEBUG:item full path:item:$item";
 467
 468         ## Add to list: files
 469         if [[ -f $item ]]; then
 470             vbm "DEBUG:is a file:item:$item";
 471             file_list+=("$item");
 472             vbm "DEBUG:added to file_list:$item";
 473         ## Add to list: files in dirs
 474         elif [[ -d $item ]]; then
 475             vbm "DEBUG:is a dir:item:$item";
 476             ### Check for recursive flag
 477             if [[ "$option_recursive" == "true" ]]; then
 478                 vbm "DEBUG:option_recursive:$option_recursive";
 479                 while read -r line; do
 480                     file_list+=("$line");
 481                     vbm "DEBUG:added to file_list:$line";
 482                 done < <(find "$item" -type f);
 483             else
 484                 while read -r line; do
 485                     file_list+=("$line");
 486                     vbm "DEBUG:added to file_list:$line";
 487                 done < <(find "$item" -maxdepth 1 -type f);
 488             fi;
 489         else
 490             die "FATAL:Not a file or dir:item:$item";
 491         fi;
 492     done;
 493     if [[ $opVerbose == "true" ]]; then
 494         vbm "DEBUG:file_list:";
 495         printf "%s\n" "${file_list[@]}";
 496     fi;
 497
 498     # Prune file_list
 499     for item in "${file_list[@]}"; do
 500         ## Ignore files that end in '.ots.bak'.
 501         if [[ $item =~ '.ots.bak'$ ]]; then
 502             vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item";
 503             continue; # skip to next item
 504         fi;
 505
 506         ## Ignore dotfiles
 507         if ! [[ $option_include_dotfiles == "true" ]]; then
 508             ### Ignore files if '/.' contained within canonical path
 509             pattern="/\."; # a dot after a forward slash
 510             if [[ $item =~ $pattern ]]; then
 511                 continue;
 512             fi;
 513
 514             # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
 515             # unset flag_contains_dotfile_parent;
 516             # while read -r line; do
 517             #     #### Check line from output of get_parent_dirnames
 518             #     pattern="^\.";
 519             #     if [[ $line =~ $pattern ]]; then
 520             #         ##### line starts with '.'
 521             #         vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
 522             #         vbm "DEBUG:Dotfile in path:item:$item";
 523             #         vbm "DEBUG:Dotfile parent:line:$line";
 524             #         flag_contains_dotfile_parent="true";
 525             #         break
 526             #     fi;
 527             # done < <(get_parent_dirnames "$item");
 528             # if [[ $flag_contains_dotfile_parent == "true" ]]; then
 529             #     unset flag_contains_dotfile_parent;
 530             #     continue; # skip to next item (i.e. don't add to file_list_pruned)
 531             # fi;
 532
 533             # ### Ignore dotfiles themselves
 534             # item_basename="$(basename "$item")";
 535             # pattern="^\.";
 536             # if [[ $item_basename =~ $pattern ]]; then
 537             #     vbm "INFO :Skipping dotfile:item:$item";
 538             #     continue; # skip to next item
 539             # fi;
 540         fi;
 541
 542         ## Ignore files with newlines present in filename. See [2].
 543         if [[ $item =~ $'\n' ]]; then
 544             vbm "DEBUG:Skipping file name with newline:$item";
 545             continue; # skip to next item
 546         fi;
 547
 548         ## Ignore files that end in '~'.
 549         if [[ $item =~ ~$ ]]; then
 550             vbm "DEBUG:Skipping file ending in tilde:$item";
 551             continue; # skip to next item
 552         fi;
 553
 554         ## Ignore files modified less than age_threshold. See [3].
 555         time_now="$(date +%s)"; # epoch seconds
 556         item_age="$(stat -c %Y "$item")"; # age in seconds
 557         if [[ $(( time_now - item_age )) -lt "$age_threshold" ]]; then
 558             yell "INFO :Skipping file modified less than $age_threshold seconds ago:item:$item";
 559             continue; # skip to next item
 560         fi;
 561
 562         ## Add item to file_list_pruned
 563         file_list_pruned+=("$item");
 564     done;
 565     if [[ $opVerbose == "true" ]]; then
 566         vbm "DEBUG:file_list_pruned:";
 567         printf "%s\n" "${file_list_pruned[@]}";
 568     fi;
 569
 570     # Decide what actions to take for items in file_list_pruned
 571     for item in "${file_list_pruned[@]}"; do
 572         vbm "DEBUG:considering action to take for item:$item";
 573         unset path_src path_prf dir_parent dir_source;
 574
 575         ## Check file extension
 576         if [[ $item =~ .ots$ ]]; then
 577             ### item ends in '.ots'. Item is proof file.
 578             vbm "DEBUG:item ends in '.ots'. Item is proof file:item:$item";
 579             if [[ -f ${item%.ots} ]]; then
 580                 #### Proof file (item) is adjacent to source file
 581                 vbm "DEBUG:Proof file (item) is adjacent to source file.";
 582                 ##### Upgrade and verify proof file against adjacent source file
 583                 vbm "DEBUG:Marking proof file to be upgraded and verified.";
 584                 path_src="${item%.ots}";
 585                 path_prf="$item";
 586                 files_to_upgrade+=("$(printf "%s" "$path_prf")");
 587                 files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
 588             else
 589                 #### Proof file (item) is not adjacent to source file
 590                 vbm "DEBUG:Proof file (item) is not adjacent to source file.";
 591                 #### Check if source file in parent dir
 592                 dir_parent="$(dirname "$(dirname "$item")" )";
 593                 cand_src_filename="$(basename "$item")";
 594                 cand_src_path="$dir_parent/$cand_src_filename";
 595                 if [[ -f "$cand_src_path" ]]; then
 596                     ##### source file in parent dir
 597                     vbm "DEBUG:found source file in parent:cand_src_path:$cand_src_path";
 598                     path_src="$cand_src_path";
 599                     path_prf="$item";
 600                     files_to_upgrade+=("$(printf "%s" "$path_prf")");
 601                     files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
 602                 else
 603                     #### Throw non-fatal error
 604                     vbm "DEBUG:Source file not found for proof file:item:$item";
 605                     yell "ERROR:Item is proof file but source filei not adjacent in parent dir. item:$item";
 606                     #### Attempt upgrade only
 607                     vbm "DEBUG:Marking proof file to be upgraded.";
 608                     path_prf="$item";
 609                     files_to_upgrade+=("$(printf "%s" "$path_prf")");
 610                 fi;
 611             fi;
 612         else
 613             ### item does not end in '.ots'. Item is source file.
 614             vbm "DEBUG:item does NOT end in '.ots'. Item is source file.";
 615             if [[ -f "$item".ots ]]; then
 616                 #### Proof file is adjacent to source file (item).
 617                 vbm "DEBUG:Proof file is adjacent to source file (item).";
 618                 ##### Upgrade and verify proof file against adjacent source file.
 619                 vbm "DEBUG:Marking proof file to be upgraded and verified.";
 620                 path_src="$item";
 621                 path_prf="$item.ots";
 622                 files_to_upgrade+=("$(printf "%s" "$path_prf")");
 623                 files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
 624             else
 625                 #### Proof file is not adjacent to source file (item).
 626                 #### Check if proof file is in subdir
 627                 vbm "DEBUG:checking if proof file for source file (item) is in subdir:item:$item";
 628                 unset flag_proof_in_subdir;
 629                 dir_item="$(dirname "$item")";
 630                 cand_prf_filename="$(basename "$item")".ots;
 631                 while read -r line; do
 632                     line_basename="$(basename "$line")";
 633                     if [[ $line_basename == "$cand_prf_filename" ]]; then
 634                         flag_proof_in_subdir="true";
 635                         path_prf="$line";
 636                         vbm "DEBUG:proof found in subdir at:line:$line";
 637                         break;
 638                     fi;
 639                 done < <(find "$dir_item" -mindepth 2 -maxdepth 2 -type f)
 640                 if [[ $flag_proof_in_subdir == "true" ]]; then
 641                     ##### Proof file is in subdir
 642                     vbm "DEBUG:Proof file detected in subdir relative to source file (item)";
 643                     #path_prf="$path_prf"; # set in while loop
 644                     path_src="$item";
 645                     files_to_upgrade+=("$(printf "%s" "$path_prf")");
 646                     files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
 647                 else
 648                     ##### Proof file is not in subdir
 649                     vbm "DEBUG:Proof file not detected in subdir relative to source file (item).";
 650                     #### Stamp source file
 651                     vbm "DEBUG:Marking source file to be stamped.";
 652                     path_src="$item";
 653                     files_to_stamp+=("$(printf "%s" "$path_src")")
 654                 fi;
 655                 unset flag_proof_in_subdir;
 656             fi;
 657         fi;
 658     done;
 659     unset path_src path_prf dir_item dir_parent cand_prf_filename cand_src_filename line_basename cand_src_path
 660
 661     # Prune action lists.
 662     ## Sort and prune file action arrays
 663     ### files to upgrade
 664     while read -r -d $'\0' line; do
 665         if [[ -z $line ]]; then continue; fi; # Skip empty lines
 666         vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line";
 667         files_to_upgrade_pruned+=("$line");
 668     done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1]
 669     if [[ $opVerbose == "true" ]]; then
 670         vbm "DEBUG:files_to_upgrade_pruned:";
 671         printf "%s\n" "${files_to_upgrade_pruned[@]}";
 672     fi;
 673
 674     ### files to verify
 675     while read -r -d $'\0' line; do
 676         if [[ -z $line ]]; then continue; fi; # Skip empty lines
 677         vbm "DEBUG:adding to files_to_verify_pruned:line:$line";
 678         files_to_verify_pruned+=("$line");
 679     done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1]
 680     if [[ $opVerbose == "true" ]]; then
 681         vbm "DEBUG:files_to_verify_pruned:";
 682         printf "%s\n\n" "${files_to_verify_pruned[@]}";
 683     fi;
 684
 685     ### files to stamp
 686     while read -r -d $'\0' line; do
 687         if [[ -z $line ]]; then continue; fi; # Skip empty lines
 688         vbm "DEBUG:adding to files_to_stamp_pruned:line:$line";
 689         files_to_stamp_pruned+=("$line");
 690     done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1]
 691     if [[ $opVerbose == "true" ]]; then
 692         vbm "DEBUG:files_to_stamp_pruned:";
 693         printf "%s\n" "${files_to_stamp_pruned[@]}";
 694     fi;
 695
 696     # Act on files
 697     ## Assemble and execute upgrade file commands
 698     if [[ ${#files_to_upgrade_pruned[@]} -gt 0 ]]; then
 699         if [[ $option_dry_run == "true" ]]; then
 700             yell "DEBUG:DRY RUN:Not running:\"ots upgrade\"";
 701         else
 702             ### Pick random calendar
 703             function get_calurl() {
 704                 # Desc: Returns random calendar url
 705                 # Input:  string  OTS_CALS   newline-delimited list of calendars
 706                 # Output: stdout             single calendar url
 707                 printf "%s" "$OTS_CALS" | shuf -n1;
 708             }; # print random calendar url to stdout
 709             export -f get_calurl;
 710
 711             ### Perform upgrade command in xargs batches
 712             printf '%s\0' "${files_to_upgrade_pruned[@]}" \
 713                 | shuf -z \
 714                 | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots --no-default-whitelist -l "$(get_calurl)" upgrade "$@";' _ ;
 715         fi;
 716     fi;
 717
 718     # ## Assemble and execute verify file commands
 719     if [[ $option_verify == "true" ]]; then
 720         for item in "${files_to_verify_pruned[@]}"; do
 721             wait_for_jobslot && {
 722                 path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
 723                 path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
 724                 if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
 725                     yell "ERROR:blank verify item encountered. Skipping:item:$item";
 726                     return 1; # would have been `continue` were it not in a subshell
 727                 fi;
 728                 vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
 729                 vbm "DEBUG:    against proof file:          path_prf:$path_prf";
 730                 if [[ ! $option_dry_run == "true" ]]; then
 731                     ### Try verify with known calendars in random order
 732                     while read -r url; do
 733                         vbm "DEBUG:Verifying with calendar:url:$url";
 734
 735                         #### assemble command
 736                         local -a cmd_temp;
 737                         cmd_temp=("ots");
 738                         if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
 739                         cmd_temp+=("-l" "$url" "--no-default-whitelist");
 740                         cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
 741                         if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
 742
 743                         #### execute command
 744                         "${cmd_temp[@]}";
 745                         unset cmd_temp;
 746                         break;
 747                         #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
 748                     done < <(printf "%s\n" "${calendars[@]}" | shuf);
 749                 else
 750                     yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
 751                 fi;
 752             } &
 753         done;
 754
 755         ## Wait for jobs to finish.
 756         wait;
 757     else
 758         vbm "DEBUG:Skipping operation:ots verify.";
 759     fi;
 760
 761     ## Assemble and execute stamp file commands
 762     if [[ ${#files_to_stamp_pruned[@]} -gt 0 ]]; then
 763         if [[ $option_dry_run == "true" ]]; then
 764             yell "DEBUG:DRY RUN:Not running:\"ots stamp\"";
 765         else
 766             function get_calurl() {
 767                 # Desc: Returns random calendar url
 768                 # Input:  string  OTS_CALS   newline-delimited list of calendars
 769                 # Output: stdout             single calendar url
 770                 printf "%s" "$OTS_CALS" | shuf -n1;
 771             }; # print random calendar url to stdout
 772             export -f get_calurl;
 773
 774             ### Perform stamp command in xargs batches
 775             printf '%s\0' "${files_to_stamp_pruned[@]}" \
 776                 | shuf -z \
 777                 | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots stamp "$@";' _ ;
 778         fi;
 779     fi;
 780
 781
 782 }; # main program
 783
 784 # Run program
 785 main "$@";
 786 exit 0;