feat(user/bkots):Optimize with GNU Parallel for executing commands

[BK-2020-03.git] / user / bkots

#!/usr/bin/env bash

# Define variables
declare -Ag appRollCall # Associative array for storing app status
declare -Ag fileRollCall # Associative array for storing file status
declare -Ag dirRollCall # Associative array for storing dir status
declare -ag arrayPosArgs # Associative array for processArgs() function
declare -ag calendars;
calendars+=("https://finney.calendar.eternitywall.com");
calendars+=("https://btc.calendar.catallaxy.com");
calendars+=("https://alice.btc.calendar.opentimestamps.org");
calendars+=("https://bob.btc.calendar.opentimestamps.org");
declare -a commands # array for storing assembled commands
age_threshold="60"; # min age to add file; seconds;

# Declare functions
yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
checkapp() {
    # Desc: If arg is a command, save result in assoc array 'appRollCall'
    # Usage: checkapp arg1 arg2 arg3 ...
    # Version: 0.1.1
    # Input: global assoc. array 'appRollCall'
    # Output: adds/updates key(value) to global assoc array 'appRollCall'
    # Depends: bash 5.0.3
    local returnState    

    #===Process Args===
    for arg in "$@"; do
	if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
	    appRollCall[$arg]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
	else
	    appRollCall[$arg]="false"; returnState="false";
	fi;
    done;

    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi;
} # Check that app exists
checkfile() {
    # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
    # Usage: checkfile arg1 arg2 arg3 ...
    # Version: 0.1.1
    # Input: global assoc. array 'fileRollCall'
    # Output: adds/updates key(value) to global assoc array 'fileRollCall';
    # Output: returns 0 if app found, 1 otherwise
    # Depends: bash 5.0.3
    local returnState

    #===Process Args===
    for arg in "$@"; do
	if [ -f "$arg" ]; then
	    fileRollCall["$arg"]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
	else
	    fileRollCall["$arg"]="false"; returnState="false";
	fi;
    done;
    
    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi;
} # Check that file exists
checkdir() {
    # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
    # Usage: checkdir arg1 arg2 arg3 ...
    # Version 0.1.2
    # Input: global assoc. array 'dirRollCall'
    # Output: adds/updates key(value) to global assoc array 'dirRollCall';
    # Output: returns 0 if all args are dirs; 1 otherwise
    # Depends: Bash 5.0.3
    local returnState

    #===Process Args===
    for arg in "$@"; do
	if [ -z "$arg" ]; then
	    dirRollCall["(Unspecified Dirname(s))"]="false"; returnState="false";
	elif [ -d "$arg" ]; then
	    dirRollCall["$arg"]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi
	else
	    dirRollCall["$arg"]="false"; returnState="false";
	fi
    done
    
    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi
} # Check that dir exists
displayMissing() {
    # Desc: Displays missing apps, files, and dirs
    # Usage: displayMissing
    # Version 1.0.0
    # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
    # Output: stderr: messages indicating missing apps, file, or dirs
    # Output: returns exit code 0 if nothing missing; 1 otherwise
    # Depends: bash 5, checkAppFileDir()
    local missingApps value appMissing missingFiles fileMissing
    local missingDirs dirMissing

    #==BEGIN Display errors==
    #===BEGIN Display Missing Apps===
    missingApps="Missing apps  :";
    #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
    for key in "${!appRollCall[@]}"; do
	value="${appRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing apps: $key => $value";
	    missingApps="$missingApps""$key ";
	    appMissing="true";
	fi;
    done;
    if [ "$appMissing" = "true" ]; then  # Only indicate if an app is missing.
	echo "$missingApps" 1>&2;
    fi;
    unset value;
    #===END Display Missing Apps===

    #===BEGIN Display Missing Files===
    missingFiles="Missing files:";
    #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
    for key in "${!fileRollCall[@]}"; do
	value="${fileRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing files: $key => $value";
	    missingFiles="$missingFiles""$key ";
	    fileMissing="true";
	fi;
    done;
    if [ "$fileMissing" = "true" ]; then  # Only indicate if an app is missing.
	echo "$missingFiles" 1>&2;
    fi;
    unset value;
    #===END Display Missing Files===

    #===BEGIN Display Missing Directories===
    missingDirs="Missing dirs:";
    #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
    for key in "${!dirRollCall[@]}"; do
	value="${dirRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing dirs: $key => $value";
	    missingDirs="$missingDirs""$key ";
	    dirMissing="true";
	fi;
    done;
    if [ "$dirMissing" = "true" ]; then  # Only indicate if an dir is missing.
	echo "$missingDirs" 1>&2;
    fi;
    unset value;
    #===END Display Missing Directories===

    #==END Display errors==
    #==BEGIN Determine function return code===
    if [ "$appMissing" == "true" ] || [ "$fileMissing" == "true" ] || [ "$dirMissing" == "true" ]; then
	return 1;
    else
	return 0;
    fi
    #==END Determine function return code===
} # Display missing apps, files, dirs
vbm() {
    # Description: Prints verbose message ("vbm") to stderr if opVerbose is set to "true".
    # Usage: vbm "DEBUG :verbose message here"
    # Version 0.2.0
    # Input: arg1: string
    #        vars: opVerbose
    # Output: stderr
    # Depends: bash 5.0.3, GNU-coreutils 8.30 (echo, date)

    if [ "$opVerbose" = "true" ]; then
	functionTime="$(date --iso-8601=ns)"; # Save current time in nano seconds.
	echo "[$functionTime]:$0:""$*" 1>&2;  # Display argument text.
    fi

    # End function
    return 0; # Function finished.
} # Displays message if opVerbose true
showVersion() {
    # Desc: Displays script version and license information.
    # Usage: showVersion
    # Version: 0.0.1
    # Input: scriptVersion   var containing version string
    # Output: stdout
    # Depends: vbm(), yell, GNU-coreutils 8.30

    # Initialize function
    vbm "DEBUG:showVersion function called."

    cat <<'EOF'
bkots 1.0.0
Copyright (C) 2022 Steven Baltakatei Sandoval
License GPLv3: GNU GPL version 3
This is free software; you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

    GNU Coreutils 8.32
    Copyright (C) 2020 Free Software Foundation, Inc.
    License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
    This is free software: you are free to change and redistribute it.
    There is NO WARRANTY, to the extent permitted by law.
EOF
    
    # End function
    vbm "DEBUG:showVersion function ended."
    return 0; # Function finished.
} # Display script version.
showUsage() {
    # Desc: Display script usage information
    # Usage: showUsage
    # Version 0.0.1
    # Input: none
    # Output: stdout
    # Depends: GNU-coreutils 8.30 (cat)
    cat <<'EOF'
    USAGE:
        bkots [ options ] [PATH...]

    POSITIONAL ARGUMENTS:
        PATH    Path(s) of file(s) or directory(ies)

    OPTIONS:
        --dry-run
                Do everything except run 'ots' commands.
        -h, --help
                Display help information.
        --include-dotfiles
                Include files and directories starting with '.' (not
                included by default).
        -r, --recursive
                Consider files in dirs recursively.
        --version
                Display script version.
        -v, --verbose
                Display debugging info.
        --
                Mark end of options. Interpret remaining arguments as
                positional arguments.

    DESCRIPTION:
        Scans files by file paths or directory paths provided by
        positional arguments to see if Open Timestamps '.ots' file
        exists. If so, attempt to upgrade and verify the '.ots'
        file. If no '.ots' file exists, attempt to create one.

        Files with a dotfile parent directory located anywhere in the
        file path are ignored by default. (e.g. 'HEAD' in
        '/home/user/diary/.git/logs/HEAD' because of '.git'). Dotfiles
        themselves are also ignored by default
        (e.g. '/home/user/.gitconfig').

        Files modified less than 1 minute ago are ignored.

    EXAMPLES:
      bkots -v foo.txt
      bkots foo.txt bar.pdf /home/username/Pictures/
EOF
} # Display information on how to use this script.
processArgs() {
    # Desc: Processes arguments provided to script.
    # Usage: processArgs "$@"
    # Version: 1.0.0
    # Input: "$@"          (list of arguments provided to the function)
    # Output: Sets following variables used by other functions:
    #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
    #   arrayPosArgs         Array of remaining positional argments
    # Depends:
    #   yell()           Displays messages to stderr.
    #   vbm()            Displays messsages to stderr if opVerbose set to "true".
    #   showUsage()      Displays usage information about parent script.
    #   showVersion()    Displays version about parent script.
    #   arrayPosArgs     Global array for storing non-option positional arguments (i.e. arguments following the `--` option).
    # External dependencies: bash (5.1.16), echo
    # Ref./Attrib.:
    #  [1]: Marco Aurelio (2014-05-08). "echo that outputs to stderr". https://stackoverflow.com/a/23550347
    #  [2]: "Handling positional parameters" (2018-05-12). https://wiki.bash-hackers.org/scripting/posparams

    # Initialize function
    vbm "DEBUG:processArgs function called."

    # Perform work
    while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
	#yell "DEBUG:Starting processArgs while loop." # Debug stderr message. See [1].
        #yell "DEBUG:Provided arguments are:""$*"      # Debug stderr message. See [1].
	case "$1" in
            --dry-run) # Do not run ots commands
                option_dry_run="true";
                vbm "DEBUG:Option enabled:dry run";;
	    -h | --help) showUsage; exit 1;; # Display usage.
            --include-dotfiles) # Include dotfiles
                option_include_dotfiles="true";
                vbm "DEBUG:Option enabled:include dotfiles";;
            -r | --recursive) # Specify recursive option
                option_recursive="true";
                vbm "DEBUG:option enabled:include files in dirs recursively";;
	    --version) showVersion; exit 1;; # Show version
	    -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
            --) # End of all options. See [2].
                shift;
                for arg in "$@"; do
                    vbm "DEBUG:adding to arrayPosArgs:$arg";
                    arrayPosArgs+=("$arg");
                done;
                break;;
            -*) showUsage; yell "ERROR: Unrecognized option."; exit 1;; # Display usage
            *) # Assume remaining arguments are positional arguments
                for arg in "$@"; do
                    vbm "DEBUG:adding to arrayPosArgs:$arg";
                    arrayPosArgs+=("$arg");
                done;
                break;;
	    #*) showUsage; yell "ERROR: Unrecognized argument."; exit 1;; # Handle unrecognized options. See [1].
	esac
	shift
    done

    # End function
    vbm "DEBUG:processArgs function ended."
    return  0; # Function finished.
}; # Evaluate script options from positional arguments (ex: $1, $2, $3, etc.).
get_parent_dirnames() {
    # Desc: Provides newline-delimited list of each parent dir of a file or dir
    # Usage: get_parent_dirnames arg1
    # Input: arg1  input  path
    # Output: stdout   newline-delimited list of parent dirs
    # Version: 0.0.1
    # Depends: yell(), die(), try()
    local path

    # Check input
    if [[ $# -ne 1 ]]; then die "FATAL:Incorrect number of arguments:$#"; fi;
    if ! { [[ -f $1 ]] || [[ -d $1 ]]; }; then die "FATAL:Not a file or dir:$1"; fi;

    # Process path
    path="$1";
    while [[ -f $path ]] || [[ -d $path ]]; do
        path="$(dirname "$path")";
        name_base_previous="$name_base";
        name_base="$(basename "$path")";
        ## Check for stop condition (dirname returns same result as previous iteration)
        if [[ $name_base == "$name_base_previous" ]]; then break; fi;
        echo "$name_base";
    done;    
}; # Output parent dirnames to stdout
cmdwrap() {
    # print command to stderr
    echo "$@" 1>&2;

    # execute command
    "$@";
}; # print and execute string together
export -f cmdwrap; # export cmdwrap for use in other functions
main() {
    # Desc: Creates `.ots` file:
    #     - for each file specified in arrayPosArgs array
    #     - for each file in each dir specified in arrayPosArgs array
    #   Output file created alongside each file or in output directory specified by pathDirIn1
    # Usage: main "$@";
    # Input: arrayPosArgs   array with positional arguments
    #        pathDirOut1    path for output `.ots` files (if pathDirOut1 is specified and is a path)
    #        age_threshold  var: mininum age in seconds to timestamp file
    
    # Output: file(s) creates `.ots` file alongside specified files
    # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822
    # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
    #             [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
    #             [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377
    #             [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001
    local -a file_list file_list_pruned;
    local -a files_to_verify files_to_upgrade files_to_stamp
    local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned
    
    # Process args
    processArgs "$@";
    
    # Check dependencies
    if ! checkapp ots find parallel; then
        displayMissing;
        die "FATAL:Missing dependencies.";
    fi;
    
    # Check arguments
    for arg in "${arrayPosArgs[@]}"; do
        arg="$(readlink -f "$arg")";
        if ! { [[ -d $arg ]] || [[ -f $arg ]]; }; then
            die "FATAL:Not a file or dir:arg:$arg";
        fi;
    done;

    # Display ots details
    vbm "$(type ots)"; # show how 'ots' is defined
        #TODO: add option to define 'ots' as a bash function that
        #populates the ots option '--bitcoin-node FILE' with a
        #user-specified FILE.
    
    # Populate file_list
    vbm "DEBUG:begin populate file_list array";
    for item in "${arrayPosArgs[@]}"; do
        vbm "DEBUG:adding to file list:item:$item";

        ## Get full canonicalized path (follow symlinks)
        item="$(readlink -f "$item")";
        vbm "DEBUG:item full path:item:$item";
        
        ## Add to list: files
        if [[ -f $item ]]; then
            vbm "DEBUG:is a file:item:$item";
            file_list+=("$item");
            vbm "DEBUG:added to file_list:$item";
        ## Add to list: files in dirs
        elif [[ -d $item ]]; then
            vbm "DEBUG:is a dir:item:$item";
            ### Check for recursive flag
            if [[ "$option_recursive" == "true" ]]; then
                vbm "DEBUG:option_recursive:$option_recursive";
                while read -r line; do
                    file_list+=("$line");
                    vbm "DEBUG:added to file_list:$line";
                done < <(find "$item" -type f);
            else
                while read -r line; do
                    file_list+=("$line");
                    vbm "DEBUG:added to file_list:$line";
                done < <(find "$item" -maxdepth 1 -type f);
            fi;
        else
            die "FATAL:Not a file or dir:item:$item";
        fi;
    done;
    if [[ $opVerbose == "true" ]]; then
        vbm "DEBUG:file_list:";
        printf "%s\n" "${file_list[@]}";
    fi;

    # Prune file_list
    for item in "${file_list[@]}"; do
        ## Ignore files that end in '.ots.bak'.
        if [[ $item =~ '.ots.bak'$ ]]; then
            vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item";
            continue; # skip to next item
        fi;

        ## Ignore dotfiles
        if ! [[ $option_include_dotfiles == "true" ]]; then
            ### Ignore files if '/.' contained within canonical path
            pattern="/\."; # a dot after a forward slash
            if [[ $item =~ $pattern ]]; then
                continue;
            fi;
            
            # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
            # unset flag_contains_dotfile_parent;
            # while read -r line; do
            #     #### Check line from output of get_parent_dirnames
            #     pattern="^\.";
            #     if [[ $line =~ $pattern ]]; then
            #         ##### line starts with '.'
            #         vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
            #         vbm "DEBUG:Dotfile in path:item:$item";
            #         vbm "DEBUG:Dotfile parent:line:$line";
            #         flag_contains_dotfile_parent="true";
            #         break
            #     fi;
            # done < <(get_parent_dirnames "$item");
            # if [[ $flag_contains_dotfile_parent == "true" ]]; then
            #     unset flag_contains_dotfile_parent;
            #     continue; # skip to next item (i.e. don't add to file_list_pruned)
            # fi;

            # ### Ignore dotfiles themselves
            # item_basename="$(basename "$item")";
            # pattern="^\.";
            # if [[ $item_basename =~ $pattern ]]; then
            #     vbm "INFO :Skipping dotfile:item:$item";
            #     continue; # skip to next item
            # fi;
        fi;
        
        ## Ignore files with newlines present in filename. See [2].
        if [[ $item =~ $'\n' ]]; then
            vbm "DEBUG:Skipping file name with newline:$item";
            continue; # skip to next item
        fi;

        ## Ignore files that end in '~'.
        if [[ $item =~ ~$ ]]; then
            vbm "DEBUG:Skipping file ending in tilde:$item";
            continue; # skip to next item
        fi;

        ## Ignore files modified less than age_threshold. See [3].
        time_now="$(date +%s)"; # epoch seconds
        item_age="$(stat -c %Y "$item")"; # age in seconds
        if [[ $(( time_now - item_age )) -lt "$age_threshold" ]]; then
            yell "INFO :Skipping file modified less than $age_threshold seconds ago:item:$item";
            continue; # skip to next item
        fi;
        
        ## Add item to file_list_pruned
        file_list_pruned+=("$item");
    done;
    if [[ $opVerbose == "true" ]]; then
        vbm "DEBUG:file_list_pruned:";
        printf "%s\n" "${file_list_pruned[@]}";
    fi;
    
    # Decide what actions to take for items in file_list_pruned
    for item in "${file_list_pruned[@]}"; do
        vbm "DEBUG:considering action to take for item:$item";
        unset path_src path_prf dir_parent dir_source;
            
        ## Check file extension
        if [[ $item =~ .ots$ ]]; then
            ### item ends in '.ots'. Item is proof file.
            vbm "DEBUG:item ends in '.ots'. Item is proof file:item:$item";
            if [[ -f ${item%.ots} ]]; then
                #### Proof file (item) is adjacent to source file
                vbm "DEBUG:Proof file (item) is adjacent to source file.";
                ##### Upgrade and verify proof file against adjacent source file
                vbm "DEBUG:Marking proof file to be upgraded and verified.";
                path_src="${item%.ots}";
                path_prf="$item";
                files_to_upgrade+=("$(printf "%s" "$path_prf")");
                files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
            else
                #### Proof file (item) is not adjacent to source file
                vbm "DEBUG:Proof file (item) is not adjacent to source file.";
                #### Check if source file in parent dir
                dir_parent="$(dirname "$(dirname "$item")" )";
                cand_src_filename="$(basename "$item")";
                cand_src_path="$dir_parent/$cand_src_filename";
                if [[ -f "$cand_src_path" ]]; then
                    ##### source file in parent dir
                    vbm "DEBUG:found source file in parent:cand_src_path:$cand_src_path";
                    path_src="$cand_src_path";
                    path_prf="$item";
                    files_to_upgrade+=("$(printf "%s" "$path_prf")");
                    files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
                else
                    #### Throw non-fatal error
                    vbm "DEBUG:Source file not found for proof file:item:$item";
                    yell "ERROR:Item is proof file but source filei not adjacent in parent dir. item:$item";
                    #### Attempt upgrade only
                    vbm "DEBUG:Marking proof file to be upgraded.";
                    path_prf="$item";
                    files_to_upgrade+=("$(printf "%s" "$path_prf")");
                fi;
            fi;
        else
            ### item does not end in '.ots'. Item is source file.
            vbm "DEBUG:item does NOT end in '.ots'. Item is source file.";
            if [[ -f "$item".ots ]]; then
                #### Proof file is adjacent to source file (item).
                vbm "DEBUG:Proof file is adjacent to source file (item).";
                ##### Upgrade and verify proof file against adjacent source file.
                vbm "DEBUG:Marking proof file to be upgraded and verified.";
                path_src="$item";
                path_prf="$item.ots";                
                files_to_upgrade+=("$(printf "%s" "$path_prf")");
                files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
            else
                #### Proof file is not adjacent to source file (item).
                #### Check if proof file is in subdir
                vbm "DEBUG:checking if proof file for source file (item) is in subdir:item:$item";
                unset flag_proof_in_subdir;
                dir_item="$(dirname "$item")";
                cand_prf_filename="$(basename "$item")".ots;
                while read -r line; do
                    line_basename="$(basename "$line")";
                    if [[ $line_basename == "$cand_prf_filename" ]]; then
                        flag_proof_in_subdir="true";
                        path_prf="$line";
                        vbm "DEBUG:proof found in subdir at:line:$line";
                        break;
                    fi;
                done < <(find "$dir_item" -mindepth 2 -maxdepth 2 -type f)
                if [[ $flag_proof_in_subdir == "true" ]]; then
                    ##### Proof file is in subdir
                    vbm "DEBUG:Proof file detected in subdir relative to source file (item)";
                    #path_prf="$path_prf"; # set in while loop
                    path_src="$item";
                    files_to_upgrade+=("$(printf "%s" "$path_prf")");
                    files_to_verify+=("$(printf "%s\n%s" "$path_src" "$path_prf")");
                else
                    ##### Proof file is not in subdir
                    vbm "DEBUG:Proof file not detected in subdir relative to source file (item).";
                    #### Stamp source file
                    vbm "DEBUG:Marking source file to be stamped.";
                    path_src="$item";
                    files_to_stamp+=("$(printf "%s" "$path_src")")
                fi;
                unset flag_proof_in_subdir;
            fi;            
        fi;
    done;
    unset path_src path_prf dir_item dir_parent cand_prf_filename cand_src_filename line_basename cand_src_path

    # Prune action lists.
    ## Sort and prune file action arrays
    ### files to upgrade
    while read -r -d $'\0' line; do
        vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line";
        files_to_upgrade_pruned+=("$line");
    done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1]
    if [[ $opVerbose == "true" ]]; then
        vbm "DEBUG:files_to_upgrade_pruned:";
        printf "%s\n" "${files_to_upgrade_pruned[@]}";
    fi;

    ### files to verify
    while read -r -d $'\0' line; do
        vbm "DEBUG:adding to files_to_verify_pruned:line:$line";
        files_to_verify_pruned+=("$line");
    done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1]
    if [[ $opVerbose == "true" ]]; then
        vbm "DEBUG:files_to_verify_pruned:";
        printf "%s\n\n" "${files_to_verify_pruned[@]}";
    fi;

    ### files to stamp
    while read -r -d $'\0' line; do
        vbm "DEBUG:adding to files_to_stamp_pruned:line:$line";
        files_to_stamp_pruned+=("$line");
    done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1]
    if [[ $opVerbose == "true" ]]; then
        vbm "DEBUG:files_to_stamp_pruned:";
        printf "%s\n" "${files_to_stamp_pruned[@]}";
    fi;
    
    # Act on files
    ## Assemble upgrade file commands
    for item in "${files_to_upgrade_pruned[@]}"; do
        path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].
        if [[ -z "$path_prf" ]]; then
            yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
            continue;
        fi;
        vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
        if [[ ! $option_dry_run == "true" ]]; then
            ### Try upgrade with known calendars in random order
            while read -r url; do
                vbm "DEBUG:Upgrading with calendar:url:$url";
                if [[ "$opVerbose" = "true" ]]; then
                    commands+=("cmdwrap ots -v -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break;
                else
                    commands+=("cmdwrap ots -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break;
                fi;
                #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
            done < <(printf "%s\n" "${calendars[@]}" | shuf);
        else
            yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
        fi;
    done;

    ## Assemble verify file commands
    for item in "${files_to_verify_pruned[@]}"; do
        path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
        path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].
        path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4].
        if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
            yell "ERROR:blank verify item encountered. Skipping:item:$item";
            continue;
        fi;
        vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
        vbm "DEBUG:    against proof file:          path_prf:$path_prf";
        if [[ ! $option_dry_run == "true" ]]; then
            ### Try verify with known calendars in random order
            while read -r url; do
                vbm "DEBUG:Verifying with calendar:url:$url";
                if [[ "$opVerbose" = "true" ]]; then
                    commands+=("cmdwrap ots -v -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break;
                else
                    commands+=("cmdwrap ots -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break;
                fi;
                #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
            done < <(printf "%s\n" "${calendars[@]}" | shuf);
        else
            yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
        fi;
    done;
    
    ## Assemble stamp file commands
    for item in "${files_to_stamp_pruned[@]}"; do
        path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
        path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4].
        if [[ -z "$path_src" ]]; then
            yell "ERROR:blank stamp item encountered. Skipping:item:$item";
            continue;
        fi;
        vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
        if [[ ! $option_dry_run == "true" ]]; then
            if [[ "$opVerbose" = "true" ]]; then
                commands+=("cmdwrap ots -v stamp \"$path_src_sesc\"");
            else
                commands+=("cmdwrap ots stamp \"$path_src_sesc\"");
            fi;
            #ots stamp "$path_src";
        else
            yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";
        fi;
    done;

    ## Run commands
    yell "DEBUG:commands:$(printf "%s\n" "${commands[@]}")";
    printf "%s\n" "${commands[@]}" | parallel --group;

}; # main program

# Run program
main "$@";