#!/usr/bin/env bash
# Define variables
+declare -g max_job_count="2"; # default max job count
+declare -g age_threshold="60"; # min age to add file; seconds;
declare -Ag appRollCall # Associative array for storing app status
declare -Ag fileRollCall # Associative array for storing file status
declare -Ag dirRollCall # Associative array for storing dir status
declare -ag arrayPosArgs # Associative array for processArgs() function
-declare -g ots_delay; ots_delay=1 # minimum time in seconds between ots operations
declare -ag calendars;
calendars+=("https://finney.calendar.eternitywall.com");
calendars+=("https://btc.calendar.catallaxy.com");
# Declare functions
yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
-try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
+must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
checkapp() {
# Desc: If arg is a command, save result in assoc array 'appRollCall'
# Usage: checkapp arg1 arg2 arg3 ...
vbm "DEBUG:showVersion function called."
cat <<'EOF'
-bkots 0.0.4
+bkots 2.1.1
Copyright (C) 2022 Steven Baltakatei Sandoval
License GPLv3: GNU GPL version 3
This is free software; you are free to change and redistribute it.
--include-dotfiles
Include files and directories starting with '.' (not
included by default).
+ -j, --jobs
+ Specify simultaneous job count (default: 2)
-r, --recursive
Consider files in dirs recursively.
--version
themselves are also ignored by default
(e.g. '/home/user/.gitconfig').
+ Files modified less than 1 minute ago are ignored.
+
EXAMPLES:
bkots -v foo.txt
bkots foo.txt bar.pdf /home/username/Pictures/
EOF
} # Display information on how to use this script.
+count_jobs() {
+ # Desc: Count and return total number of jobs
+ # Usage: count_jobs
+ # Input: None.
+ # Output: stdout integer number of jobs
+ # Depends: Bash 5.1.16
+ # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+ # Version: 0.0.1
+
+ local job_count;
+ job_count="$(jobs -r | wc -l | tr -d ' ' )";
+ #yell "DEBUG:job_count:$job_count";
+ if [[ -z $job_count ]]; then job_count="0"; fi;
+ echo "$job_count";
+}; # Return number of background jobs
+wait_for_jobslot() {
+ # Desc: Does not return until count_jobs() falls below $max_job_count
+ # Input: var max_job_count
+ # Output: return code 0
+ # Depends: count_jobs(), yell();
+ while [[ $(count_jobs) -ge $max_job_count ]]; do
+ printf "\r%d:Working..." "$SECONDS";
+ sleep 1;
+ done;
+ return 0;
+};
processArgs() {
# Desc: Processes arguments provided to script.
# Usage: processArgs "$@"
--include-dotfiles) # Include dotfiles
option_include_dotfiles="true";
vbm "DEBUG:Option enabled:include dotfiles";;
+ -j | --jobs) # Specify max_job_count
+ if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+ max_job_count="$2";
+ vbm "STATUS:Max job count set to:$max_job_count";
+ shift;
+ else
+ showUsage;
+ die "FATAL:Invalid job count:$2";
+ fi;;
-r | --recursive) # Specify recursive option
option_recursive="true";
vbm "DEBUG:option enabled:include files in dirs recursively";;
# Input: arg1 input path
# Output: stdout newline-delimited list of parent dirs
# Version: 0.0.1
- # Depends: yell(), die(), try()
+ # Depends: yell(), die(), must()
local path
# Check input
# - for each file in each dir specified in arrayPosArgs array
# Output file created alongside each file or in output directory specified by pathDirIn1
# Usage: main "$@";
- # Input: arrayPosArgs array with positional arguments
- # pathDirOut1 path for output `.ots` files (if pathDirOut1 is specified and is a path)
+ # Input: arrayPosArgs array with positional arguments
+ # pathDirOut1 path for output `.ots` files (if pathDirOut1 is specified and is a path)
+ # age_threshold var: mininum age in seconds to timestamp file
+
# Output: file(s) creates `.ots` file alongside specified files
- # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort)
+ # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822
# Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194
# [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028
+ # [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377
local -a file_list file_list_pruned;
local -a files_to_verify files_to_upgrade files_to_stamp
local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned
processArgs "$@";
# Check dependencies
- if ! checkapp ots find; then
+ if ! checkapp ots find parallel; then
displayMissing;
die "FATAL:Missing dependencies.";
fi;
- # Check arguments
- ## Mark if output dir option specified
- if [[ -v pathDirOut1 ]]; then
- vbm "DEBUG:output directory specified:pathDirOut1:$pathDirOut1";
- if [[ -d $pathDirOut1 ]]; then
- vbm "DEBUG:pathDirOut1:$pathDirOut1";
- config_output_dir="true";
- else
- die "ERROR:Not a dir:$pathDirOut1";
+ # Check arguments
+ for arg in "${arrayPosArgs[@]}"; do
+ arg="$(readlink -f "$arg")";
+ if ! { [[ -d $arg ]] || [[ -f $arg ]]; }; then
+ die "FATAL:Not a file or dir:arg:$arg";
fi;
- fi;
+ done;
# Display ots details
vbm "$(type ots)"; # show how 'ots' is defined
done < <(find "$item" -maxdepth 1 -type f);
fi;
else
- die "ERROR:Not a file or dir:item:$item";
+ die "FATAL:Not a file or dir:item:$item";
fi;
done;
if [[ $opVerbose == "true" ]]; then
# Prune file_list
for item in "${file_list[@]}"; do
- if ! [[ $option_include_dotfiles == "true" ]]; then
- ## Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
- unset flag_contains_dotfile_parent;
- while read -r line; do
- ### Check line from output of get_parent_dirnames
- pattern="^\.";
- if [[ $line =~ $pattern ]]; then
- #### line starts with '.'
- vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
- vbm "DEBUG:Dotfile in path:item:$item";
- vbm "DEBUG:Dotfile parent:line:$line";
- flag_contains_dotfile_parent="true";
- break
- fi;
- done < <(get_parent_dirnames "$item");
- if [[ $flag_contains_dotfile_parent == "true" ]]; then
- unset flag_contains_dotfile_parent;
- continue; # skip to next item (i.e. don't add to file_list_pruned)
- fi;
+ ## Ignore files that end in '.ots.bak'.
+ if [[ $item =~ '.ots.bak'$ ]]; then
+ vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item";
+ continue; # skip to next item
+ fi;
- ## Ignore dotfiles themselves
- item_basename="$(basename "$item")";
- pattern="^\.";
- if [[ $item_basename =~ $pattern ]]; then
- vbm "INFO :Skipping dotfile:item:$item";
- continue; # skip to next item
+ ## Ignore dotfiles
+ if ! [[ $option_include_dotfiles == "true" ]]; then
+ ### Ignore files if '/.' contained within canonical path
+ pattern="/\."; # a dot after a forward slash
+ if [[ $item =~ $pattern ]]; then
+ continue;
fi;
+
+ # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config')
+ # unset flag_contains_dotfile_parent;
+ # while read -r line; do
+ # #### Check line from output of get_parent_dirnames
+ # pattern="^\.";
+ # if [[ $line =~ $pattern ]]; then
+ # ##### line starts with '.'
+ # vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item";
+ # vbm "DEBUG:Dotfile in path:item:$item";
+ # vbm "DEBUG:Dotfile parent:line:$line";
+ # flag_contains_dotfile_parent="true";
+ # break
+ # fi;
+ # done < <(get_parent_dirnames "$item");
+ # if [[ $flag_contains_dotfile_parent == "true" ]]; then
+ # unset flag_contains_dotfile_parent;
+ # continue; # skip to next item (i.e. don't add to file_list_pruned)
+ # fi;
+
+ # ### Ignore dotfiles themselves
+ # item_basename="$(basename "$item")";
+ # pattern="^\.";
+ # if [[ $item_basename =~ $pattern ]]; then
+ # vbm "INFO :Skipping dotfile:item:$item";
+ # continue; # skip to next item
+ # fi;
fi;
-
+
## Ignore files with newlines present in filename. See [2].
if [[ $item =~ $'\n' ]]; then
- yell "INFO :Skipping file name with newline:$item";
+ vbm "DEBUG:Skipping file name with newline:$item";
continue; # skip to next item
fi;
## Ignore files that end in '~'.
if [[ $item =~ ~$ ]]; then
- yell "INFO :Skipping file ending in tilde:$item";
+ vbm "DEBUG:Skipping file ending in tilde:$item";
+ continue; # skip to next item
+ fi;
+
+ ## Ignore files modified less than age_threshold. See [3].
+ time_now="$(date +%s)"; # epoch seconds
+ item_age="$(stat -c %Y "$item")"; # age in seconds
+ if [[ $(( time_now - item_age )) -lt "$age_threshold" ]]; then
+ yell "INFO :Skipping file modified less than $age_threshold seconds ago:item:$item";
continue; # skip to next item
fi;
vbm "DEBUG:files_to_stamp_pruned:";
printf "%s\n" "${files_to_stamp_pruned[@]}";
fi;
-
+
# Act on files
- ## Upgrade files
+ ## Assemble and execute upgrade file commands
for item in "${files_to_upgrade_pruned[@]}"; do
- path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
- if [[ -z "$path_prf" ]]; then
- yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
- if [[ ! $option_dry_run == "true" ]]; then
- #ots upgrade "$path_prf";
- for url in "${calendars[@]}"; do
- ots -l "$url" --no-default-whitelist upgrade "$path_prf";
- done;
- else
- yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
- fi;
- sleep "$ots_delay";
+ wait_for_jobslot && {
+ path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))";
+ if [[ -z "$path_prf" ]]; then
+ yell "ERROR:blank upgrade item encountered. Skipping:item:$item";
+ return 1; # would have been `continue` were it not in a subshell
+ fi;
+ vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf";
+ if [[ ! $option_dry_run == "true" ]]; then
+ ### Try upgrade with known calendars in random order
+ while read -r url; do
+ vbm "DEBUG:Upgrading with calendar:url:$url";
+
+ #### assemble command
+ local -a cmd_temp;
+ cmd_temp=("ots");
+ if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+ cmd_temp+=("-l" "$url" "--no-default-whitelist");
+ cmd_temp+=("upgrade" "$path_prf");
+ if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+ #### execute command
+ "${cmd_temp[@]}";
+ unset cmd_temp;
+ break;
+ #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break;
+ done < <(printf "%s\n" "${calendars[@]}" | shuf);
+ else
+ yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\"";
+ fi;
+ } &
done;
- ## Verify files
+ ## Assemble and execute verify file commands
for item in "${files_to_verify_pruned[@]}"; do
- path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
- path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
- if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
- yell "ERROR:blank verify item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
- vbm "DEBUG: against proof file: path_prf:$path_prf";
- if [[ ! $option_dry_run == "true" ]]; then
- #ots verify -f "$path_src" "$path_prf";
- for url in "${calendars[@]}"; do
- ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf";
- done;
- else
- yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
- fi;
- sleep "$ots_delay";
+ wait_for_jobslot && {
+ path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+ path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))";
+ if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then
+ yell "ERROR:blank verify item encountered. Skipping:item:$item";
+ return 1; # would have been `continue` were it not in a subshell
+ fi;
+ vbm "DEBUG:Attempting to verify source file:path_src:$path_src";
+ vbm "DEBUG: against proof file: path_prf:$path_prf";
+ if [[ ! $option_dry_run == "true" ]]; then
+ ### Try verify with known calendars in random order
+ while read -r url; do
+ vbm "DEBUG:Verifying with calendar:url:$url";
+
+ #### assemble command
+ local -a cmd_temp;
+ cmd_temp=("ots");
+ if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+ cmd_temp+=("-l" "$url" "--no-default-whitelist");
+ cmd_temp+=("verify" "-f" "$path_src" "$path_prf");
+ if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+ #### execute command
+ "${cmd_temp[@]}";
+ unset cmd_temp;
+ break;
+ #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break;
+ done < <(printf "%s\n" "${calendars[@]}" | shuf);
+ else
+ yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\"";
+ fi;
+ } &
done;
-
- ## Stamp files
+
+ ## Assemble and execute stamp file commands
for item in "${files_to_stamp_pruned[@]}"; do
- path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
- if [[ -z "$path_src" ]]; then
- yell "ERROR:blank stamp item encountered. Skipping:item:$item";
- continue;
- fi;
- vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
- if [[ ! $option_dry_run == "true" ]]; then
- #ots stamp "$item";
- for url in "${calendars[@]}"; do
- ots -l "$url" --no-default-whitelist stamp "$item";
- done;
- else
- yell "DEBUG:DRY RUN:Not running:\"ots stamp $item\"";
- fi;
- sleep "$ots_delay";
+ wait_for_jobslot && {
+ path_src="$(cut -d $'\n' -f1 < <(echo "$item"))";
+ if [[ -z "$path_src" ]]; then
+ yell "ERROR:blank stamp item encountered. Skipping:item:$item";
+ return 1; # would have been `continue` were it not in a subshell
+ fi;
+ vbm "DEBUG:Attempting to stamp source file:path_src:$path_src";
+ if [[ ! $option_dry_run == "true" ]]; then
+
+ #### assemble command
+ local -a cmd_temp;
+ cmd_temp=("ots");
+ if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi;
+ cmd_temp+=("stamp" "$path_src");
+ if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi;
+
+ #### execute command
+ "${cmd_temp[@]}";
+ unset cmd_temp;
+ #ots stamp "$path_src";
+ else
+ yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\"";
+ fi;
+ } &
done;
+ ## Wait for jobs to finish.
+ wait;
}; # main program
# Run program
main "$@";
+exit 0;
projects / BK-2020-03.git / blobdiff