X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/blobdiff_plain/3a8398d26ae4b63193df831da2e4d33c26d19a53..105fa8b43670a1e5d7e21054f25cd7c318f0551e:/user/bkots?ds=inline diff --git a/user/bkots b/user/bkots old mode 100644 new mode 100755 index dba0ef5..acea7f5 --- a/user/bkots +++ b/user/bkots @@ -1,21 +1,29 @@ #!/usr/bin/env bash +# Desc: Recursively timestamp files with OpenTimestamp +# Usage: bkots [PATH] +# Version: 3.0.0 # Define variables +declare -g max_job_count="2"; # default max job count +declare -g max_batch_size="500"; # default xargs batch size for upgrading and stamping +declare -g age_threshold="60"; # min age to add file; seconds; +declare -g stamp_throttle="0.1"; # seconds delay between stamps +declare -g stamp_throttle_fail="1"; # seconds delay if stamp errors declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status declare -ag arrayPosArgs # Associative array for processArgs() function -declare -g ots_delay; ots_delay=1 # minimum time in seconds between ots operations declare -ag calendars; calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); +export OTS_CALS; OTS_CALS="$(printf '%s\n' "${calendars[@]}")"; # convert calendars array into string list # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails checkapp() { # Desc: If arg is a command, save result in assoc array 'appRollCall' # Usage: checkapp arg1 arg2 arg3 ... @@ -199,8 +207,8 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 0.0.3 -Copyright (C) 2022 Steven Baltakatei Sandoval +bkots 3.0.0 +Copyright (C) 2026 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. @@ -238,8 +246,14 @@ showUsage() { --include-dotfiles Include files and directories starting with '.' (not included by default). + -j, --jobs + Specify simultaneous job count (default: 2) + -n, --batch-size + Specify file batch sizes for upgrade and stamp operations (default:500) -r, --recursive Consider files in dirs recursively. + --verify + Verify OTS files. --version Display script version. -v, --verbose @@ -260,11 +274,39 @@ showUsage() { themselves are also ignored by default (e.g. '/home/user/.gitconfig'). + Files modified less than 1 minute ago are ignored. + EXAMPLES: bkots -v foo.txt bkots foo.txt bar.pdf /home/username/Pictures/ EOF } # Display information on how to use this script. +count_jobs() { + # Desc: Count and return total number of jobs + # Usage: count_jobs + # Input: None. + # Output: stdout integer number of jobs + # Depends: Bash 5.1.16 + # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; + # Version: 0.0.1 + + local job_count; + job_count="$(jobs -r | wc -l | tr -d ' ' )"; + #yell "DEBUG:job_count:$job_count"; + if [[ -z $job_count ]]; then job_count="0"; fi; + echo "$job_count"; +}; # Return number of background jobs +wait_for_jobslot() { + # Desc: Does not return until count_jobs() falls below $max_job_count + # Input: var max_job_count + # Output: return code 0 + # Depends: count_jobs(), yell(); + while [[ $(count_jobs) -ge $max_job_count ]]; do + printf "\r%d:Working..." "$SECONDS"; + sleep 1; + done; + return 0; +}; processArgs() { # Desc: Processes arguments provided to script. # Usage: processArgs "$@" @@ -299,9 +341,30 @@ processArgs() { --include-dotfiles) # Include dotfiles option_include_dotfiles="true"; vbm "DEBUG:Option enabled:include dotfiles";; + -j | --jobs) # Specify max_job_count + if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then + max_job_count="$2"; + vbm "STATUS:Max job count set to:$max_job_count"; + shift; + else + showUsage; + die "FATAL:Invalid job count:$2"; + fi;; + -n | --batch-size) + if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then + max_batch_size="$2"; + vbm "STATUS:Max batch size set to:$max_batch_size"; + shift; + else + showUsage; + die "FATAL:Invalid xargs file batch size:$2"; + fi;; -r | --recursive) # Specify recursive option option_recursive="true"; - vbm "DEBUG:option enabled:include files in dirs recursively";; + vbm "DEBUG:Option enabled:include files in dirs recursively";; + --verify) + option_verify="true"; + vbm "DEBUG:Option enabled:verify OTS files";; --version) showVersion; exit 1;; # Show version -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1]. --) # End of all options. See [2]. @@ -333,7 +396,7 @@ get_parent_dirnames() { # Input: arg1 input path # Output: stdout newline-delimited list of parent dirs # Version: 0.0.1 - # Depends: yell(), die(), try() + # Depends: yell(), die(), must() local path # Check input @@ -357,12 +420,15 @@ main() { # - for each file in each dir specified in arrayPosArgs array # Output file created alongside each file or in output directory specified by pathDirIn1 # Usage: main "$@"; - # Input: arrayPosArgs array with positional arguments - # pathDirOut1 path for output `.ots` files (if pathDirOut1 is specified and is a path) + # Input: arrayPosArgs array with positional arguments + # pathDirOut1 path for output `.ots` files (if pathDirOut1 is specified and is a path) + # age_threshold var: mininum age in seconds to timestamp file + # Output: file(s) creates `.ots` file alongside specified files - # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort) + # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822 # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194 # [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028 + # [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377 local -a file_list file_list_pruned; local -a files_to_verify files_to_upgrade files_to_stamp local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned @@ -371,22 +437,18 @@ main() { processArgs "$@"; # Check dependencies - if ! checkapp ots find; then + if ! checkapp ots find parallel; then displayMissing; die "FATAL:Missing dependencies."; fi; - # Check arguments - ## Mark if output dir option specified - if [[ -v pathDirOut1 ]]; then - vbm "DEBUG:output directory specified:pathDirOut1:$pathDirOut1"; - if [[ -d $pathDirOut1 ]]; then - vbm "DEBUG:pathDirOut1:$pathDirOut1"; - config_output_dir="true"; - else - die "ERROR:Not a dir:$pathDirOut1"; + # Check arguments + for arg in "${arrayPosArgs[@]}"; do + arg="$(readlink -f "$arg")"; + if ! { [[ -d $arg ]] || [[ -f $arg ]]; }; then + die "FATAL:Not a file or dir:arg:$arg"; fi; - fi; + done; # Display ots details vbm "$(type ots)"; # show how 'ots' is defined @@ -425,7 +487,7 @@ main() { done < <(find "$item" -maxdepth 1 -type f); fi; else - die "ERROR:Not a file or dir:item:$item"; + die "FATAL:Not a file or dir:item:$item"; fi; done; if [[ $opVerbose == "true" ]]; then @@ -435,44 +497,65 @@ main() { # Prune file_list for item in "${file_list[@]}"; do - if ! [[ $option_include_dotfiles == "true" ]]; then - ## Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config') - unset flag_contains_dotfile_parent; - while read -r line; do - ### Check line from output of get_parent_dirnames - pattern="^\."; - if [[ $line =~ $pattern ]]; then - #### line starts with '.' - vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item"; - vbm "DEBUG:Dotfile in path:item:$item"; - vbm "DEBUG:Dotfile parent:line:$line"; - flag_contains_dotfile_parent="true"; - break - fi; - done < <(get_parent_dirnames "$item"); - if [[ $flag_contains_dotfile_parent == "true" ]]; then - unset flag_contains_dotfile_parent; - continue; # skip to next item (i.e. don't add to file_list_pruned) - fi; + ## Ignore files that end in '.ots.bak'. + if [[ $item =~ '.ots.bak'$ ]]; then + vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item"; + continue; # skip to next item + fi; - ## Ignore dotfiles themselves - item_basename="$(basename "$item")"; - pattern="^\."; - if [[ $item_basename =~ $pattern ]]; then - vbm "INFO :Skipping dotfile:item:$item"; - continue; # skip to next item + ## Ignore dotfiles + if ! [[ $option_include_dotfiles == "true" ]]; then + ### Ignore files if '/.' contained within canonical path + pattern="/\."; # a dot after a forward slash + if [[ $item =~ $pattern ]]; then + continue; fi; + + # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config') + # unset flag_contains_dotfile_parent; + # while read -r line; do + # #### Check line from output of get_parent_dirnames + # pattern="^\."; + # if [[ $line =~ $pattern ]]; then + # ##### line starts with '.' + # vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item"; + # vbm "DEBUG:Dotfile in path:item:$item"; + # vbm "DEBUG:Dotfile parent:line:$line"; + # flag_contains_dotfile_parent="true"; + # break + # fi; + # done < <(get_parent_dirnames "$item"); + # if [[ $flag_contains_dotfile_parent == "true" ]]; then + # unset flag_contains_dotfile_parent; + # continue; # skip to next item (i.e. don't add to file_list_pruned) + # fi; + + # ### Ignore dotfiles themselves + # item_basename="$(basename "$item")"; + # pattern="^\."; + # if [[ $item_basename =~ $pattern ]]; then + # vbm "INFO :Skipping dotfile:item:$item"; + # continue; # skip to next item + # fi; fi; - + ## Ignore files with newlines present in filename. See [2]. if [[ $item =~ $'\n' ]]; then - yell "INFO :Skipping file name with newline:$item"; + vbm "DEBUG:Skipping file name with newline:$item"; continue; # skip to next item fi; ## Ignore files that end in '~'. if [[ $item =~ ~$ ]]; then - yell "INFO :Skipping file ending in tilde:$item"; + vbm "DEBUG:Skipping file ending in tilde:$item"; + continue; # skip to next item + fi; + + ## Ignore files modified less than age_threshold. See [3]. + time_now="$(date +%s)"; # epoch seconds + item_age="$(stat -c %Y "$item")"; # age in seconds + if [[ $(( time_now - item_age )) -lt "$age_threshold" ]]; then + yell "INFO :Skipping file modified less than $age_threshold seconds ago:item:$item"; continue; # skip to next item fi; @@ -579,6 +662,7 @@ main() { ## Sort and prune file action arrays ### files to upgrade while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_upgrade_pruned:line:$line"; files_to_upgrade_pruned+=("$line"); done < <(printf "%s\0" "${files_to_upgrade[@]}" | sort -zu | shuf -z); # See [1] @@ -589,6 +673,7 @@ main() { ### files to verify while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_verify_pruned:line:$line"; files_to_verify_pruned+=("$line"); done < <(printf "%s\0" "${files_to_verify[@]}" | sort -zu | shuf -z); # See [1] @@ -599,6 +684,7 @@ main() { ### files to stamp while read -r -d $'\0' line; do + if [[ -z $line ]]; then continue; fi; # Skip empty lines vbm "DEBUG:adding to files_to_stamp_pruned:line:$line"; files_to_stamp_pruned+=("$line"); done < <(printf "%s\0" "${files_to_stamp[@]}" | sort -zu | shuf -z); # See [1] @@ -606,68 +692,95 @@ main() { vbm "DEBUG:files_to_stamp_pruned:"; printf "%s\n" "${files_to_stamp_pruned[@]}"; fi; - + # Act on files - ## Upgrade files - for item in "${files_to_upgrade_pruned[@]}"; do - path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_prf" ]]; then - yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - #ots upgrade "$path_prf"; - for url in "${calendars[@]}"; do - ots -l "$url" --no-default-whitelist upgrade "$path_prf"; - done; + ## Assemble and execute upgrade file commands + if [[ ${#files_to_upgrade_pruned[@]} -gt 0 ]]; then + if [[ $option_dry_run == "true" ]]; then + yell "DEBUG:DRY RUN:Not running:\"ots upgrade\""; else - yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; + ### Pick random calendar + function get_calurl() { + # Desc: Returns random calendar url + # Input: string OTS_CALS newline-delimited list of calendars + # Output: stdout single calendar url + printf "%s" "$OTS_CALS" | shuf -n1; + }; # print random calendar url to stdout + export -f get_calurl; + + ### Perform upgrade command in xargs batches + printf '%s\0' "${files_to_upgrade_pruned[@]}" \ + | shuf -z \ + | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots --no-default-whitelist -l "$(get_calurl)" upgrade "$@";' _ ; fi; - sleep "$ots_delay"; - done; + fi; - ## Verify files - for item in "${files_to_verify_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then - yell "ERROR:blank verify item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; - vbm "DEBUG: against proof file: path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - #ots verify -f "$path_src" "$path_prf"; - for url in "${calendars[@]}"; do - ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf"; - done; - else - yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; - fi; - sleep "$ots_delay"; - done; + # ## Assemble and execute verify file commands + if [[ $option_verify == "true" ]]; then + for item in "${files_to_verify_pruned[@]}"; do + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then + yell "ERROR:blank verify item encountered. Skipping:item:$item"; + return 1; # would have been `continue` were it not in a subshell + fi; + vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; + vbm "DEBUG: against proof file: path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try verify with known calendars in random order + while read -r url; do + vbm "DEBUG:Verifying with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; + fi; + } & + done; + + ## Wait for jobs to finish. + wait; + else + vbm "DEBUG:Skipping operation:ots verify."; + fi; - ## Stamp files - for item in "${files_to_stamp_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_src" ]]; then - yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; - if [[ ! $option_dry_run == "true" ]]; then - #ots stamp "$item"; - for url in "${calendars[@]}"; do - ots -l "$url" --no-default-whitelist verify stamp "$item"; - done; + ## Assemble and execute stamp file commands + if [[ ${#files_to_stamp_pruned[@]} -gt 0 ]]; then + if [[ $option_dry_run == "true" ]]; then + yell "DEBUG:DRY RUN:Not running:\"ots stamp\""; else - yell "DEBUG:DRY RUN:Not running:\"ots stamp $item\""; + function get_calurl() { + # Desc: Returns random calendar url + # Input: string OTS_CALS newline-delimited list of calendars + # Output: stdout single calendar url + printf "%s" "$OTS_CALS" | shuf -n1; + }; # print random calendar url to stdout + export -f get_calurl; + + ### Perform stamp command in xargs batches + printf '%s\0' "${files_to_stamp_pruned[@]}" \ + | shuf -z \ + | xargs -0 -r -n "$max_batch_size" -P "$max_job_count" bash -c 'ots stamp "$@";' _ ; fi; - sleep "$ots_delay"; - done; + fi; + }; # main program # Run program main "$@"; +exit 0;