X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/blobdiff_plain/aa8be4bf5eaeac58707062fe0a5b506eea9cf3e6..f943b9f29ad49237b05278807f86ae93d55655a4:/user/bkotslu diff --git a/user/bkotslu b/user/bkotslu index f2e1220..f7bf9cc 100755 --- a/user/bkotslu +++ b/user/bkotslu @@ -1,9 +1,16 @@ #!/bin/bash # Desc: Utility for backing up and retrieving ots files -# Version: 0.0.1 +# Usage: bkotslu -I [dir] +# Version: 0.1.3 +# Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org ) +# GNU Coreutils 8.32 +# NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid. OTS_FCACHE_DIR="$HOME/.cache/bkotslu/"; -MAX_FIND_DEPTH=12; +MAX_FIND_DEPTH=1; +MAX_JOBS=32; + +declare -a pathsFilesIn; yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status @@ -43,11 +50,15 @@ showUsage() { Display script version. -v, --verbose Display debugging info. - -i, --input-file - Define input file path for file to add ots file to. + -i, --input-file [FILE] + Provide path for file to submit/lookup OTS files for + -I, --input-dir [DIR] + Provide dir containing files to submit/lookup OTS files for -O, --output-dir Define output directory path for storing ots backup files. DEFAULT: $HOME/.cache/bkotslu/ + -r, --recursive + Follow subdirectories in provided directories. -- Indicate end of options. @@ -55,7 +66,7 @@ showUsage() { Hash foo.txt and lookup matching ots file bkotslu -i foo.txt - Archive ots files from home directory + Store and lookup older ots files of a directory bkotslu -I $HOME/ EOF @@ -89,17 +100,27 @@ EOF vbm "DEBUG:showVersion function ended." return 0; # Function finished. }; # Display script version. +checkDepends() { + vbm "STATUS:Starting checkDepends()"; + if ! command -v sha256sum 1>/dev/urandom 2>&1; then + die "FATAL:sha256sum not available."; fi; + if ! command -v grep 1>/dev/urandom 2>&1; then + die "FATAL:grep not available."; fi; + if ! command -v find 1>/dev/urandom 2>&1; then + die "FATAL:find not available."; fi; +}; processArgs() { # Desc: Processes arguments provided to script. # Usage: processArgs "$@" # Version: 1.0.0 - # Input: "$@" (list of arguments provided to the function) + # Input: "$@" (list of arguments provided to the function) # Output: Sets following variables used by other functions: # opVerbose Indicates verbose mode enable status. (ex: "true", "false") - # pathDirOut1 Path to output directory. - # pathDirIn1 Path to input directory. - # pathFileIn1 Path to input file. + # pathOtsStore Path to output directory. + #X pathDirIn1 Path to input directory. + #X pathFileIn1 Path to input file. # arrayPosArgs Array of remaining positional argments + # pathsFilesIn input files # Depends: # yell() Displays messages to stderr. # vbm() Displays messsages to stderr if opVerbose set to "true". @@ -123,29 +144,34 @@ processArgs() { -h | --help) showUsage; exit 1;; # Display usage. --version) showVersion; exit 1;; # Show version -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1]. + -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";; -i | --input-file) # Define input file path - if [ -f "$2" ]; then # If $2 is file that exists, set pathFileIn1 to $2, pop $2. - pathFileIn1="$2"; - vbm "DEBUG:Input file pathFileIn1 set to:${pathFileIn1}"; + if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2. + pathsFilesIn+=("$(readlink -f "$2")"); + vbm "DEBUG:Added to pathsFilesIn array:$2"; shift; else - die "FATAL: Specified input file does not exist:$2"; + die "FATAL:The provided input file does not exist:$2"; fi;; -I | --input-dir) # Define input directory path - if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirIn1 to $2, pop $2. - pathDirIn1="$2"; - vbm "DEBUG:Input directory pathDirIn1 set to:${pathDirIn1}"; - shift; + if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2. + while read -r file; do + file="$(readlink -f "$file")"; + vbm "STATUS:Added to pathsFilesIn array:${file}"; + pathsFilesIn+=("$(readlink -f "$file")"); + done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f); + vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2"; + shift; else # Display error if $2 is not a valid dir. - die "FATAL:Specified input directory does not exist:$2"; + die "FATAL:The specified input directory does not exist:$2"; fi;; -O | --output-dir) # Define output directory path - if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirOut1 to $2, pop $2 - pathDirOut1="$2"; - vbm "DEBUG:Output directory pathDirOut1 set to:${pathDirOut1}"; - shift; + if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2 + pathOtsStore="$2"; + vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}"; + shift; else - die "FATAL:Specified output directory is not valid:$2"; + die "FATAL:The specified output directory is not valid:$2"; fi;; --) # End of all options. See [2]. shift; @@ -161,15 +187,15 @@ processArgs() { done; ## Identify ots file cache dir - if [[ -z "$pathDirOut1" ]]; then + if [[ -z "$pathOtsStore" ]]; then vbm "STATUS:No output directory for caching OTS files specified."; - pathDirOut1="$OTS_FCACHE_DIR"; - vbm "STATUS:Assuming OTS files to be cached in:${pathDirOut1}"; + pathOtsStore="$OTS_FCACHE_DIR"; + vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}"; fi; ## Create cache dir if necessary - if [[ ! -d "$pathDirOut1" ]]; then - vbm "STATUS:Creating OTS file cache directory:${pathDirOut1}"; - must mkdir -p "$pathDirOut1"; + if [[ ! -d "$pathOtsStore" ]]; then + vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}"; + must mkdir -p "$pathOtsStore"; fi; # End function @@ -186,19 +212,20 @@ get_ots_filehash() { # Input: arg1 OTS file path # Output: stdout sha256 file hash (lowercase) local output; - vbm "DEBUG:Starting get_ots_filehash() on:$1"; + + re='[0-9a-f]{64}'; if output="$( "$(which ots)" info "$1" | \ grep -E "^File sha256 hash: " | \ head -n1 | \ sed -E -e 's/(^File sha256 hash: )([0-9a-f]+$)/\2/g'; )" && \ - [[ -n "$output" ]]; then - vbm "STATUS:Read file hash via ots from:$1"; + [[ -n "$output" ]] && \ + [[ "$output" =~ $re ]]; then + vbm "STATUS:Read file digest (${output}) via ots from:$1"; printf "%s" "$output"; return 0; else - yell "ERROR:Encountered problem getting file hash via ots from:$1"; - return 1; + die "ERROR:Encountered problem getting file hash via ots from:$1"; fi; }; # Gets hash of file from ots file get_ots_oldestblock() { @@ -211,54 +238,236 @@ get_ots_oldestblock() { # vbm() # BK-2020-03: yell() local output; + vbm "DEBUG:Starting get_ots_oldestblock() on:$1"; - vbm "DEBUG:Starting get_ots_oldestblock() on:$1"; + re='[0-9]+'; if output="$( "$(which ots)" info "$1" | \ grep -E "verify BitcoinBlockHeaderAttestation\([0-9]+\)" | \ sort | head -n1 | \ sed -E -e 's/(^ verify BitcoinBlockHeaderAttestation)\(([0-9]+)(\))/\2/g'; )" && \ - [[ -n "$output" ]]; then - vbm "STATUS:Retrieved Bitcoin block via ots from:$1"; + [[ -n "$output" ]] && \ + [[ "$output" =~ $re ]]; then + vbm "STATUS:Retrieved Bitcoin block (${output}) via ots from:$1"; printf "%s" "$output"; return 0; else - yell "ERROR:Encountered problem getting Bitcoin block number via ots from:$1"; - return 1; + die "ERROR:Encountered problem getting Bitcoin block number via ots from:$1"; fi; }; # Gets oldest Bitcoin block from ots file -cache_ots_file() { - # Desc: Scans and caches an OTS file for storing - if fhash="$(must get_ots_filehash "$line")" && \ - block="$(must get_ots_oldestblock "$line")"; then - fout="${fhash}_${block}.otsu"; # file name out - pout="${pathDirOut1}/${fout}"; # file path out - vbm "STATUS:Found OTS file at ${line} with hash ${fhash} and block ${block} and saving to ${pout}"; - must cp -n "$line" "$pout"; +store_ots_file() { + # Desc: Scans and stores an OTS file if none already stored + # Usage: store_ots_file FILE + # Example: store_ots_file foo.txt.ots + # Input: arg1 OTS file + # Output: exit code + local fin="$1"; + vbm "STATUS:Starting store_ots_file()"; + + # Check if provided OTS file exists + if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi; + + # Read file hash and oldest block from provided OTS file + if ! { fhash="$(must get_ots_filehash "$fin")" && \ + block="$(must get_ots_oldestblock "$fin")"; }; then + yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}"; + return 1; + fi; + vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}."; + + # Copy provided OTS if no matching OTS stored + fout="${fhash}_${block}.otsu"; # file name out + pout="${pathOtsStore}/${fout}"; # file path out + if [[ ! -f "$pout" ]]; then + vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}"; + must cp -n "$fin" "$pout"; + return 0; + else + vbm "STATUS:Stored OTS file with matching file hash and block number in file name found."; + fi; + + # Get block number for provided and stored OTS files. + if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then + yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )"; + fi; + re='[0-9]+'; + if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then + die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)"; + fi; + + # Copy provided OTS if matching OTS found stored but provided is older + if [[ "$blk_provid" -lt "$blk_stored" ]]; then + vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?"; + must mv "$pout" "${pout}--$(date +%s)"; + must cp "$fin" "$pout"; + return 0; else - yell "ERROR:Problem analyzing file with OpenTimestamps:${line}"; + vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file."; + fi; +}; # Stores provided OTS file is none already stored +get_sha256_digest() { + # Depends: GNU Coreutils 8.32 (sha256sum) + # Input: arg1 path file path + # Output: stdout str sha256 digest (lowercase hexadecimal) + vbm "DEBUG:Starting get_sha256_digest()"; + sha256sum "$1" | head -n1 | sed -E -e 's/(^[0-9a-f]{64})(.+)/\1/'; +}; +get_oldest_stored_ots_path() { + # Desc: Lookup most recent OTS file from storage + # Input: pathOtsStore var path to OTS storage dir + # arg1 str sha256 digest (lowercase hexadecimal) + # Output: stdout path OTS file with matching sha256 digest + vbm "DEBUG:Starting get_oldest_stored_ots_path()"; + local -a otsStorePaths; + local i_oldest; + + digest="$1"; + mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; ); + if [[ "${#otsStorePaths[@]}" -le 0 ]]; then + yell "NOTICE:No OTS file for digest ${digest} found in ${pathOtsStore}."; + return 1; fi; -}; # Scans a single OTS file + i_oldest=0; + re='[0-9]+'; + blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )"; + if ! [[ "$blockNumOldest" =~ $re ]]; then die "FATAL:Invalid block number:${blockNumOldest}"; fi; + for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do + blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )"; + if ! [[ "$blockNum" =~ $re ]]; then die "FATAL:Invalid block number:${blockNum}"; fi; + if [[ $blockNum -lt $blockNumOldest ]]; then + blockNumOldest=$blockNum; + i_oldest=$i; + fi; + done; + output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )"; -main() { - processArgs "$@"; + if [[ -n "$output" ]] && [[ -f "$output" ]]; then + vbm "STATUS:Found matching OTS file with digest ${digest} at:$output"; + printf "%s" "$output"; + return 0; + else + yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} ."; + return 1; + fi; +}; # Print to stdout path of OTS file with oldest block +get_block_num_from_stored_ots_path() { + # Desc: Return block number from stored OTS path + # Input: arg1 input file path + # Output: stdout block number (int) + # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'. + local fin fbase block re; + fpath="$1"; + fbase="$(basename "$fpath")"; + block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")"; + re='[0-9]+'; + if [[ "$block" =~ $re ]]; then + printf "%s" "$block"; + else + yell "ERROR:Invalid block number:$(declare -p fpath fbase block)"; + return 1; + fi; +}; # Print block number from stored OTS file +store_and_lookup() { + # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible + # Usage: store_and_lookup [path] + # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock() + local pathFileIn="$1"; + vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}"; + # Validate path + if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi; - # Scan provided dir for OTS files to cache. - if [[ -n "$pathDirIn1" ]]; then - while read -r line; do - line="$(readlink -f "$line")"; - cache_ots_file; - done < <(find "$pathDirIn1" -maxdepth "$MAX_FIND_DEPTH" -type f -name "*.ots"); + # Check for and store any OTS file attached to provided file + ## Check if provided file is an OTS file itself + if [[ "$pathFileIn" =~ \.ots$ ]]; then + vbm "STATUS:The provided file is itself an OTS file. Store OTS file only."; + store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file."; + return 0; + fi; + ## Check if provided file has an accompanying OTS file + if [[ -f "${pathFileIn}.ots" ]]; then + vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots"; + store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file."; fi; # Lookup OTS file from archive for provided file. - if [[ -n "$pathFileIn1" ]]; then - : + ## Get file hash + fhash="$(get_sha256_digest "$pathFileIn"; )"; + + ## Get stored OTS path if possible. + if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then + yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}"; + return 0; fi; + vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}."; + blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )"; + vbm "STATUS:The stored OTS file has block number ${blk_stored}."; + + ## Check for OTS file accompanying provided file + if [[ -f "${pathFileIn}.ots" ]]; then + vbm "STATUS:An OTS file is next to provided file ${pathFileIn}."; + blk_provid="$(must get_ots_oldestblock "${pathFileIn}.ots"; )"; + vbm "STATUS:The provided file's OTS file has block number ${blk_provid}"; + re='[0-9]+'; + if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then + die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)"; + fi; + if [[ "$blk_stored" -lt "$blk_provid" ]]; then + vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored})."; + if [[ ! -f "${pathFileIn}.ots.baku" ]]; then + must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \ + vbm "STATUS:Backed up existing OTS file."; + else + must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \ + yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present."; + fi; + must cp "$path_stored_ots" "${pathFileIn}.ots" && \ + vbm "STATUS:Replaced provided OTS file with stored OTS file."; + return 0; + else + yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} ."; + fi; + else + vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots"; + must cp "$path_stored_ots" "${pathFileIn}.ots"; + return 0; + fi; +}; # stores provided OTS files and retrieves older OTS files if available +count_jobs() { + # Desc: Count and return total number of jobs + # Usage: count_jobs + # Input: None. + # Output: stdout integer number of jobs + # Depends: Bash 5.1.16 + # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; + # Version: 0.0.1 - + local job_count; + job_count="$(jobs -r | wc -l | tr -d ' ' )"; + #yell "DEBUG:job_count:$job_count"; + if [[ -z $job_count ]]; then job_count="0"; fi; + echo "$job_count"; +}; # Return number of background jobs +main() { + checkDepends; + processArgs "$@"; + vbm "DEBUG:Starting rest of main()"; + vbm "$(declare -p pathsFilesIn)"; + # Process files from provided input args + for fpath in "${pathsFilesIn[@]}"; do + # throttle if too many jobs + if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then + sleep 0.01; + fi; + + # start new job + must store_and_lookup "$fpath" & + done; + wait; }; # main program main "$@"; + +# Author: Steven Baltakatei Sandoval +# License: GPLv3+