#!/bin/bash
# Desc: Utility for backing up and retrieving ots files
-# Version: 0.0.2
+# Usage: bkotslu -I [dir]
+# Version: 0.1.0
+# Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org )
+# GNU Coreutils 8.32
+# NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid.
OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";
-MAX_FIND_DEPTH=12;
+MAX_FIND_DEPTH=1;
+MAX_JOBS=32;
+
+declare -a pathsFilesIn;
yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
Display script version.
-v, --verbose
Display debugging info.
- -i, --input-file
- Define input file path for file to add ots file to.
+ -i, --input-file [FILE]
+ Provide path for file to submit/lookup OTS files for
+ -I, --input-dir [DIR]
+ Provide dir containing files to submit/lookup OTS files for
-O, --output-dir
Define output directory path for storing ots backup files.
DEFAULT: $HOME/.cache/bkotslu/
+ -r, --recursive
+ Follow subdirectories in provided directories.
--
Indicate end of options.
Hash foo.txt and lookup matching ots file
bkotslu -i foo.txt
- Archive ots files from home directory
+ Store and lookup older ots files of a directory
bkotslu -I $HOME/
EOF
}; # Display script version.
checkDepends() {
vbm "STATUS:Starting checkDepends()";
- if ! command -v sha256sum 1>/dev/random 2>&1; then
+ if ! command -v sha256sum 1>/dev/urandom 2>&1; then
die "FATAL:sha256sum not available."; fi;
- if ! command -v grep 1>/dev/random 2>&1; then
- die "FATAL:grep not available."; fi;
- if ! command -v find 1>/dev/random 2>&1; then
+ if ! command -v grep 1>/dev/urandom 2>&1; then
die "FATAL:grep not available."; fi;
+ if ! command -v find 1>/dev/urandom 2>&1; then
+ die "FATAL:find not available."; fi;
};
processArgs() {
# Desc: Processes arguments provided to script.
# Usage: processArgs "$@"
# Version: 1.0.0
- # Input: "$@" (list of arguments provided to the function)
+ # Input: "$@" (list of arguments provided to the function)
# Output: Sets following variables used by other functions:
# opVerbose Indicates verbose mode enable status. (ex: "true", "false")
- # pathDirOut1 Path to output directory.
- # pathDirIn1 Path to input directory.
- # pathFileIn1 Path to input file.
+ # pathOtsStore Path to output directory.
+ #X pathDirIn1 Path to input directory.
+ #X pathFileIn1 Path to input file.
# arrayPosArgs Array of remaining positional argments
+ # pathsFilesIn input files
# Depends:
# yell() Displays messages to stderr.
# vbm() Displays messsages to stderr if opVerbose set to "true".
-h | --help) showUsage; exit 1;; # Display usage.
--version) showVersion; exit 1;; # Show version
-v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
+ -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";;
-i | --input-file) # Define input file path
- if [ -f "$2" ]; then # If $2 is file that exists, set pathFileIn1 to $2, pop $2.
- pathFileIn1="$2";
- vbm "DEBUG:Input file pathFileIn1 set to:${pathFileIn1}";
+ if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2.
+ pathsFilesIn+=("$(readlink -f "$2")");
+ vbm "DEBUG:Added to pathsFilesIn array:$2";
shift;
else
- die "FATAL: Specified input file does not exist:$2";
+ die "FATAL:The provided input file does not exist:$2";
fi;;
-I | --input-dir) # Define input directory path
- if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirIn1 to $2, pop $2.
- pathDirIn1="$2";
- vbm "DEBUG:Input directory pathDirIn1 set to:${pathDirIn1}";
- shift;
+ if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2.
+ while read -r file; do
+ file="$(readlink -f "$file")";
+ vbm "STATUS:Added to pathsFilesIn array:${file}";
+ pathsFilesIn+=("$(readlink -f "$file")");
+ done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f);
+ vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2";
+ shift;
else # Display error if $2 is not a valid dir.
- die "FATAL:Specified input directory does not exist:$2";
+ die "FATAL:The specified input directory does not exist:$2";
fi;;
-O | --output-dir) # Define output directory path
- if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirOut1 to $2, pop $2
- pathDirOut1="$2";
- vbm "DEBUG:Output directory pathDirOut1 set to:${pathDirOut1}";
- shift;
+ if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2
+ pathOtsStore="$2";
+ vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}";
+ shift;
else
- die "FATAL:Specified output directory is not valid:$2";
+ die "FATAL:The specified output directory is not valid:$2";
fi;;
--) # End of all options. See [2].
shift;
done;
## Identify ots file cache dir
- if [[ -z "$pathDirOut1" ]]; then
+ if [[ -z "$pathOtsStore" ]]; then
vbm "STATUS:No output directory for caching OTS files specified.";
- pathDirOut1="$OTS_FCACHE_DIR";
- vbm "STATUS:Assuming OTS files to be cached in:${pathDirOut1}";
+ pathOtsStore="$OTS_FCACHE_DIR";
+ vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}";
fi;
## Create cache dir if necessary
- if [[ ! -d "$pathDirOut1" ]]; then
- vbm "STATUS:Creating OTS file cache directory:${pathDirOut1}";
- must mkdir -p "$pathDirOut1";
+ if [[ ! -d "$pathOtsStore" ]]; then
+ vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}";
+ must mkdir -p "$pathOtsStore";
fi;
# End function
return 1;
fi;
}; # Gets oldest Bitcoin block from ots file
-cache_ots_file() {
- # Desc: Scans and caches an OTS file for storing
- # Input: arg1 input file
+store_ots_file() {
+ # Desc: Scans and stores an OTS file if none already stored
+ # Usage: store_ots_file FILE
+ # Example: store_ots_file foo.txt.ots
+ # Input: arg1 OTS file
# Output: exit code
local fin="$1";
- vbm "STATUS:Starting cache_ots_file()";
-
- if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
- if fhash="$(must get_ots_filehash "$fin")" && \
- block="$(must get_ots_oldestblock "$fin")"; then
- fout="${fhash}_${block}.otsu"; # file name out
- pout="${pathDirOut1}/${fout}"; # file path out
- vbm "STATUS:Found OTS file at ${fin} with hash ${fhash} and block ${block}.";
- vbm "STATUS:Saving found OTS file to ${pout}";
+ vbm "STATUS:Starting store_ots_file()";
+
+ # Check if provided OTS file exists
+ if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
+
+ # Read file hash and oldest block from provided OTS file
+ if ! { fhash="$(must get_ots_filehash "$fin")" && \
+ block="$(must get_ots_oldestblock "$fin")"; }; then
+ yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
+ return 1;
+ fi;
+ vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}.";
+
+ # Copy provided OTS if no matching OTS stored
+ fout="${fhash}_${block}.otsu"; # file name out
+ pout="${pathOtsStore}/${fout}"; # file path out
+ if [[ ! -f "$pout" ]]; then
+ vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}";
must cp -n "$fin" "$pout";
return 0;
else
- yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
- return 1;
+ vbm "STATUS:Stored OTS file with matching file hash and block number in file name found.";
fi;
-}; # Scans a single OTS file
+
+ # Get block number for provided and stored OTS files.
+ if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then
+ yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )";
+ fi;
+
+ # Copy provided OTS if matching OTS found stored but provided is older
+ if [[ "$blk_provid" -lt "$blk_stored" ]]; then
+ vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?";
+ must mv "$pout" "${pout}--$(date +%s)";
+ must cp "$fin" "$pout";
+ return 0;
+ else
+ vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file.";
+ fi;
+}; # Stores provided OTS file is none already stored
get_sha256_digest() {
# Depends: GNU Coreutils 8.32 (sha256sum)
# Input: arg1 path file path
};
get_oldest_stored_ots_path() {
# Desc: Lookup most recent OTS file from storage
- # Input: pathDirOut1 var path to OTS storage dir
+ # Input: pathOtsStore var path to OTS storage dir
# arg1 str sha256 digest (lowercase hexadecimal)
# Output: stdout path OTS file with matching sha256 digest
vbm "DEBUG:Starting get_oldest_stored_ots_path()";
+ local -a otsStorePaths;
+ local i_oldest;
digest="$1";
- if output="$( find "$pathDirOut1" -type f -name "${digest}*" | \
- sort | \
- head -n1 )" && \
- [[ -n "$output" ]]; then
- output="$(readlink -f "$output")";
+ mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; );
+ if [[ "${#otsStorePaths[@]}" -le 0 ]]; then
+ yell "ERROR:No OTS file in OTS storage dir found. $(declare -p pathOtsStore digest otsStorePaths)";
+ return 1;
+ fi;
+ i_oldest=0;
+ blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )";
+ for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do
+ blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )";
+ if [[ $blockNum -lt $blockNumOldest ]]; then
+ blockNumOldest=$blockNum;
+ i_oldest=$i;
+ fi;
+ done;
+ output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )";
+
+ if [[ -n "$output" ]] && [[ -f "$output" ]]; then
vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
printf "%s" "$output";
return 0;
else
- yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathDirOut1}";
+ yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} .";
return 1;
fi;
-};
-main() {
- checkDepends;
- processArgs "$@";
- vbm "DEBUG:Starting rest of main()";
+}; # Print to stdout path of OTS file with oldest block
+get_block_num_from_stored_ots_path() {
+ # Desc: Return block number from stored OTS path
+ # Input: arg1 input file path
+ # Output: stdout block number (int)
+ # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'.
+ local fin fbase block re;
+ fpath="$1";
+ fbase="$(basename "$fpath")";
+ block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")";
+ re='[0-9]+';
+ if [[ "$block" =~ $re ]]; then
+ printf "%s" "$block";
+ else
+ yell "ERROR:Invalid block number:$(declare -p fpath fbase block)";
+ return 1;
+ fi;
+}; # Print block number from stored OTS file
+store_and_lookup() {
+ # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible
+ # Usage: store_and_lookup [path]
+ # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock()
+ local pathFileIn="$1";
+ vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}";
+
+ # Validate path
+ if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi;
- # Scan provided dir for OTS files to cache.
- if [[ -n "$pathDirIn1" ]]; then
- while read -r line; do
- line="$(readlink -f "$line")";
- cache_ots_file "$line";
- done < <(find "$pathDirIn1" -maxdepth "$MAX_FIND_DEPTH" -type f -name "*.ots");
+ # Check for and store any OTS file attached to provided file
+ ## Check if provided file is an OTS file itself
+ if [[ "$pathFileIn" =~ \.ots$ ]]; then
+ vbm "STATUS:The provided file is itself an OTS file. Store OTS file only.";
+ store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file.";
+ return 0;
+ fi;
+ ## Check if provided file has an accompanying OTS file
+ if [[ -f "${pathFileIn}.ots" ]]; then
+ vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots";
+ store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file.";
fi;
# Lookup OTS file from archive for provided file.
- if [[ -n "$pathFileIn1" ]]; then
- # Check for and store any OTS file attached to provided file
- if [[ -f "${pathFileIn1}.ots" ]]; then
- vbm "STATUS:OTS file accompanying provided file found:${pathFileIn1}.ots";
- cache_ots_file "${pathFileIn1}.ots" && vbm "STATUS:Stored provided file's OTS file.";
- fi;
-
- # Get file hash
- fhash="$(get_sha256_digest "$pathFileIn1"; )";
- if path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
- vbm "STATUS:Stored OTS found with matching hash.";
- blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
- vbm "STATUS:Stored OTS file has block number ${blk_stored}.";
- if [[ -f "${pathFileIn1}.ots" ]]; then
- vbm "STATUS:An OTS file is next to provided file.";
- blk_provid="$(get_ots_oldestblock "${pathFileIn1}.ots"; )";
- vbm "STATUS:Provided file's OTS file has block number ${blk_provid}";
- if [[ "$blk_stored" -lt "$blk_provid" ]]; then
- vbm "STATUS:Older timestamp in OTS store found. Replacing ${pathFileIn1} (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
- #must mv "$pathFileIn1" "${pathFileIn1}.baku";
- #must cp "$path_stored_ots" "$pathFileIn1";
- else
- yell "STATUS:Stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn1}";
- fi;
+ ## Get file hash
+ fhash="$(get_sha256_digest "$pathFileIn"; )";
+
+ ## Get stored OTS path if possible.
+ if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
+ yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}";
+ return 0;
+ fi;
+ vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}.";
+ blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
+ vbm "STATUS:The stored OTS file has block number ${blk_stored}.";
+
+ ## Check for OTS file accompanying provided file
+ if [[ -f "${pathFileIn}.ots" ]]; then
+ vbm "STATUS:An OTS file is next to provided file ${pathFileIn}.";
+ blk_provid="$(get_ots_oldestblock "${pathFileIn}.ots"; )";
+ vbm "STATUS:The provided file's OTS file has block number ${blk_provid}";
+ if [[ "$blk_stored" -lt "$blk_provid" ]]; then
+ vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
+ if [[ ! -f "${pathFileIn}.ots.baku" ]]; then
+ must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \
+ vbm "STATUS:Backed up existing OTS file.";
else
- vbm "STATUS:Stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn1}.ots";
- must cp "$path_stored_ots" "${pathFileIn1}.ots";
+ must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \
+ yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present.";
fi;
+ must cp "$path_stored_ots" "${pathFileIn}.ots" && \
+ vbm "STATUS:Replaced provided OTS file with stored OTS file.";
+ return 0;
else
- vbm "STATUS:No stored OTS found. No action taken for:${pathFileIn1}";
+ yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} .";
fi;
+ else
+ vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots";
+ must cp "$path_stored_ots" "${pathFileIn}.ots";
+ return 0;
fi;
+}; # stores provided OTS files and retrieves older OTS files if available
+count_jobs() {
+ # Desc: Count and return total number of jobs
+ # Usage: count_jobs
+ # Input: None.
+ # Output: stdout integer number of jobs
+ # Depends: Bash 5.1.16
+ # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+ # Version: 0.0.1
+
+ local job_count;
+ job_count="$(jobs -r | wc -l | tr -d ' ' )";
+ #yell "DEBUG:job_count:$job_count";
+ if [[ -z $job_count ]]; then job_count="0"; fi;
+ echo "$job_count";
+}; # Return number of background jobs
+main() {
+ checkDepends;
+ processArgs "$@";
+ vbm "DEBUG:Starting rest of main()";
+ vbm "$(declare -p pathsFilesIn)";
+
+ # Process files from provided input args
+ for fpath in "${pathsFilesIn[@]}"; do
+ # throttle if too many jobs
+ if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then
+ sleep 0.01;
+ fi;
+
+ # start new job
+ must store_and_lookup "$fpath" &
+ done;
+ wait;
}; # main program
main "$@";
+
+# Author: Steven Baltakatei Sandoval
+# License: GPLv3+
projects / BK-2020-03.git / commitdiff