CRITICAL(user/bkotslu):Fix incorrect renaming of base files

- Note: no data lost, but runs of v0.0.2 caused renaming problems that
  requires careful restoration of `.baku` files and renaming of
  incorrectly named `.ots` files.

diff --git a/user/bkotslu b/user/bkotslu
index b6f79e51c2104ec9b3514675974dcce6241206ca..cb372b58982bfe5d011494cb4e9823bc8dd99375 100755 (executable)
--- a/user/bkotslu
+++ b/user/bkotslu
@@ -1,9 +1,16 @@
 #!/bin/bash
 # Desc: Utility for backing up and retrieving ots files
 #!/bin/bash
 # Desc: Utility for backing up and retrieving ots files

-# Version: 0.0.2
+# Usage: bkotslu -I [dir]
+# Version: 0.1.0
+# Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org )
+#          GNU Coreutils 8.32
+# NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid.

 
 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";
 
 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";

-MAX_FIND_DEPTH=12;
+MAX_FIND_DEPTH=1;
+MAX_JOBS=32;
+
+declare -a pathsFilesIn;

 
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
 
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status


@@ -43,11 +50,15 @@ showUsage() {
                 Display script version.
         -v, --verbose
                 Display debugging info.
                 Display script version.
         -v, --verbose
                 Display debugging info.

-        -i, --input-file
-                Define input file path for file to add ots file to.
+        -i, --input-file [FILE]
+                Provide path for file to submit/lookup OTS files for
+        -I, --input-dir [DIR]
+                Provide dir containing files to submit/lookup OTS files for

         -O, --output-dir
                 Define output directory path for storing ots backup files.
                   DEFAULT: $HOME/.cache/bkotslu/
         -O, --output-dir
                 Define output directory path for storing ots backup files.
                   DEFAULT: $HOME/.cache/bkotslu/

+        -r, --recursive
+                Follow subdirectories in provided directories.

         --
                 Indicate end of options.
 
         --
                 Indicate end of options.
 


@@ -55,7 +66,7 @@ showUsage() {
       Hash foo.txt and lookup matching ots file
         bkotslu -i foo.txt
 
       Hash foo.txt and lookup matching ots file
         bkotslu -i foo.txt
 

-      Archive ots files from home directory
+      Store and lookup older ots files of a directory

         bkotslu -I $HOME/
 
 EOF
         bkotslu -I $HOME/
 
 EOF


@@ -91,24 +102,25 @@ EOF
 }; # Display script version.
 checkDepends() {
     vbm "STATUS:Starting checkDepends()";
 }; # Display script version.
 checkDepends() {
     vbm "STATUS:Starting checkDepends()";

-    if ! command -v sha256sum 1>/dev/random 2>&1; then
+    if ! command -v sha256sum 1>/dev/urandom 2>&1; then

         die "FATAL:sha256sum not available."; fi;
         die "FATAL:sha256sum not available."; fi;

-    if ! command -v grep 1>/dev/random 2>&1; then
-        die "FATAL:grep not available."; fi;
-    if ! command -v find 1>/dev/random 2>&1; then
+    if ! command -v grep 1>/dev/urandom 2>&1; then

         die "FATAL:grep not available."; fi;
         die "FATAL:grep not available."; fi;

+    if ! command -v find 1>/dev/urandom 2>&1; then
+        die "FATAL:find not available."; fi;

 };
 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
     # Version: 1.0.0
 };
 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
     # Version: 1.0.0

-    # Input: "$@"          (list of arguments provided to the function)
+    # Input: "$@"             (list of arguments provided to the function)

     # Output: Sets following variables used by other functions:
     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
     # Output: Sets following variables used by other functions:
     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")

-    #   pathDirOut1          Path to output directory.
-    #   pathDirIn1           Path to input directory.
-    #   pathFileIn1          Path to input file.
+    #   pathOtsStore          Path to output directory.
+    #X   pathDirIn1           Path to input directory.
+    #X   pathFileIn1          Path to input file.

     #   arrayPosArgs         Array of remaining positional argments
     #   arrayPosArgs         Array of remaining positional argments

+    #   pathsFilesIn         input files

     # Depends:
     #   yell()           Displays messages to stderr.
     #   vbm()            Displays messsages to stderr if opVerbose set to "true".
     # Depends:
     #   yell()           Displays messages to stderr.
     #   vbm()            Displays messsages to stderr if opVerbose set to "true".


@@ -132,29 +144,34 @@ processArgs() {
            -h | --help) showUsage; exit 1;; # Display usage.
            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
            -h | --help) showUsage; exit 1;; # Display usage.
            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].

+            -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";;

            -i | --input-file) # Define input file path
            -i | --input-file) # Define input file path

-               if [ -f "$2" ]; then # If $2 is file that exists, set pathFileIn1 to $2, pop $2.
-                   pathFileIn1="$2";
-                    vbm "DEBUG:Input file pathFileIn1 set to:${pathFileIn1}";
+               if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2.
+                    pathsFilesIn+=("$(readlink -f "$2")");
+                    vbm "DEBUG:Added to pathsFilesIn array:$2";

                    shift;
                else
                    shift;
                else

-                   die "FATAL: Specified input file does not exist:$2";
+                   die "FATAL:The provided input file does not exist:$2";

                fi;; 
            -I | --input-dir) # Define input directory path
                fi;; 
            -I | --input-dir) # Define input directory path

-               if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirIn1 to $2, pop $2.
-                   pathDirIn1="$2";
-                    vbm "DEBUG:Input directory pathDirIn1 set to:${pathDirIn1}";
-                   shift;                  
+               if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2.
+                    while read -r file; do
+                        file="$(readlink -f "$file")";
+                        vbm "STATUS:Added to pathsFilesIn array:${file}";
+                        pathsFilesIn+=("$(readlink -f "$file")");
+                    done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f);
+                    vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2";
+                   shift;

                else # Display error if $2 is not a valid dir.
                else # Display error if $2 is not a valid dir.

-                   die "FATAL:Specified input directory does not exist:$2";
+                   die "FATAL:The specified input directory does not exist:$2";

                fi;;
            -O | --output-dir) # Define output directory path
                fi;;
            -O | --output-dir) # Define output directory path

-               if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirOut1 to $2, pop $2
-                   pathDirOut1="$2";
-                    vbm "DEBUG:Output directory pathDirOut1 set to:${pathDirOut1}";
-                   shift;                  
+               if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2
+                   pathOtsStore="$2";
+                    vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}";
+                   shift;

                else
                else

-                   die "FATAL:Specified output directory is not valid:$2";
+                   die "FATAL:The specified output directory is not valid:$2";

                fi;;
             --) # End of all options. See [2].
                 shift;
                fi;;
             --) # End of all options. See [2].
                 shift;


@@ -170,15 +187,15 @@ processArgs() {
     done;
 
     ## Identify ots file cache dir
     done;
 
     ## Identify ots file cache dir

-    if [[ -z "$pathDirOut1" ]]; then
+    if [[ -z "$pathOtsStore" ]]; then

         vbm "STATUS:No output directory for caching OTS files specified.";
         vbm "STATUS:No output directory for caching OTS files specified.";

-        pathDirOut1="$OTS_FCACHE_DIR";
-        vbm "STATUS:Assuming OTS files to be cached in:${pathDirOut1}";
+        pathOtsStore="$OTS_FCACHE_DIR";
+        vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}";

     fi;
     ## Create cache dir if necessary
     fi;
     ## Create cache dir if necessary

-    if [[ ! -d "$pathDirOut1" ]]; then
-        vbm "STATUS:Creating OTS file cache directory:${pathDirOut1}";
-        must mkdir -p "$pathDirOut1";
+    if [[ ! -d "$pathOtsStore" ]]; then
+        vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}";
+        must mkdir -p "$pathOtsStore";

     fi;
     
     # End function
     fi;
     
     # End function


@@ -235,27 +252,52 @@ get_ots_oldestblock() {
         return 1;
     fi;
 }; # Gets oldest Bitcoin block from ots file
         return 1;
     fi;
 }; # Gets oldest Bitcoin block from ots file

-cache_ots_file() {
-    # Desc: Scans and caches an OTS file for storing
-    # Input: arg1  input file
+store_ots_file() {
+    # Desc: Scans and stores an OTS file if none already stored
+    # Usage: store_ots_file FILE
+    # Example: store_ots_file foo.txt.ots
+    # Input:  arg1  OTS file

     # Output: exit code
     local fin="$1";
     # Output: exit code
     local fin="$1";

-    vbm "STATUS:Starting cache_ots_file()";
-    
-    if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi; 
-    if fhash="$(must get_ots_filehash "$fin")" && \
-            block="$(must get_ots_oldestblock "$fin")"; then
-        fout="${fhash}_${block}.otsu"; # file name out
-        pout="${pathDirOut1}/${fout}"; # file path out
-        vbm "STATUS:Found OTS file at ${fin} with hash ${fhash} and block ${block}.";
-        vbm "STATUS:Saving found OTS file to ${pout}";
+    vbm "STATUS:Starting store_ots_file()";
+
+    # Check if provided OTS file exists
+    if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
+
+    # Read file hash and oldest block from provided OTS file
+    if ! { fhash="$(must get_ots_filehash "$fin")" && \
+               block="$(must get_ots_oldestblock "$fin")"; }; then
+        yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
+        return 1;
+    fi;
+    vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}.";
+
+    # Copy provided OTS if no matching OTS stored
+    fout="${fhash}_${block}.otsu"; # file name out
+    pout="${pathOtsStore}/${fout}"; # file path out    
+    if [[ ! -f "$pout" ]]; then
+        vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}";

         must cp -n "$fin" "$pout";
         return 0;
     else
         must cp -n "$fin" "$pout";
         return 0;
     else

-        yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
-        return 1;
+        vbm "STATUS:Stored OTS file with matching file hash and block number in file name found.";

     fi;
     fi;

-}; # Scans a single OTS file
+    
+    # Get block number for provided and stored OTS files.
+    if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then
+        yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )";
+    fi;
+    
+    # Copy provided OTS if matching OTS found stored but provided is older
+    if [[ "$blk_provid" -lt "$blk_stored" ]]; then
+        vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?";
+        must mv "$pout" "${pout}--$(date +%s)";
+        must cp "$fin" "$pout";
+        return 0;
+    else
+        vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file.";
+    fi;
+}; # Stores provided OTS file is none already stored

 get_sha256_digest() {
     # Depends: GNU Coreutils 8.32 (sha256sum)
     # Input:  arg1    path  file path
 get_sha256_digest() {
     # Depends: GNU Coreutils 8.32 (sha256sum)
     # Input:  arg1    path  file path


@@ -265,72 +307,154 @@ get_sha256_digest() {
 };
 get_oldest_stored_ots_path() {
     # Desc: Lookup most recent OTS file from storage
 };
 get_oldest_stored_ots_path() {
     # Desc: Lookup most recent OTS file from storage

-    # Input:  pathDirOut1  var  path to OTS storage dir
+    # Input:  pathOtsStore  var  path to OTS storage dir

     #         arg1         str  sha256 digest (lowercase hexadecimal)
     # Output: stdout       path OTS file with matching sha256 digest
     vbm "DEBUG:Starting get_oldest_stored_ots_path()";
     #         arg1         str  sha256 digest (lowercase hexadecimal)
     # Output: stdout       path OTS file with matching sha256 digest
     vbm "DEBUG:Starting get_oldest_stored_ots_path()";

+    local -a otsStorePaths;
+    local i_oldest;

     
     digest="$1";
     
     digest="$1";

-    if output="$( find "$pathDirOut1" -type f -name "${digest}*" | \
-            sort | \
-            head -n1 )" && \
-            [[ -n "$output" ]]; then
-        output="$(readlink -f "$output")";
+    mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; );
+    if [[ "${#otsStorePaths[@]}" -le 0 ]]; then
+        yell "ERROR:No OTS file in OTS storage dir found. $(declare -p pathOtsStore digest otsStorePaths)";
+        return 1;
+    fi;
+    i_oldest=0;
+    blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )";
+    for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do
+        blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )";
+        if [[ $blockNum -lt $blockNumOldest ]]; then
+            blockNumOldest=$blockNum;
+            i_oldest=$i;
+        fi;
+    done;
+    output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )";
+
+    if [[ -n "$output" ]] && [[ -f "$output" ]]; then

         vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
         printf "%s" "$output";
         return 0;
     else
         vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
         printf "%s" "$output";
         return 0;
     else

-        yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathDirOut1}";
+        yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} .";

         return 1;
     fi;
         return 1;
     fi;

-};
-main() {
-    checkDepends;
-    processArgs "$@";
-    vbm "DEBUG:Starting rest of main()";
+}; # Print to stdout path of OTS file with oldest block
+get_block_num_from_stored_ots_path() {
+    # Desc: Return block number from stored OTS path
+    # Input:  arg1    input file path
+    # Output: stdout  block number (int)
+    # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'.
+    local fin fbase block re;
+    fpath="$1";
+    fbase="$(basename "$fpath")";
+    block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")";
+    re='[0-9]+';
+    if [[ "$block" =~ $re ]]; then
+        printf "%s" "$block";
+    else
+        yell "ERROR:Invalid block number:$(declare -p fpath fbase block)";
+        return 1;
+    fi;
+}; # Print block number from stored OTS file
+store_and_lookup() {
+    # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible
+    # Usage: store_and_lookup [path]
+    # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock()
+    local pathFileIn="$1";
+    vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}";
+
+    # Validate path
+    if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi;

 
 

-    # Scan provided dir for OTS files to cache.
-    if [[ -n "$pathDirIn1" ]]; then
-        while read -r line; do
-            line="$(readlink -f "$line")";
-            cache_ots_file "$line";
-        done < <(find "$pathDirIn1" -maxdepth "$MAX_FIND_DEPTH" -type f -name "*.ots");
+    # Check for and store any OTS file attached to provided file
+    ## Check if provided file is an OTS file itself
+    if [[ "$pathFileIn" =~ \.ots$ ]]; then
+        vbm "STATUS:The provided file is itself an OTS file. Store OTS file only.";
+        store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file.";
+        return 0;
+    fi;
+    ## Check if provided file has an accompanying OTS file
+    if [[ -f "${pathFileIn}.ots" ]]; then
+        vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots";
+        store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file.";

     fi;
 
     # Lookup OTS file from archive for provided file.
     fi;
 
     # Lookup OTS file from archive for provided file.

-    if [[ -n "$pathFileIn1" ]]; then
-        # Check for and store any OTS file attached to provided file
-        if [[ -f "${pathFileIn1}.ots" ]]; then
-            vbm "STATUS:OTS file accompanying provided file found:${pathFileIn1}.ots";
-            cache_ots_file "${pathFileIn1}.ots" && vbm "STATUS:Stored provided file's OTS file.";
-        fi;
-        
-        # Get file hash
-        fhash="$(get_sha256_digest "$pathFileIn1"; )";
-        if path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
-            vbm "STATUS:Stored OTS found with matching hash.";
-            blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
-            vbm "STATUS:Stored OTS file has block number ${blk_stored}.";
-            if [[ -f "${pathFileIn1}.ots" ]]; then
-                vbm "STATUS:An OTS file is next to provided file.";
-                blk_provid="$(get_ots_oldestblock "${pathFileIn1}.ots"; )";
-                vbm "STATUS:Provided file's OTS file has block number ${blk_provid}";
-                if [[ "$blk_stored" -lt "$blk_provid" ]]; then
-                    vbm "STATUS:Older timestamp in OTS store found. Replacing ${pathFileIn1} (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
-                    #must mv "$pathFileIn1" "${pathFileIn1}.baku";
-                    #must cp "$path_stored_ots" "$pathFileIn1";
-                else
-                    yell "STATUS:Stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn1}";
-                fi;
+    ## Get file hash
+    fhash="$(get_sha256_digest "$pathFileIn"; )";
+
+    ## Get stored OTS path if possible.
+    if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
+        yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}";
+        return 0;
+    fi;
+    vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}.";
+    blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
+    vbm "STATUS:The stored OTS file has block number ${blk_stored}.";
+
+    ## Check for OTS file accompanying provided file
+    if [[ -f "${pathFileIn}.ots" ]]; then
+        vbm "STATUS:An OTS file is next to provided file ${pathFileIn}.";
+        blk_provid="$(get_ots_oldestblock "${pathFileIn}.ots"; )";
+        vbm "STATUS:The provided file's OTS file has block number ${blk_provid}";
+        if [[ "$blk_stored" -lt "$blk_provid" ]]; then
+            vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
+            if [[ ! -f "${pathFileIn}.ots.baku" ]]; then
+                must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \
+                    vbm "STATUS:Backed up existing OTS file.";

             else
             else

-                vbm "STATUS:Stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn1}.ots";
-                must cp "$path_stored_ots" "${pathFileIn1}.ots";
+                must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \
+                    yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present.";

             fi;
             fi;

+            must cp "$path_stored_ots" "${pathFileIn}.ots" && \
+                vbm "STATUS:Replaced provided OTS file with stored OTS file.";
+            return 0;

         else
         else

-            vbm "STATUS:No stored OTS found. No action taken for:${pathFileIn1}";
+            yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} .";

         fi;
         fi;

+    else
+        vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots";
+        must cp "$path_stored_ots" "${pathFileIn}.ots";
+        return 0;

     fi;
     fi;

+}; # stores provided OTS files and retrieves older OTS files if available
+count_jobs() {
+    # Desc: Count and return total number of jobs
+    # Usage: count_jobs
+    # Input: None.
+    # Output: stdout   integer number of jobs
+    # Depends: Bash 5.1.16
+    # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+    # Version: 0.0.1
+    
+    local job_count;
+    job_count="$(jobs -r | wc -l | tr -d ' ' )";
+    #yell "DEBUG:job_count:$job_count";
+    if [[ -z $job_count ]]; then job_count="0"; fi;
+    echo "$job_count";
+}; # Return number of background jobs
+main() {
+    checkDepends;
+    processArgs "$@";
+    vbm "DEBUG:Starting rest of main()";
+    vbm "$(declare -p pathsFilesIn)";
+
+    # Process files from provided input args
+    for fpath in "${pathsFilesIn[@]}"; do
+        # throttle if too many jobs
+        if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then
+            sleep 0.01;
+        fi;
+        
+        # start new job
+        must store_and_lookup "$fpath" &
+    done;
+    wait;

 
 }; # main program
 
 main "$@";
 
 }; # main program
 
 main "$@";

+
+# Author: Steven Baltakatei Sandoval
+# License: GPLv3+