feat(user/htmlz_to_cbz):Deduplicate consecutive repeat img urls

[BK-2020-03.git] / user / bkotslu

diff --git a/user/bkotslu b/user/bkotslu
index f2e1220516be82d6512932256205c1f41050c96e..f7bf9ccdde04be40ae66f044a5e1d059a6383a1f 100755 (executable)
--- a/user/bkotslu
+++ b/user/bkotslu
@@ -1,9 +1,16 @@
 #!/bin/bash
 # Desc: Utility for backing up and retrieving ots files
 #!/bin/bash
 # Desc: Utility for backing up and retrieving ots files

-# Version: 0.0.1
+# Usage: bkotslu -I [dir]
+# Version: 0.1.3
+# Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org )
+#          GNU Coreutils 8.32
+# NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid.

 
 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";
 
 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";

-MAX_FIND_DEPTH=12;
+MAX_FIND_DEPTH=1;
+MAX_JOBS=32;
+
+declare -a pathsFilesIn;

 
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
 
 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status


@@ -43,11 +50,15 @@ showUsage() {
                 Display script version.
         -v, --verbose
                 Display debugging info.
                 Display script version.
         -v, --verbose
                 Display debugging info.

-        -i, --input-file
-                Define input file path for file to add ots file to.
+        -i, --input-file [FILE]
+                Provide path for file to submit/lookup OTS files for
+        -I, --input-dir [DIR]
+                Provide dir containing files to submit/lookup OTS files for

         -O, --output-dir
                 Define output directory path for storing ots backup files.
                   DEFAULT: $HOME/.cache/bkotslu/
         -O, --output-dir
                 Define output directory path for storing ots backup files.
                   DEFAULT: $HOME/.cache/bkotslu/

+        -r, --recursive
+                Follow subdirectories in provided directories.

         --
                 Indicate end of options.
 
         --
                 Indicate end of options.
 


@@ -55,7 +66,7 @@ showUsage() {
       Hash foo.txt and lookup matching ots file
         bkotslu -i foo.txt
 
       Hash foo.txt and lookup matching ots file
         bkotslu -i foo.txt
 

-      Archive ots files from home directory
+      Store and lookup older ots files of a directory

         bkotslu -I $HOME/
 
 EOF
         bkotslu -I $HOME/
 
 EOF


@@ -89,17 +100,27 @@ EOF
     vbm "DEBUG:showVersion function ended."
     return 0; # Function finished.
 }; # Display script version.
     vbm "DEBUG:showVersion function ended."
     return 0; # Function finished.
 }; # Display script version.

+checkDepends() {
+    vbm "STATUS:Starting checkDepends()";
+    if ! command -v sha256sum 1>/dev/urandom 2>&1; then
+        die "FATAL:sha256sum not available."; fi;
+    if ! command -v grep 1>/dev/urandom 2>&1; then
+        die "FATAL:grep not available."; fi;
+    if ! command -v find 1>/dev/urandom 2>&1; then
+        die "FATAL:find not available."; fi;
+};

 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
     # Version: 1.0.0
 processArgs() {
     # Desc: Processes arguments provided to script.
     # Usage: processArgs "$@"
     # Version: 1.0.0

-    # Input: "$@"          (list of arguments provided to the function)
+    # Input: "$@"             (list of arguments provided to the function)

     # Output: Sets following variables used by other functions:
     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
     # Output: Sets following variables used by other functions:
     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")

-    #   pathDirOut1          Path to output directory.
-    #   pathDirIn1           Path to input directory.
-    #   pathFileIn1          Path to input file.
+    #   pathOtsStore          Path to output directory.
+    #X   pathDirIn1           Path to input directory.
+    #X   pathFileIn1          Path to input file.

     #   arrayPosArgs         Array of remaining positional argments
     #   arrayPosArgs         Array of remaining positional argments

+    #   pathsFilesIn         input files

     # Depends:
     #   yell()           Displays messages to stderr.
     #   vbm()            Displays messsages to stderr if opVerbose set to "true".
     # Depends:
     #   yell()           Displays messages to stderr.
     #   vbm()            Displays messsages to stderr if opVerbose set to "true".


@@ -123,29 +144,34 @@ processArgs() {
            -h | --help) showUsage; exit 1;; # Display usage.
            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
            -h | --help) showUsage; exit 1;; # Display usage.
            --version) showVersion; exit 1;; # Show version
            -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].

+            -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";;

            -i | --input-file) # Define input file path
            -i | --input-file) # Define input file path

-               if [ -f "$2" ]; then # If $2 is file that exists, set pathFileIn1 to $2, pop $2.
-                   pathFileIn1="$2";
-                    vbm "DEBUG:Input file pathFileIn1 set to:${pathFileIn1}";
+               if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2.
+                    pathsFilesIn+=("$(readlink -f "$2")");
+                    vbm "DEBUG:Added to pathsFilesIn array:$2";

                    shift;
                else
                    shift;
                else

-                   die "FATAL: Specified input file does not exist:$2";
+                   die "FATAL:The provided input file does not exist:$2";

                fi;; 
            -I | --input-dir) # Define input directory path
                fi;; 
            -I | --input-dir) # Define input directory path

-               if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirIn1 to $2, pop $2.
-                   pathDirIn1="$2";
-                    vbm "DEBUG:Input directory pathDirIn1 set to:${pathDirIn1}";
-                   shift;                  
+               if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2.
+                    while read -r file; do
+                        file="$(readlink -f "$file")";
+                        vbm "STATUS:Added to pathsFilesIn array:${file}";
+                        pathsFilesIn+=("$(readlink -f "$file")");
+                    done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f);
+                    vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2";
+                   shift;

                else # Display error if $2 is not a valid dir.
                else # Display error if $2 is not a valid dir.

-                   die "FATAL:Specified input directory does not exist:$2";
+                   die "FATAL:The specified input directory does not exist:$2";

                fi;;
            -O | --output-dir) # Define output directory path
                fi;;
            -O | --output-dir) # Define output directory path

-               if [ -d "$2" ]; then # If $2 is dir that exists, set pathDirOut1 to $2, pop $2
-                   pathDirOut1="$2";
-                    vbm "DEBUG:Output directory pathDirOut1 set to:${pathDirOut1}";
-                   shift;                  
+               if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2
+                   pathOtsStore="$2";
+                    vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}";
+                   shift;

                else
                else

-                   die "FATAL:Specified output directory is not valid:$2";
+                   die "FATAL:The specified output directory is not valid:$2";

                fi;;
             --) # End of all options. See [2].
                 shift;
                fi;;
             --) # End of all options. See [2].
                 shift;


@@ -161,15 +187,15 @@ processArgs() {
     done;
 
     ## Identify ots file cache dir
     done;
 
     ## Identify ots file cache dir

-    if [[ -z "$pathDirOut1" ]]; then
+    if [[ -z "$pathOtsStore" ]]; then

         vbm "STATUS:No output directory for caching OTS files specified.";
         vbm "STATUS:No output directory for caching OTS files specified.";

-        pathDirOut1="$OTS_FCACHE_DIR";
-        vbm "STATUS:Assuming OTS files to be cached in:${pathDirOut1}";
+        pathOtsStore="$OTS_FCACHE_DIR";
+        vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}";

     fi;
     ## Create cache dir if necessary
     fi;
     ## Create cache dir if necessary

-    if [[ ! -d "$pathDirOut1" ]]; then
-        vbm "STATUS:Creating OTS file cache directory:${pathDirOut1}";
-        must mkdir -p "$pathDirOut1";
+    if [[ ! -d "$pathOtsStore" ]]; then
+        vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}";
+        must mkdir -p "$pathOtsStore";

     fi;
     
     # End function
     fi;
     
     # End function


@@ -186,19 +212,20 @@ get_ots_filehash() {
     # Input:  arg1    OTS file path
     # Output: stdout  sha256 file hash (lowercase)
     local output;
     # Input:  arg1    OTS file path
     # Output: stdout  sha256 file hash (lowercase)
     local output;

-

     vbm "DEBUG:Starting get_ots_filehash() on:$1";
     vbm "DEBUG:Starting get_ots_filehash() on:$1";

+
+    re='[0-9a-f]{64}';

     if output="$( "$(which ots)" info "$1" | \
             grep -E "^File sha256 hash: " | \
             head -n1 | \
             sed -E -e 's/(^File sha256 hash: )([0-9a-f]+$)/\2/g'; )" && \
     if output="$( "$(which ots)" info "$1" | \
             grep -E "^File sha256 hash: " | \
             head -n1 | \
             sed -E -e 's/(^File sha256 hash: )([0-9a-f]+$)/\2/g'; )" && \

-            [[ -n "$output" ]]; then
-        vbm "STATUS:Read file hash via ots from:$1";
+            [[ -n "$output" ]] && \
+            [[ "$output" =~ $re ]]; then
+        vbm "STATUS:Read file digest (${output}) via ots from:$1";

         printf "%s" "$output";
         return 0;
     else
         printf "%s" "$output";
         return 0;
     else

-        yell "ERROR:Encountered problem getting file hash via ots from:$1";
-        return 1;
+        die "ERROR:Encountered problem getting file hash via ots from:$1";

     fi;
 }; # Gets hash of file from ots file
 get_ots_oldestblock() {
     fi;
 }; # Gets hash of file from ots file
 get_ots_oldestblock() {


@@ -211,54 +238,236 @@ get_ots_oldestblock() {
     #          vbm()
     #          BK-2020-03: yell()
     local output;
     #          vbm()
     #          BK-2020-03: yell()
     local output;

+    vbm "DEBUG:Starting get_ots_oldestblock() on:$1";

 
 

-    vbm "DEBUG:Starting get_ots_oldestblock() on:$1";    
+    re='[0-9]+';

     if output="$( "$(which ots)" info "$1" | \
             grep -E "verify BitcoinBlockHeaderAttestation\([0-9]+\)" | \
             sort | head -n1 | \
             sed -E -e 's/(^    verify BitcoinBlockHeaderAttestation)\(([0-9]+)(\))/\2/g'; )" && \
     if output="$( "$(which ots)" info "$1" | \
             grep -E "verify BitcoinBlockHeaderAttestation\([0-9]+\)" | \
             sort | head -n1 | \
             sed -E -e 's/(^    verify BitcoinBlockHeaderAttestation)\(([0-9]+)(\))/\2/g'; )" && \

-            [[ -n "$output" ]]; then
-        vbm "STATUS:Retrieved Bitcoin block via ots from:$1";
+            [[ -n "$output" ]] && \
+            [[ "$output" =~ $re ]]; then
+        vbm "STATUS:Retrieved Bitcoin block (${output}) via ots from:$1";

         printf "%s" "$output";
         return 0;
     else
         printf "%s" "$output";
         return 0;
     else

-        yell "ERROR:Encountered problem getting Bitcoin block number via ots from:$1";
-        return 1;
+        die "ERROR:Encountered problem getting Bitcoin block number via ots from:$1";

     fi;
 }; # Gets oldest Bitcoin block from ots file
     fi;
 }; # Gets oldest Bitcoin block from ots file

-cache_ots_file() {
-    # Desc: Scans and caches an OTS file for storing
-    if fhash="$(must get_ots_filehash "$line")" && \
-            block="$(must get_ots_oldestblock "$line")"; then
-        fout="${fhash}_${block}.otsu"; # file name out
-        pout="${pathDirOut1}/${fout}"; # file path out
-        vbm "STATUS:Found OTS file at ${line} with hash ${fhash} and block ${block} and saving to ${pout}";
-        must cp -n "$line" "$pout";
+store_ots_file() {
+    # Desc: Scans and stores an OTS file if none already stored
+    # Usage: store_ots_file FILE
+    # Example: store_ots_file foo.txt.ots
+    # Input:  arg1  OTS file
+    # Output: exit code
+    local fin="$1";
+    vbm "STATUS:Starting store_ots_file()";
+
+    # Check if provided OTS file exists
+    if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
+
+    # Read file hash and oldest block from provided OTS file
+    if ! { fhash="$(must get_ots_filehash "$fin")" && \
+               block="$(must get_ots_oldestblock "$fin")"; }; then
+        yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
+        return 1;
+    fi;
+    vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}.";
+
+    # Copy provided OTS if no matching OTS stored
+    fout="${fhash}_${block}.otsu"; # file name out
+    pout="${pathOtsStore}/${fout}"; # file path out    
+    if [[ ! -f "$pout" ]]; then
+        vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}";
+        must cp -n "$fin" "$pout";
+        return 0;
+    else
+        vbm "STATUS:Stored OTS file with matching file hash and block number in file name found.";
+    fi;
+    
+    # Get block number for provided and stored OTS files.
+    if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then
+        yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )";
+    fi;
+    re='[0-9]+';
+    if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then
+        die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)";
+    fi;
+    
+    # Copy provided OTS if matching OTS found stored but provided is older
+    if [[ "$blk_provid" -lt "$blk_stored" ]]; then
+        vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?";
+        must mv "$pout" "${pout}--$(date +%s)";
+        must cp "$fin" "$pout";
+        return 0;

     else
     else

-        yell "ERROR:Problem analyzing file with OpenTimestamps:${line}";
+        vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file.";
+    fi;
+}; # Stores provided OTS file is none already stored
+get_sha256_digest() {
+    # Depends: GNU Coreutils 8.32 (sha256sum)
+    # Input:  arg1    path  file path
+    # Output: stdout  str   sha256 digest (lowercase hexadecimal)
+    vbm "DEBUG:Starting get_sha256_digest()";
+    sha256sum "$1" | head -n1 | sed -E -e 's/(^[0-9a-f]{64})(.+)/\1/';
+};
+get_oldest_stored_ots_path() {
+    # Desc: Lookup most recent OTS file from storage
+    # Input:  pathOtsStore  var  path to OTS storage dir
+    #         arg1         str  sha256 digest (lowercase hexadecimal)
+    # Output: stdout       path OTS file with matching sha256 digest
+    vbm "DEBUG:Starting get_oldest_stored_ots_path()";
+    local -a otsStorePaths;
+    local i_oldest;
+    
+    digest="$1";
+    mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; );
+    if [[ "${#otsStorePaths[@]}" -le 0 ]]; then
+        yell "NOTICE:No OTS file for digest ${digest} found in ${pathOtsStore}.";
+        return 1;

     fi;
     fi;

-}; # Scans a single OTS file
+    i_oldest=0;
+    re='[0-9]+';
+    blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )";
+    if ! [[ "$blockNumOldest" =~ $re ]]; then die "FATAL:Invalid block number:${blockNumOldest}"; fi;
+    for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do
+        blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )";
+        if ! [[ "$blockNum" =~ $re ]]; then die "FATAL:Invalid block number:${blockNum}"; fi;
+        if [[ $blockNum -lt $blockNumOldest ]]; then
+            blockNumOldest=$blockNum;
+            i_oldest=$i;
+        fi;
+    done;
+    output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )";

 
 

-main() {
-    processArgs "$@";
+    if [[ -n "$output" ]] && [[ -f "$output" ]]; then
+        vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
+        printf "%s" "$output";
+        return 0;
+    else
+        yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} .";
+        return 1;
+    fi;
+}; # Print to stdout path of OTS file with oldest block
+get_block_num_from_stored_ots_path() {
+    # Desc: Return block number from stored OTS path
+    # Input:  arg1    input file path
+    # Output: stdout  block number (int)
+    # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'.
+    local fin fbase block re;
+    fpath="$1";
+    fbase="$(basename "$fpath")";
+    block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")";
+    re='[0-9]+';
+    if [[ "$block" =~ $re ]]; then
+        printf "%s" "$block";
+    else
+        yell "ERROR:Invalid block number:$(declare -p fpath fbase block)";
+        return 1;
+    fi;
+}; # Print block number from stored OTS file
+store_and_lookup() {
+    # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible
+    # Usage: store_and_lookup [path]
+    # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock()
+    local pathFileIn="$1";
+    vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}";

 
 

+    # Validate path
+    if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi;

 
 

-    # Scan provided dir for OTS files to cache.
-    if [[ -n "$pathDirIn1" ]]; then
-        while read -r line; do
-            line="$(readlink -f "$line")";
-            cache_ots_file;
-        done < <(find "$pathDirIn1" -maxdepth "$MAX_FIND_DEPTH" -type f -name "*.ots");
+    # Check for and store any OTS file attached to provided file
+    ## Check if provided file is an OTS file itself
+    if [[ "$pathFileIn" =~ \.ots$ ]]; then
+        vbm "STATUS:The provided file is itself an OTS file. Store OTS file only.";
+        store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file.";
+        return 0;
+    fi;
+    ## Check if provided file has an accompanying OTS file
+    if [[ -f "${pathFileIn}.ots" ]]; then
+        vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots";
+        store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file.";

     fi;
 
     # Lookup OTS file from archive for provided file.
     fi;
 
     # Lookup OTS file from archive for provided file.

-    if [[ -n "$pathFileIn1" ]]; then
-        :
+    ## Get file hash
+    fhash="$(get_sha256_digest "$pathFileIn"; )";
+
+    ## Get stored OTS path if possible.
+    if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
+        yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}";
+        return 0;

     fi;
     fi;

+    vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}.";
+    blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
+    vbm "STATUS:The stored OTS file has block number ${blk_stored}.";
+
+    ## Check for OTS file accompanying provided file
+    if [[ -f "${pathFileIn}.ots" ]]; then
+        vbm "STATUS:An OTS file is next to provided file ${pathFileIn}.";
+        blk_provid="$(must get_ots_oldestblock "${pathFileIn}.ots"; )";
+        vbm "STATUS:The provided file's OTS file has block number ${blk_provid}";
+        re='[0-9]+';
+        if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then
+            die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)";
+        fi;
+        if [[ "$blk_stored" -lt "$blk_provid" ]]; then
+            vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
+            if [[ ! -f "${pathFileIn}.ots.baku" ]]; then
+                must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \
+                    vbm "STATUS:Backed up existing OTS file.";
+            else
+                must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \
+                    yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present.";
+            fi;
+            must cp "$path_stored_ots" "${pathFileIn}.ots" && \
+                vbm "STATUS:Replaced provided OTS file with stored OTS file.";
+            return 0;
+        else
+            yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} .";
+        fi;
+    else
+        vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots";
+        must cp "$path_stored_ots" "${pathFileIn}.ots";
+        return 0;
+    fi;
+}; # stores provided OTS files and retrieves older OTS files if available
+count_jobs() {
+    # Desc: Count and return total number of jobs
+    # Usage: count_jobs
+    # Input: None.
+    # Output: stdout   integer number of jobs
+    # Depends: Bash 5.1.16
+    # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
+    # Version: 0.0.1

     
     

-    
+    local job_count;
+    job_count="$(jobs -r | wc -l | tr -d ' ' )";
+    #yell "DEBUG:job_count:$job_count";
+    if [[ -z $job_count ]]; then job_count="0"; fi;
+    echo "$job_count";
+}; # Return number of background jobs
+main() {
+    checkDepends;
+    processArgs "$@";
+    vbm "DEBUG:Starting rest of main()";
+    vbm "$(declare -p pathsFilesIn)";

 
 

+    # Process files from provided input args
+    for fpath in "${pathsFilesIn[@]}"; do
+        # throttle if too many jobs
+        if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then
+            sleep 0.01;
+        fi;
+        
+        # start new job
+        must store_and_lookup "$fpath" &
+    done;
+    wait;

 
 }; # main program
 
 main "$@";
 
 }; # main program
 
 main "$@";

+
+# Author: Steven Baltakatei Sandoval
+# License: GPLv3+