user/bkotslu

   1 #!/bin/bash
   2 # Desc: Utility for backing up and retrieving ots files
   3 # Usage: bkotslu -I [dir]
   4 # Version: 0.1.3
   5 # Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org )
   6 #          GNU Coreutils 8.32
   7 # NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid.
   8
   9 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";
  10 MAX_FIND_DEPTH=1;
  11 MAX_JOBS=32;
  12
  13 declare -a pathsFilesIn;
  14
  15 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
  16 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
  17 must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
  18 vbm() {
  19     # Description: Prints verbose message ("vbm") to stderr if opVerbose is set to "true".
  20     # Usage: vbm "DEBUG :verbose message here"
  21     # Version 0.2.0
  22     # Input: arg1: string
  23     #        vars: opVerbose
  24     # Output: stderr
  25     # Depends: bash 5.1.16, GNU-coreutils 8.30 (echo, date)
  26
  27     if [ "$opVerbose" = "true" ]; then
  28         functionTime="$(date --iso-8601=ns)"; # Save current time in nano seconds.
  29         echo "[$functionTime]:$0:""$*" 1>&2;  # Display argument text.
  30     fi
  31
  32     # End function
  33     return 0; # Function finished.
  34 }; # Displays message if opVerbose true
  35 showUsage() {
  36     # Desc: Display script usage information
  37     # Usage: showUsage
  38     # Version 0.0.2
  39     # Input: none
  40     # Output: stdout
  41     # Depends: GNU-coreutils 8.30 (cat)
  42     cat <<'EOF'
  43     USAGE:
  44         bkotslu [ options ] [FILE...]
  45
  46     OPTIONS:
  47         -h, --help
  48                 Display help information.
  49         --version
  50                 Display script version.
  51         -v, --verbose
  52                 Display debugging info.
  53         -i, --input-file [FILE]
  54                 Provide path for file to submit/lookup OTS files for
  55         -I, --input-dir [DIR]
  56                 Provide dir containing files to submit/lookup OTS files for
  57         -O, --output-dir
  58                 Define output directory path for storing ots backup files.
  59                   DEFAULT: $HOME/.cache/bkotslu/
  60         -r, --recursive
  61                 Follow subdirectories in provided directories.
  62         --
  63                 Indicate end of options.
  64
  65     EXAMPLE:
  66       Hash foo.txt and lookup matching ots file
  67         bkotslu -i foo.txt
  68
  69       Store and lookup older ots files of a directory
  70         bkotslu -I $HOME/
  71
  72 EOF
  73 }; # Display information on how to use this script.
  74 showVersion() {
  75     # Desc: Displays script version and license information.
  76     # Usage: showVersion
  77     # Version: 0.0.2
  78     # Input: scriptVersion   var containing version string
  79     # Output: stdout
  80     # Depends: vbm(), yell, GNU-coreutils 8.30
  81
  82     # Initialize function
  83     vbm "DEBUG:showVersion function called."
  84
  85     cat <<'EOF'
  86 bkotslu 0.0.1
  87 Copyright (C) 2024 Steven Baltakatei Sandoval
  88 License GPLv3: GNU GPL version 3
  89 This is free software; you are free to change and redistribute it.
  90 There is NO WARRANTY, to the extent permitted by law.
  91
  92     GNU Coreutils 8.32
  93     Copyright (C) 2020 Free Software Foundation, Inc.
  94     License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
  95     This is free software: you are free to change and redistribute it.
  96     There is NO WARRANTY, to the extent permitted by law.
  97 EOF
  98
  99     # End function
 100     vbm "DEBUG:showVersion function ended."
 101     return 0; # Function finished.
 102 }; # Display script version.
 103 checkDepends() {
 104     vbm "STATUS:Starting checkDepends()";
 105     if ! command -v sha256sum 1>/dev/urandom 2>&1; then
 106         die "FATAL:sha256sum not available."; fi;
 107     if ! command -v grep 1>/dev/urandom 2>&1; then
 108         die "FATAL:grep not available."; fi;
 109     if ! command -v find 1>/dev/urandom 2>&1; then
 110         die "FATAL:find not available."; fi;
 111 };
 112 processArgs() {
 113     # Desc: Processes arguments provided to script.
 114     # Usage: processArgs "$@"
 115     # Version: 1.0.0
 116     # Input: "$@"             (list of arguments provided to the function)
 117     # Output: Sets following variables used by other functions:
 118     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
 119     #   pathOtsStore          Path to output directory.
 120     #X   pathDirIn1           Path to input directory.
 121     #X   pathFileIn1          Path to input file.
 122     #   arrayPosArgs         Array of remaining positional argments
 123     #   pathsFilesIn         input files
 124     # Depends:
 125     #   yell()           Displays messages to stderr.
 126     #   vbm()            Displays messsages to stderr if opVerbose set to "true".
 127     #   showUsage()      Displays usage information about parent script.
 128     #   showVersion()    Displays version about parent script.
 129     #   arrayPosArgs     Global array for storing non-option positional arguments (i.e. arguments following the `--` option).
 130     # External dependencies: bash (5.1.16), echo
 131     # Ref./Attrib.:
 132     #  [1]: Marco Aurelio (2014-05-08). "echo that outputs to stderr". https://stackoverflow.com/a/23550347
 133     #  [2]: "Handling positional parameters" (2018-05-12). https://wiki.bash-hackers.org/scripting/posparams
 134
 135     # Initialize function
 136     vbm "DEBUG:processArgs function called."
 137
 138     # Perform work
 139     if [ $# -le 0 ]; then yell "FATAL:No arguments provided."; showUsage; fi;
 140     while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
 141         #yell "DEBUG:Starting processArgs while loop." # Debug stderr message. See [1].
 142         #yell "DEBUG:Provided arguments are:""$*"      # Debug stderr message. See [1].
 143         case "$1" in
 144             -h | --help) showUsage; exit 1;; # Display usage.
 145             --version) showVersion; exit 1;; # Show version
 146             -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
 147             -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";;
 148             -i | --input-file) # Define input file path
 149                 if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2.
 150                     pathsFilesIn+=("$(readlink -f "$2")");
 151                     vbm "DEBUG:Added to pathsFilesIn array:$2";
 152                     shift;
 153                 else
 154                     die "FATAL:The provided input file does not exist:$2";
 155                 fi;;
 156             -I | --input-dir) # Define input directory path
 157                 if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2.
 158                     while read -r file; do
 159                         file="$(readlink -f "$file")";
 160                         vbm "STATUS:Added to pathsFilesIn array:${file}";
 161                         pathsFilesIn+=("$(readlink -f "$file")");
 162                     done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f);
 163                     vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2";
 164                     shift;
 165                 else # Display error if $2 is not a valid dir.
 166                     die "FATAL:The specified input directory does not exist:$2";
 167                 fi;;
 168             -O | --output-dir) # Define output directory path
 169                 if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2
 170                     pathOtsStore="$2";
 171                     vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}";
 172                     shift;
 173                 else
 174                     die "FATAL:The specified output directory is not valid:$2";
 175                 fi;;
 176             --) # End of all options. See [2].
 177                 shift;
 178                 for arg in "$@"; do
 179                     vbm "DEBUG:adding to arrayPosArgs:$arg";
 180                     arrayPosArgs+=("$arg");
 181                 done;
 182                 break;;
 183             -*) showUsage; die "FATAL: Unrecognized option.";; # Display usage
 184             *) showUsage; die "FATAL: Unrecognized argument.";; # Handle unrecognized options. See [1].
 185         esac;
 186         shift;
 187     done;
 188
 189     ## Identify ots file cache dir
 190     if [[ -z "$pathOtsStore" ]]; then
 191         vbm "STATUS:No output directory for caching OTS files specified.";
 192         pathOtsStore="$OTS_FCACHE_DIR";
 193         vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}";
 194     fi;
 195     ## Create cache dir if necessary
 196     if [[ ! -d "$pathOtsStore" ]]; then
 197         vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}";
 198         must mkdir -p "$pathOtsStore";
 199     fi;
 200
 201     # End function
 202     vbm "DEBUG:processArgs function ended.";
 203     return  0; # Function finished.
 204 }; # Evaluate script options from positional arguments (ex: $1, $2, $3, etc.).
 205 get_ots_filehash() {
 206     # Desc: Gets hash of an opentimestamp'd file from ots file
 207     # Usage: get_ots_filehash FILE
 208     # Example: get_ots_filehash foo.txt.ots
 209     # Depends: ots 0.7.0, GNU grep 3.7, GNU Coreutils 8.32
 210     #          vbm()              verbose output
 211     #          BK-2020-03  yell()
 212     # Input:  arg1    OTS file path
 213     # Output: stdout  sha256 file hash (lowercase)
 214     local output;
 215     vbm "DEBUG:Starting get_ots_filehash() on:$1";
 216
 217     re='[0-9a-f]{64}';
 218     if output="$( "$(which ots)" info "$1" | \
 219             grep -E "^File sha256 hash: " | \
 220             head -n1 | \
 221             sed -E -e 's/(^File sha256 hash: )([0-9a-f]+$)/\2/g'; )" && \
 222             [[ -n "$output" ]] && \
 223             [[ "$output" =~ $re ]]; then
 224         vbm "STATUS:Read file digest (${output}) via ots from:$1";
 225         printf "%s" "$output";
 226         return 0;
 227     else
 228         die "ERROR:Encountered problem getting file hash via ots from:$1";
 229     fi;
 230 }; # Gets hash of file from ots file
 231 get_ots_oldestblock() {
 232     # Desc: Gets earliest Bitcoin block number from ots file
 233     # Usage: get_ots_oldestblock FILE
 234     # Example: get_ots_oldestblock foo.txt.ots
 235     # Input: arg1     path  OTS file path
 236     # Output: stdout  int   Bitcoin block number
 237     # Depends: OpenTimestamps 0.7.0, GNU grep 3.7, GNU Coreutils 8.32
 238     #          vbm()
 239     #          BK-2020-03: yell()
 240     local output;
 241     vbm "DEBUG:Starting get_ots_oldestblock() on:$1";
 242
 243     re='[0-9]+';
 244     if output="$( "$(which ots)" info "$1" | \
 245             grep -E "verify BitcoinBlockHeaderAttestation\([0-9]+\)" | \
 246             sort | head -n1 | \
 247             sed -E -e 's/(^    verify BitcoinBlockHeaderAttestation)\(([0-9]+)(\))/\2/g'; )" && \
 248             [[ -n "$output" ]] && \
 249             [[ "$output" =~ $re ]]; then
 250         vbm "STATUS:Retrieved Bitcoin block (${output}) via ots from:$1";
 251         printf "%s" "$output";
 252         return 0;
 253     else
 254         die "ERROR:Encountered problem getting Bitcoin block number via ots from:$1";
 255     fi;
 256 }; # Gets oldest Bitcoin block from ots file
 257 store_ots_file() {
 258     # Desc: Scans and stores an OTS file if none already stored
 259     # Usage: store_ots_file FILE
 260     # Example: store_ots_file foo.txt.ots
 261     # Input:  arg1  OTS file
 262     # Output: exit code
 263     local fin="$1";
 264     vbm "STATUS:Starting store_ots_file()";
 265
 266     # Check if provided OTS file exists
 267     if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
 268
 269     # Read file hash and oldest block from provided OTS file
 270     if ! { fhash="$(must get_ots_filehash "$fin")" && \
 271                block="$(must get_ots_oldestblock "$fin")"; }; then
 272         yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
 273         return 1;
 274     fi;
 275     vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}.";
 276
 277     # Copy provided OTS if no matching OTS stored
 278     fout="${fhash}_${block}.otsu"; # file name out
 279     pout="${pathOtsStore}/${fout}"; # file path out
 280     if [[ ! -f "$pout" ]]; then
 281         vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}";
 282         must cp -n "$fin" "$pout";
 283         return 0;
 284     else
 285         vbm "STATUS:Stored OTS file with matching file hash and block number in file name found.";
 286     fi;
 287
 288     # Get block number for provided and stored OTS files.
 289     if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then
 290         yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )";
 291     fi;
 292     re='[0-9]+';
 293     if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then
 294         die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)";
 295     fi;
 296
 297     # Copy provided OTS if matching OTS found stored but provided is older
 298     if [[ "$blk_provid" -lt "$blk_stored" ]]; then
 299         vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?";
 300         must mv "$pout" "${pout}--$(date +%s)";
 301         must cp "$fin" "$pout";
 302         return 0;
 303     else
 304         vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file.";
 305     fi;
 306 }; # Stores provided OTS file is none already stored
 307 get_sha256_digest() {
 308     # Depends: GNU Coreutils 8.32 (sha256sum)
 309     # Input:  arg1    path  file path
 310     # Output: stdout  str   sha256 digest (lowercase hexadecimal)
 311     vbm "DEBUG:Starting get_sha256_digest()";
 312     sha256sum "$1" | head -n1 | sed -E -e 's/(^[0-9a-f]{64})(.+)/\1/';
 313 };
 314 get_oldest_stored_ots_path() {
 315     # Desc: Lookup most recent OTS file from storage
 316     # Input:  pathOtsStore  var  path to OTS storage dir
 317     #         arg1         str  sha256 digest (lowercase hexadecimal)
 318     # Output: stdout       path OTS file with matching sha256 digest
 319     vbm "DEBUG:Starting get_oldest_stored_ots_path()";
 320     local -a otsStorePaths;
 321     local i_oldest;
 322
 323     digest="$1";
 324     mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; );
 325     if [[ "${#otsStorePaths[@]}" -le 0 ]]; then
 326         yell "NOTICE:No OTS file for digest ${digest} found in ${pathOtsStore}.";
 327         return 1;
 328     fi;
 329     i_oldest=0;
 330     re='[0-9]+';
 331     blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )";
 332     if ! [[ "$blockNumOldest" =~ $re ]]; then die "FATAL:Invalid block number:${blockNumOldest}"; fi;
 333     for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do
 334         blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )";
 335         if ! [[ "$blockNum" =~ $re ]]; then die "FATAL:Invalid block number:${blockNum}"; fi;
 336         if [[ $blockNum -lt $blockNumOldest ]]; then
 337             blockNumOldest=$blockNum;
 338             i_oldest=$i;
 339         fi;
 340     done;
 341     output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )";
 342
 343     if [[ -n "$output" ]] && [[ -f "$output" ]]; then
 344         vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
 345         printf "%s" "$output";
 346         return 0;
 347     else
 348         yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} .";
 349         return 1;
 350     fi;
 351 }; # Print to stdout path of OTS file with oldest block
 352 get_block_num_from_stored_ots_path() {
 353     # Desc: Return block number from stored OTS path
 354     # Input:  arg1    input file path
 355     # Output: stdout  block number (int)
 356     # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'.
 357     local fin fbase block re;
 358     fpath="$1";
 359     fbase="$(basename "$fpath")";
 360     block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")";
 361     re='[0-9]+';
 362     if [[ "$block" =~ $re ]]; then
 363         printf "%s" "$block";
 364     else
 365         yell "ERROR:Invalid block number:$(declare -p fpath fbase block)";
 366         return 1;
 367     fi;
 368 }; # Print block number from stored OTS file
 369 store_and_lookup() {
 370     # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible
 371     # Usage: store_and_lookup [path]
 372     # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock()
 373     local pathFileIn="$1";
 374     vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}";
 375
 376     # Validate path
 377     if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi;
 378
 379     # Check for and store any OTS file attached to provided file
 380     ## Check if provided file is an OTS file itself
 381     if [[ "$pathFileIn" =~ \.ots$ ]]; then
 382         vbm "STATUS:The provided file is itself an OTS file. Store OTS file only.";
 383         store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file.";
 384         return 0;
 385     fi;
 386     ## Check if provided file has an accompanying OTS file
 387     if [[ -f "${pathFileIn}.ots" ]]; then
 388         vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots";
 389         store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file.";
 390     fi;
 391
 392     # Lookup OTS file from archive for provided file.
 393     ## Get file hash
 394     fhash="$(get_sha256_digest "$pathFileIn"; )";
 395
 396     ## Get stored OTS path if possible.
 397     if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
 398         yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}";
 399         return 0;
 400     fi;
 401     vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}.";
 402     blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
 403     vbm "STATUS:The stored OTS file has block number ${blk_stored}.";
 404
 405     ## Check for OTS file accompanying provided file
 406     if [[ -f "${pathFileIn}.ots" ]]; then
 407         vbm "STATUS:An OTS file is next to provided file ${pathFileIn}.";
 408         blk_provid="$(must get_ots_oldestblock "${pathFileIn}.ots"; )";
 409         vbm "STATUS:The provided file's OTS file has block number ${blk_provid}";
 410         re='[0-9]+';
 411         if [[ ! "$blk_stored" =~ $re ]] || [[ ! "$blk_provid" =~ $re ]]; then
 412             die "FATAL:Invalid block number(s):$(declare -p blk_stored blk_provid)";
 413         fi;
 414         if [[ "$blk_stored" -lt "$blk_provid" ]]; then
 415             vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
 416             if [[ ! -f "${pathFileIn}.ots.baku" ]]; then
 417                 must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \
 418                     vbm "STATUS:Backed up existing OTS file.";
 419             else
 420                 must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \
 421                     yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present.";
 422             fi;
 423             must cp "$path_stored_ots" "${pathFileIn}.ots" && \
 424                 vbm "STATUS:Replaced provided OTS file with stored OTS file.";
 425             return 0;
 426         else
 427             yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} .";
 428         fi;
 429     else
 430         vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots";
 431         must cp "$path_stored_ots" "${pathFileIn}.ots";
 432         return 0;
 433     fi;
 434 }; # stores provided OTS files and retrieves older OTS files if available
 435 count_jobs() {
 436     # Desc: Count and return total number of jobs
 437     # Usage: count_jobs
 438     # Input: None.
 439     # Output: stdout   integer number of jobs
 440     # Depends: Bash 5.1.16
 441     # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
 442     # Version: 0.0.1
 443
 444     local job_count;
 445     job_count="$(jobs -r | wc -l | tr -d ' ' )";
 446     #yell "DEBUG:job_count:$job_count";
 447     if [[ -z $job_count ]]; then job_count="0"; fi;
 448     echo "$job_count";
 449 }; # Return number of background jobs
 450 main() {
 451     checkDepends;
 452     processArgs "$@";
 453     vbm "DEBUG:Starting rest of main()";
 454     vbm "$(declare -p pathsFilesIn)";
 455
 456     # Process files from provided input args
 457     for fpath in "${pathsFilesIn[@]}"; do
 458         # throttle if too many jobs
 459         if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then
 460             sleep 0.01;
 461         fi;
 462
 463         # start new job
 464         must store_and_lookup "$fpath" &
 465     done;
 466     wait;
 467
 468 }; # main program
 469
 470 main "$@";
 471
 472 # Author: Steven Baltakatei Sandoval
 473 # License: GPLv3+