user/bkotslu

   1 #!/bin/bash
   2 # Desc: Utility for backing up and retrieving ots files
   3 # Usage: bkotslu -I [dir]
   4 # Version: 0.1.0
   5 # Depends: OpenTimestamps 0.7.0 (see https://opentimestamps.org )
   6 #          GNU Coreutils 8.32
   7 # NOTE: This script does not verify OTS files; it assumes the contents of OTS files fed to it are valid.
   8
   9 OTS_FCACHE_DIR="$HOME/.cache/bkotslu/";
  10 MAX_FIND_DEPTH=1;
  11 MAX_JOBS=32;
  12
  13 declare -a pathsFilesIn;
  14
  15 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
  16 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
  17 must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
  18 vbm() {
  19     # Description: Prints verbose message ("vbm") to stderr if opVerbose is set to "true".
  20     # Usage: vbm "DEBUG :verbose message here"
  21     # Version 0.2.0
  22     # Input: arg1: string
  23     #        vars: opVerbose
  24     # Output: stderr
  25     # Depends: bash 5.1.16, GNU-coreutils 8.30 (echo, date)
  26
  27     if [ "$opVerbose" = "true" ]; then
  28         functionTime="$(date --iso-8601=ns)"; # Save current time in nano seconds.
  29         echo "[$functionTime]:$0:""$*" 1>&2;  # Display argument text.
  30     fi
  31
  32     # End function
  33     return 0; # Function finished.
  34 }; # Displays message if opVerbose true
  35 showUsage() {
  36     # Desc: Display script usage information
  37     # Usage: showUsage
  38     # Version 0.0.2
  39     # Input: none
  40     # Output: stdout
  41     # Depends: GNU-coreutils 8.30 (cat)
  42     cat <<'EOF'
  43     USAGE:
  44         bkotslu [ options ] [FILE...]
  45
  46     OPTIONS:
  47         -h, --help
  48                 Display help information.
  49         --version
  50                 Display script version.
  51         -v, --verbose
  52                 Display debugging info.
  53         -i, --input-file [FILE]
  54                 Provide path for file to submit/lookup OTS files for
  55         -I, --input-dir [DIR]
  56                 Provide dir containing files to submit/lookup OTS files for
  57         -O, --output-dir
  58                 Define output directory path for storing ots backup files.
  59                   DEFAULT: $HOME/.cache/bkotslu/
  60         -r, --recursive
  61                 Follow subdirectories in provided directories.
  62         --
  63                 Indicate end of options.
  64
  65     EXAMPLE:
  66       Hash foo.txt and lookup matching ots file
  67         bkotslu -i foo.txt
  68
  69       Store and lookup older ots files of a directory
  70         bkotslu -I $HOME/
  71
  72 EOF
  73 }; # Display information on how to use this script.
  74 showVersion() {
  75     # Desc: Displays script version and license information.
  76     # Usage: showVersion
  77     # Version: 0.0.2
  78     # Input: scriptVersion   var containing version string
  79     # Output: stdout
  80     # Depends: vbm(), yell, GNU-coreutils 8.30
  81
  82     # Initialize function
  83     vbm "DEBUG:showVersion function called."
  84
  85     cat <<'EOF'
  86 bkotslu 0.0.1
  87 Copyright (C) 2024 Steven Baltakatei Sandoval
  88 License GPLv3: GNU GPL version 3
  89 This is free software; you are free to change and redistribute it.
  90 There is NO WARRANTY, to the extent permitted by law.
  91
  92     GNU Coreutils 8.32
  93     Copyright (C) 2020 Free Software Foundation, Inc.
  94     License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
  95     This is free software: you are free to change and redistribute it.
  96     There is NO WARRANTY, to the extent permitted by law.
  97 EOF
  98
  99     # End function
 100     vbm "DEBUG:showVersion function ended."
 101     return 0; # Function finished.
 102 }; # Display script version.
 103 checkDepends() {
 104     vbm "STATUS:Starting checkDepends()";
 105     if ! command -v sha256sum 1>/dev/urandom 2>&1; then
 106         die "FATAL:sha256sum not available."; fi;
 107     if ! command -v grep 1>/dev/urandom 2>&1; then
 108         die "FATAL:grep not available."; fi;
 109     if ! command -v find 1>/dev/urandom 2>&1; then
 110         die "FATAL:find not available."; fi;
 111 };
 112 processArgs() {
 113     # Desc: Processes arguments provided to script.
 114     # Usage: processArgs "$@"
 115     # Version: 1.0.0
 116     # Input: "$@"             (list of arguments provided to the function)
 117     # Output: Sets following variables used by other functions:
 118     #   opVerbose            Indicates verbose mode enable status.  (ex: "true", "false")
 119     #   pathOtsStore          Path to output directory.
 120     #X   pathDirIn1           Path to input directory.
 121     #X   pathFileIn1          Path to input file.
 122     #   arrayPosArgs         Array of remaining positional argments
 123     #   pathsFilesIn         input files
 124     # Depends:
 125     #   yell()           Displays messages to stderr.
 126     #   vbm()            Displays messsages to stderr if opVerbose set to "true".
 127     #   showUsage()      Displays usage information about parent script.
 128     #   showVersion()    Displays version about parent script.
 129     #   arrayPosArgs     Global array for storing non-option positional arguments (i.e. arguments following the `--` option).
 130     # External dependencies: bash (5.1.16), echo
 131     # Ref./Attrib.:
 132     #  [1]: Marco Aurelio (2014-05-08). "echo that outputs to stderr". https://stackoverflow.com/a/23550347
 133     #  [2]: "Handling positional parameters" (2018-05-12). https://wiki.bash-hackers.org/scripting/posparams
 134
 135     # Initialize function
 136     vbm "DEBUG:processArgs function called."
 137
 138     # Perform work
 139     if [ $# -le 0 ]; then yell "FATAL:No arguments provided."; showUsage; fi;
 140     while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
 141         #yell "DEBUG:Starting processArgs while loop." # Debug stderr message. See [1].
 142         #yell "DEBUG:Provided arguments are:""$*"      # Debug stderr message. See [1].
 143         case "$1" in
 144             -h | --help) showUsage; exit 1;; # Display usage.
 145             --version) showVersion; exit 1;; # Show version
 146             -v | --verbose) opVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1].
 147             -r | --recursive) MAX_FIND_DEPTH=12; vbm "DEBUG:Recursive mode enabled.";;
 148             -i | --input-file) # Define input file path
 149                 if [ -f "$2" ]; then # If $2 is file that exists, add $2 to pathsFilesIn array, then pop $2.
 150                     pathsFilesIn+=("$(readlink -f "$2")");
 151                     vbm "DEBUG:Added to pathsFilesIn array:$2";
 152                     shift;
 153                 else
 154                     die "FATAL:The provided input file does not exist:$2";
 155                 fi;;
 156             -I | --input-dir) # Define input directory path
 157                 if [ -d "$2" ]; then # If $2 is dir that exists, find and save files in $2 to pathsFilesIn array, then pop $2.
 158                     while read -r file; do
 159                         file="$(readlink -f "$file")";
 160                         vbm "STATUS:Added to pathsFilesIn array:${file}";
 161                         pathsFilesIn+=("$(readlink -f "$file")");
 162                     done < <(find "$2" -maxdepth "$MAX_FIND_DEPTH" -type f);
 163                     vbm "DEBUG:Added to pathsFilesIn array the contents of dir:$2";
 164                     shift;
 165                 else # Display error if $2 is not a valid dir.
 166                     die "FATAL:The specified input directory does not exist:$2";
 167                 fi;;
 168             -O | --output-dir) # Define output directory path
 169                 if [ -d "$2" ]; then # If $2 is dir that exists, set pathOtsStore to $2, pop $2
 170                     pathOtsStore="$2";
 171                     vbm "DEBUG:Output directory pathOtsStore set to:${pathOtsStore}";
 172                     shift;
 173                 else
 174                     die "FATAL:The specified output directory is not valid:$2";
 175                 fi;;
 176             --) # End of all options. See [2].
 177                 shift;
 178                 for arg in "$@"; do
 179                     vbm "DEBUG:adding to arrayPosArgs:$arg";
 180                     arrayPosArgs+=("$arg");
 181                 done;
 182                 break;;
 183             -*) showUsage; die "FATAL: Unrecognized option.";; # Display usage
 184             *) showUsage; die "FATAL: Unrecognized argument.";; # Handle unrecognized options. See [1].
 185         esac;
 186         shift;
 187     done;
 188
 189     ## Identify ots file cache dir
 190     if [[ -z "$pathOtsStore" ]]; then
 191         vbm "STATUS:No output directory for caching OTS files specified.";
 192         pathOtsStore="$OTS_FCACHE_DIR";
 193         vbm "STATUS:Assuming OTS files to be cached in:${pathOtsStore}";
 194     fi;
 195     ## Create cache dir if necessary
 196     if [[ ! -d "$pathOtsStore" ]]; then
 197         vbm "STATUS:Creating OTS file cache directory:${pathOtsStore}";
 198         must mkdir -p "$pathOtsStore";
 199     fi;
 200
 201     # End function
 202     vbm "DEBUG:processArgs function ended.";
 203     return  0; # Function finished.
 204 }; # Evaluate script options from positional arguments (ex: $1, $2, $3, etc.).
 205 get_ots_filehash() {
 206     # Desc: Gets hash of an opentimestamp'd file from ots file
 207     # Usage: get_ots_filehash FILE
 208     # Example: get_ots_filehash foo.txt.ots
 209     # Depends: ots 0.7.0, GNU grep 3.7, GNU Coreutils 8.32
 210     #          vbm()              verbose output
 211     #          BK-2020-03  yell()
 212     # Input:  arg1    OTS file path
 213     # Output: stdout  sha256 file hash (lowercase)
 214     local output;
 215     vbm "DEBUG:Starting get_ots_filehash() on:$1";
 216
 217     if output="$( "$(which ots)" info "$1" | \
 218             grep -E "^File sha256 hash: " | \
 219             head -n1 | \
 220             sed -E -e 's/(^File sha256 hash: )([0-9a-f]+$)/\2/g'; )" && \
 221             [[ -n "$output" ]]; then
 222         vbm "STATUS:Read file digest (${output}) via ots from:$1";
 223         printf "%s" "$output";
 224         return 0;
 225     else
 226         yell "ERROR:Encountered problem getting file hash via ots from:$1";
 227         return 1;
 228     fi;
 229 }; # Gets hash of file from ots file
 230 get_ots_oldestblock() {
 231     # Desc: Gets earliest Bitcoin block number from ots file
 232     # Usage: get_ots_oldestblock FILE
 233     # Example: get_ots_oldestblock foo.txt.ots
 234     # Input: arg1     path  OTS file path
 235     # Output: stdout  int   Bitcoin block number
 236     # Depends: OpenTimestamps 0.7.0, GNU grep 3.7, GNU Coreutils 8.32
 237     #          vbm()
 238     #          BK-2020-03: yell()
 239     local output;
 240     vbm "DEBUG:Starting get_ots_oldestblock() on:$1";
 241
 242     if output="$( "$(which ots)" info "$1" | \
 243             grep -E "verify BitcoinBlockHeaderAttestation\([0-9]+\)" | \
 244             sort | head -n1 | \
 245             sed -E -e 's/(^    verify BitcoinBlockHeaderAttestation)\(([0-9]+)(\))/\2/g'; )" && \
 246             [[ -n "$output" ]]; then
 247         vbm "STATUS:Retrieved Bitcoin block (${output}) via ots from:$1";
 248         printf "%s" "$output";
 249         return 0;
 250     else
 251         yell "ERROR:Encountered problem getting Bitcoin block number via ots from:$1";
 252         return 1;
 253     fi;
 254 }; # Gets oldest Bitcoin block from ots file
 255 store_ots_file() {
 256     # Desc: Scans and stores an OTS file if none already stored
 257     # Usage: store_ots_file FILE
 258     # Example: store_ots_file foo.txt.ots
 259     # Input:  arg1  OTS file
 260     # Output: exit code
 261     local fin="$1";
 262     vbm "STATUS:Starting store_ots_file()";
 263
 264     # Check if provided OTS file exists
 265     if [[ ! -f "$fin" ]]; then die "FATAL:OTS file not found:$fin"; fi;
 266
 267     # Read file hash and oldest block from provided OTS file
 268     if ! { fhash="$(must get_ots_filehash "$fin")" && \
 269                block="$(must get_ots_oldestblock "$fin")"; }; then
 270         yell "ERROR:Problem analyzing file with OpenTimestamps:${fin}";
 271         return 1;
 272     fi;
 273     vbm "STATUS:The provided OTS file at ${fin} has digest ${fhash} and block ${block}.";
 274
 275     # Copy provided OTS if no matching OTS stored
 276     fout="${fhash}_${block}.otsu"; # file name out
 277     pout="${pathOtsStore}/${fout}"; # file path out
 278     if [[ ! -f "$pout" ]]; then
 279         vbm "STATUS:No matching stored OTS file found. Copying provided file to store at:${pout}";
 280         must cp -n "$fin" "$pout";
 281         return 0;
 282     else
 283         vbm "STATUS:Stored OTS file with matching file hash and block number in file name found.";
 284     fi;
 285
 286     # Get block number for provided and stored OTS files.
 287     if ! { blk_provid="$block"; blk_stored="$(get_ots_oldestblock "$pout"; )"; }; then
 288         yell "ERROR:Could not read block numbers from OTS files: $(declare -p fhash block pout )";
 289     fi;
 290
 291     # Copy provided OTS if matching OTS found stored but provided is older
 292     if [[ "$blk_provid" -lt "$blk_stored" ]]; then
 293         vbm "WARNING:Provided OTS file somehow older despite having same name. Previous error in storing OTS file?";
 294         must mv "$pout" "${pout}--$(date +%s)";
 295         must cp "$fin" "$pout";
 296         return 0;
 297     else
 298         vbm "STATUS:Stored OTS file has block number older than or as old as provided OTS file.";
 299     fi;
 300 }; # Stores provided OTS file is none already stored
 301 get_sha256_digest() {
 302     # Depends: GNU Coreutils 8.32 (sha256sum)
 303     # Input:  arg1    path  file path
 304     # Output: stdout  str   sha256 digest (lowercase hexadecimal)
 305     vbm "DEBUG:Starting get_sha256_digest()";
 306     sha256sum "$1" | head -n1 | sed -E -e 's/(^[0-9a-f]{64})(.+)/\1/';
 307 };
 308 get_oldest_stored_ots_path() {
 309     # Desc: Lookup most recent OTS file from storage
 310     # Input:  pathOtsStore  var  path to OTS storage dir
 311     #         arg1         str  sha256 digest (lowercase hexadecimal)
 312     # Output: stdout       path OTS file with matching sha256 digest
 313     vbm "DEBUG:Starting get_oldest_stored_ots_path()";
 314     local -a otsStorePaths;
 315     local i_oldest;
 316
 317     digest="$1";
 318     mapfile -t otsStorePaths < <(find "$pathOtsStore" -type f -name "${digest}*.otsu"; );
 319     if [[ "${#otsStorePaths[@]}" -le 0 ]]; then
 320         yell "ERROR:No OTS file in OTS storage dir found. $(declare -p pathOtsStore digest otsStorePaths)";
 321         return 1;
 322     fi;
 323     i_oldest=0;
 324     blockNumOldest="$( get_block_num_from_stored_ots_path "${otsStorePaths[0]}" )";
 325     for ((i=0; i<"${#otsStorePaths[@]}"; i++ )); do
 326         blockNum="$( get_block_num_from_stored_ots_path "${otsStorePaths[$i]}" )";
 327         if [[ $blockNum -lt $blockNumOldest ]]; then
 328             blockNumOldest=$blockNum;
 329             i_oldest=$i;
 330         fi;
 331     done;
 332     output="$(readlink -f "${otsStorePaths[$i_oldest]}"; )";
 333
 334     if [[ -n "$output" ]] && [[ -f "$output" ]]; then
 335         vbm "STATUS:Found matching OTS file with digest ${digest} at:$output";
 336         printf "%s" "$output";
 337         return 0;
 338     else
 339         yell "ERROR:Could not find matching OTS file with digest ${digest} in ${pathOtsStore} .";
 340         return 1;
 341     fi;
 342 }; # Print to stdout path of OTS file with oldest block
 343 get_block_num_from_stored_ots_path() {
 344     # Desc: Return block number from stored OTS path
 345     # Input:  arg1    input file path
 346     # Output: stdout  block number (int)
 347     # Note: Assumes OTS file name pattern '{digest}_{blockNum}.otsu'.
 348     local fin fbase block re;
 349     fpath="$1";
 350     fbase="$(basename "$fpath")";
 351     block="$(sed -E -e 's/^.+_([0-9]+).otsu$/\1/g' <<< "$fbase")";
 352     re='[0-9]+';
 353     if [[ "$block" =~ $re ]]; then
 354         printf "%s" "$block";
 355     else
 356         yell "ERROR:Invalid block number:$(declare -p fpath fbase block)";
 357         return 1;
 358     fi;
 359 }; # Print block number from stored OTS file
 360 store_and_lookup() {
 361     # Desc: Stores provided file's OTS files and retrieves older OTS files from storage if possible
 362     # Usage: store_and_lookup [path]
 363     # Depends: get_sha256_digest(), get_oldest_stored_ots_path(), get_ots_oldestblock()
 364     local pathFileIn="$1";
 365     vbm "DEBUG:Starting store_and_lookup() with provided file:${pathFileIn}";
 366
 367     # Validate path
 368     if [[ ! -f "$pathFileIn" ]]; then yell "ERROR:Not a file:${pathFileIn}"; return 1; fi;
 369
 370     # Check for and store any OTS file attached to provided file
 371     ## Check if provided file is an OTS file itself
 372     if [[ "$pathFileIn" =~ \.ots$ ]]; then
 373         vbm "STATUS:The provided file is itself an OTS file. Store OTS file only.";
 374         store_ots_file "$pathFileIn" && vbm "STATUS:Stored provided OTS file.";
 375         return 0;
 376     fi;
 377     ## Check if provided file has an accompanying OTS file
 378     if [[ -f "${pathFileIn}.ots" ]]; then
 379         vbm "STATUS:The provided file is accompanied by an OTS file:${pathFileIn}.ots";
 380         store_ots_file "${pathFileIn}.ots" && vbm "STATUS:Stored provided file's OTS file.";
 381     fi;
 382
 383     # Lookup OTS file from archive for provided file.
 384     ## Get file hash
 385     fhash="$(get_sha256_digest "$pathFileIn"; )";
 386
 387     ## Get stored OTS path if possible.
 388     if ! path_stored_ots="$(get_oldest_stored_ots_path "$fhash"; )"; then
 389         yell "STATUS:No stored OTS found. No action taken for:${pathFileIn}";
 390         return 0;
 391     fi;
 392     vbm "STATUS:A stored OTS found with matching hash for provided file ${pathFileIn}.";
 393     blk_stored="$(get_ots_oldestblock "$path_stored_ots"; )";
 394     vbm "STATUS:The stored OTS file has block number ${blk_stored}.";
 395
 396     ## Check for OTS file accompanying provided file
 397     if [[ -f "${pathFileIn}.ots" ]]; then
 398         vbm "STATUS:An OTS file is next to provided file ${pathFileIn}.";
 399         blk_provid="$(get_ots_oldestblock "${pathFileIn}.ots"; )";
 400         vbm "STATUS:The provided file's OTS file has block number ${blk_provid}";
 401         if [[ "$blk_stored" -lt "$blk_provid" ]]; then
 402             vbm "STATUS:An older timestamp in OTS store found. Replacing ${pathFileIn}.ots (block ${blk_provid}) with ${path_stored_ots} (block ${blk_stored}).";
 403             if [[ ! -f "${pathFileIn}.ots.baku" ]]; then
 404                 must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku" && \
 405                     vbm "STATUS:Backed up existing OTS file.";
 406             else
 407                 must mv "${pathFileIn}.ots" "${pathFileIn}.ots.baku--$(date +%s)" && \
 408                     yell "STATUS:Backed up existing OTS file with Unix epoch since backup OTS file already present.";
 409             fi;
 410             must cp "$path_stored_ots" "${pathFileIn}.ots" && \
 411                 vbm "STATUS:Replaced provided OTS file with stored OTS file.";
 412             return 0;
 413         else
 414             yell "STATUS:The stored OTS file (block ${blk_stored}) is not older than provided file's OTS file (block ${blk_provid}). No action taken for:${pathFileIn} .";
 415         fi;
 416     else
 417         vbm "STATUS:No accompanying OTS file found and stored OTS file found with digest matching provided file. Copying ${path_stored_ots} to ${pathFileIn}.ots";
 418         must cp "$path_stored_ots" "${pathFileIn}.ots";
 419         return 0;
 420     fi;
 421 }; # stores provided OTS files and retrieves older OTS files if available
 422 count_jobs() {
 423     # Desc: Count and return total number of jobs
 424     # Usage: count_jobs
 425     # Input: None.
 426     # Output: stdout   integer number of jobs
 427     # Depends: Bash 5.1.16
 428     # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done;
 429     # Version: 0.0.1
 430
 431     local job_count;
 432     job_count="$(jobs -r | wc -l | tr -d ' ' )";
 433     #yell "DEBUG:job_count:$job_count";
 434     if [[ -z $job_count ]]; then job_count="0"; fi;
 435     echo "$job_count";
 436 }; # Return number of background jobs
 437 main() {
 438     checkDepends;
 439     processArgs "$@";
 440     vbm "DEBUG:Starting rest of main()";
 441     vbm "$(declare -p pathsFilesIn)";
 442
 443     # Process files from provided input args
 444     for fpath in "${pathsFilesIn[@]}"; do
 445         # throttle if too many jobs
 446         if [[ "$(count_jobs)" -ge "$MAX_JOBS" ]]; then
 447             sleep 0.01;
 448         fi;
 449
 450         # start new job
 451         must store_and_lookup "$fpath" &
 452     done;
 453     wait;
 454
 455 }; # main program
 456
 457 main "$@";
 458
 459 # Author: Steven Baltakatei Sandoval
 460 # License: GPLv3+