exec/bklog

   1 #!/bin/bash
   2 # Desc: Compresses, encrypts, and writes stdin every 5 seconds
   3
   4 #==BEGIN Define script parameters==
   5 #===BEGIN Initialize variables===
   6
   7 # Logging Behavior parameters
   8 bufferTTL="300"; # Time-to-live (seconds) for each buffer round
   9 scriptTTL_TE="day"; # Time element at the end of which script terminates
  10 dirTmpDefault="/dev/shm"; # Default parent of working directory
  11
  12 # Script Metadata
  13 scriptName="bklog";             # Define basename of script file.
  14 scriptVersion="0.1.2";          # Define version of script.
  15 scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
  16 scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN
  17 scriptHostname=$(hostname);     # Save hostname of system running this script.
  18 PATH="$HOME/.local/bin:$PATH";  # Add "$(systemd-path user-binaries)" path in case user apps saved there
  19 ageVersion="1.0.0-beta2";       # Define version of age (encryption program)
  20 ageURL="https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2"; # Define website hosting age.
  21
  22 # Arrays
  23 declare -a buffer          # array for storing while read buffer
  24 declare -a argRecPubKeys   # array for processArguments function
  25 declare -a recPubKeysValid # array for storing both '-r' and '-R' recipient pubkeys
  26 declare -a recPubKeysValidStatic # for storing '-r' recipient pubkeys
  27 declare -a argProcStrings argProcFileExts # for storing buffer processing strings (ex: "gpsbabel -i nmea -f - -o gpx -F - ")
  28 declare -Ag appRollCall    # Associative array for storing app status
  29 declare -Ag fileRollCall   # Associative array for storing file status
  30 declare -Ag dirRollCall    # Associative array for storing dir status
  31 declare -a procStrings procFileExts # Arrays for storing processing commands and resulting output file extensions
  32
  33 # Variables
  34 optionVerbose=""; optionEncrypt=""; dirOut=""; optionEncrypt=""; dir_tmp="";
  35 cmd_compress="";cmd_compress_suffix=""; cmd_encrypt=""; cmd_encrypt_suffix="";
  36
  37 #===END Initialize variables===
  38
  39 #===BEGIN Declare local script functions===
  40 yell() { echo "$0: $*" >&2; }      #o Yell, Die, Try Three-Fingered Claw technique
  41 die() { yell "$*"; exit 111; }     #o Ref/Attrib: https://stackoverflow.com/a/25515370
  42 try() { "$@" || die "cannot $*"; } #o
  43 processArguments() {
  44     while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
  45         case "$1" in
  46             -v | --verbose) optionVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode.
  47             -h | --help) showUsage; exit 1;; # Display usage.
  48             --version) showVersion; exit 1;; # Show version
  49             -o | --output) if [ -d "$2" ]; then dirOut="$2"; vbm "DEBUG:dirOut:$dirOut"; shift; fi ;; # Define output directory.
  50             -e | --encrypt) optionEncrypt="true"; vbm "DEBUG:Encrypted output mode enabled.";; # Enable encryption
  51             -r | --recipient) optionRecipients="true"; argRecPubKeys+=("$2"); vbm "STATUS:pubkey added:""$2"; shift;; # Add recipients
  52             -c | --compress) optionCompress="true"; vbm "DEBUG:Compressed output mode enabled.";; # Enable compression
  53             -z | --time-zone) try setTimeZoneEV "$2"; shift;; # Set timestamp timezone
  54             -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone
  55             -R | --recipient-dir) optionRecipients="true"; optionRecDir="true" && argRecDir="$2"; shift;; # Add recipient watch dir
  56             -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds)
  57             -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day")
  58             -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;;
  59             -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG:Custom label received:$argLabel"; shift;;
  60             -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG:Raw stdin file extension received:$argRawFileExt"; shift;;
  61             -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG:Option selected to not store raw stdin data."; shift;;
  62             *) yell "ERROR: Unrecognized argument: $1"; yell "STATUS:All arguments:$*"; exit 1;; # Handle unrecognized options.
  63         esac
  64         shift
  65     done
  66 } # Argument Processing
  67 vbm() {
  68     # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true".
  69     # Usage: vbm "DEBUG:verbose message here"
  70     # Version 0.1.2
  71     # Input: arg1: string
  72     #        vars: optionVerbose
  73     # Output: stderr
  74     # Depends: bash 5.0.3, echo 8.30, date 8.30
  75
  76     if [ "$optionVerbose" = "true" ]; then
  77         functionTime=$(date --iso-8601=ns); # Save current time in nano seconds.
  78         echo "[$functionTime] ""$*" 1>&2; # Display argument text.
  79     fi
  80
  81     # End function
  82     return 0; # Function finished.
  83 } # Displays message if optionVerbose true
  84 checkapp() {
  85     # Desc: If arg is a command, save result in assoc array 'appRollCall'
  86     # Usage: checkapp arg1 arg2 arg3 ...
  87     # Version: 0.1.1
  88     # Input: global assoc. array 'appRollCall'
  89     # Output: adds/updates key(value) to global assoc array 'appRollCall'
  90     # Depends: bash 5.0.3
  91     local returnState
  92
  93     #===Process Args===
  94     for arg in "$@"; do
  95         if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
  96             appRollCall[$arg]="true";
  97             if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
  98         else
  99             appRollCall[$arg]="false"; returnState="false";
 100         fi;
 101     done;
 102
 103     #===Determine function return code===
 104     if [ "$returnState" = "true" ]; then
 105         return 0;
 106     else
 107         return 1;
 108     fi;
 109 } # Check that app exists
 110 checkfile() {
 111     # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
 112     # Usage: checkfile arg1 arg2 arg3 ...
 113     # Version: 0.1.1
 114     # Input: global assoc. array 'fileRollCall'
 115     # Output: adds/updates key(value) to global assoc array 'fileRollCall';
 116     # Output: returns 0 if app found, 1 otherwise
 117     # Depends: bash 5.0.3
 118     local returnState
 119
 120     #===Process Args===
 121     for arg in "$@"; do
 122         if [ -f "$arg" ]; then
 123             fileRollCall["$arg"]="true";
 124             if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
 125         else
 126             fileRollCall["$arg"]="false"; returnState="false";
 127         fi;
 128     done;
 129
 130     #===Determine function return code===
 131     if [ "$returnState" = "true" ]; then
 132         return 0;
 133     else
 134         return 1;
 135     fi;
 136 } # Check that file exists
 137 checkdir() {
 138     # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
 139     # Usage: checkdir arg1 arg2 arg3 ...
 140     # Version 0.1.1
 141     # Input: global assoc. array 'dirRollCall'
 142     # Output: adds/updates key(value) to global assoc array 'dirRollCall';
 143     # Output: returns 0 if app found, 1 otherwise
 144     # Depends: Bash 5.0.3
 145     local returnState
 146
 147     #===Process Args===
 148     for arg in "$@"; do
 149         if [ -d "$arg" ]; then
 150             dirRollCall["$arg"]="true";
 151             if ! [ "$returnState" = "false" ]; then returnState="true"; fi
 152         else
 153             dirRollCall["$arg"]="false"; returnState="false";
 154         fi
 155     done
 156
 157     #===Determine function return code===
 158     if [ "$returnState" = "true" ]; then
 159         return 0;
 160     else
 161         return 1;
 162     fi
 163 } # Check that dir exists
 164 displayMissing() {
 165     # Desc: Displays missing apps, files, and dirs
 166     # Usage: displayMissing
 167     # Version 0.1.1
 168     # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
 169     # Output: stderr: messages indicating missing apps, file, or dirs
 170     # Depends: bash 5, checkAppFileDir()
 171     local missingApps value appMissing missingFiles fileMissing
 172     local missingDirs dirMissing
 173
 174     #==BEGIN Display errors==
 175     #===BEGIN Display Missing Apps===
 176     missingApps="Missing apps  :";
 177     #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
 178     for key in "${!appRollCall[@]}"; do
 179         value="${appRollCall[$key]}";
 180         if [ "$value" = "false" ]; then
 181             #echo "DEBUG:Missing apps: $key => $value";
 182             missingApps="$missingApps""$key ";
 183             appMissing="true";
 184         fi;
 185     done;
 186     if [ "$appMissing" = "true" ]; then  # Only indicate if an app is missing.
 187         echo "$missingApps" 1>&2;
 188     fi;
 189     unset value;
 190     #===END Display Missing Apps===
 191
 192     #===BEGIN Display Missing Files===
 193     missingFiles="Missing files:";
 194     #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
 195     for key in "${!fileRollCall[@]}"; do
 196         value="${fileRollCall[$key]}";
 197         if [ "$value" = "false" ]; then
 198             #echo "DEBUG:Missing files: $key => $value";
 199             missingFiles="$missingFiles""$key ";
 200             fileMissing="true";
 201         fi;
 202     done;
 203     if [ "$fileMissing" = "true" ]; then  # Only indicate if an app is missing.
 204         echo "$missingFiles" 1>&2;
 205     fi;
 206     unset value;
 207     #===END Display Missing Files===
 208
 209     #===BEGIN Display Missing Directories===
 210     missingDirs="Missing dirs:";
 211     #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
 212     for key in "${!dirRollCall[@]}"; do
 213         value="${dirRollCall[$key]}";
 214         if [ "$value" = "false" ]; then
 215             #echo "DEBUG:Missing dirs: $key => $value";
 216             missingDirs="$missingDirs""$key ";
 217             dirMissing="true";
 218         fi;
 219     done;
 220     if [ "$dirMissing" = "true" ]; then  # Only indicate if an dir is missing.
 221         echo "$missingDirs" 1>&2;
 222     fi;
 223     unset value;
 224     #===END Display Missing Directories===
 225
 226     #==END Display errors==
 227 } # Display missing apps, files, dirs
 228 showUsage() {
 229     cat <<'EOF'
 230     USAGE:
 231         cmd | bklog [ options ]
 232     echoerr
 233     OPTIONS:
 234         -h, --help
 235                 Display help information.
 236         --version
 237                 Display script version.
 238         -v, --verbose
 239                 Display debugging info.
 240         -e, --encrypt
 241                 Encrypt output.
 242         -r, --recipient [ string pubkey ]
 243                 Specify recipient. May be age or ssh pubkey.
 244                 May be specified multiple times for multiple pubkeys.
 245                 See https://github.com/FiloSottile/age
 246         -o, --output [ path dir ]
 247                 Specify output directory to save logs. This option is required
 248                 to save log data.
 249         -p, --process-string [ filter command ] [ output file extension]
 250                 Specify how to create and name a processed version of the stdin.
 251                 For example, if stdin is 'nmea' location data:
 252
 253                 -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx"
 254
 255                 This option would cause the stdin to 'bklog' to be piped into
 256                 the 'gpsbabel' command, interpreted as 'nmea' data, converted
 257                 into 'gpx' format, and then appended to the output tar file
 258                 as a file with a '.gpx' extension.
 259                 This option may be specified multiple times in order to output
 260                 results of multiple different processing methods.
 261         -l, --label [ string ]
 262                 Specify a label to be included in all output file names.
 263                 Ex: 'location' if stdin is location data.
 264         -w, --store-raw [ file extension ]
 265                 Specify file extension of file within output tar that contains
 266                 raw stdin data. The default behavior is to always save raw stdin
 267                 data in a '.stdin' file. Example usage when 'bklog' receives
 268                 'nmea' data from 'gpspipe -r':
 269
 270                 -w ".nmea"
 271
 272                 Stdin data is saved in a '.nmea' file within the output tar.
 273         -W, --no-store-raw
 274                 Do not store raw stdin in output tar.
 275         -c, --compress
 276                 Compress output with gzip (before encryption if enabled).
 277         -z, --time-zone
 278                 Specify time zone. (ex: "America/New_York")
 279         -t, --temp-dir [path dir]
 280                 Specify parent directory for temporary working directory.
 281                 Default: "/dev/shm"
 282         -R, --recipient-dir [path dir]
 283                 Specify directory containing files whose first lines are
 284                 to be interpreted as pubkey strings (see '-r' option).
 285         -b, --buffer-ttl [integer]
 286                 Specify custom buffer period in seconds (default: 300 seconds)
 287         -B, --script-ttl [time element string]
 288                 Specify custom script time-to-live in seconds (default: "day")
 289                 Valid values: "day", "hour"
 290     echoerr
 291     EXAMPLE: (bash script lines)
 292     $ gpspipe -r | /bin/bash bklog -v -e -c -z "UTC" -t "/dev/shm" \
 293     -r age1mrmfnwhtlprn4jquex0ukmwcm7y2nxlphuzgsgv8ew2k9mewy3rs8u7su5 \
 294     -r age1ala848kqrvxc88rzaauc6vc5v0fqrvef9dxyk79m0vjea3hagclswu0lgq \
 295     -R ~/.config/bklog/recipients -w ".nmea" -b 300 -B "day" \
 296     -o ~/Sync/Logs -l "location" \
 297     -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" \
 298     -p "gpsbabel -i nmea -f - -o kml -F - " ".gpx"
 299 EOF
 300 } # Display information on how to use this script.
 301 showVersion() {
 302     yell "$scriptVersion"
 303 } # Display script version.
 304 setTimeZoneEV(){
 305     # Desc: Set time zone environment variable TZ
 306     # Usage: setTimeZoneEV arg1
 307     # Version 0.1.2
 308     # Input: arg1: 'date'-compatible timezone string (ex: "America/New_York")
 309     #        TZDIR env var (optional; default: "/usr/share/zoneinfo")
 310     # Output: exports TZ
 311     #         exit code 0 on success
 312     #         exit code 1 on incorrect number of arguments
 313     #         exit code 2 if unable to validate arg1
 314     # Depends: yell(), printenv 8.30, bash 5.0.3
 315     # Tested on: Debian 10
 316     local tzDir returnState argTimeZone
 317
 318     argTimeZone="$1"
 319     if ! [[ $# -eq 1 ]]; then
 320         yell "ERROR:Invalid argument count.";
 321         return 1;
 322     fi
 323
 324     # Read TZDIR env var if available
 325     if printenv TZDIR 1>/dev/null 2>&1; then
 326         tzDir="$(printenv TZDIR)";
 327     else
 328         tzDir="/usr/share/zoneinfo";
 329     fi
 330
 331     # Validate TZ string
 332     if ! [[ -f "$tzDir"/"$argTimeZone" ]]; then
 333         yell "ERROR:Invalid time zone argument.";
 334         return 2;
 335     else
 336     # Export ARG1 as TZ environment variable
 337         TZ="$argTimeZone" && export TZ && returnState="true";
 338     fi
 339
 340     # Determine function return code
 341     if [ "$returnState" = "true" ]; then
 342         return 0;
 343     fi
 344 } # Exports TZ environment variable
 345 dateShort(){
 346     # Desc: Date without separators (YYYYmmdd)
 347     # Usage: dateShort ([str date])
 348     # Version: 1.1.2
 349     # Input: arg1: 'date'-parsable timestamp string (optional)
 350     # Output: stdout: date (ISO-8601, no separators)
 351     # Depends: bash 5.0.3, date 8.30, yell()
 352     local argTime timeCurrent timeInput dateCurrentShort
 353
 354     argTime="$1";
 355     # Get Current Time
 356     timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second.
 357     # Decide to parse current or supplied date
 358     ## Check if time argument empty
 359     if [[ -z "$argTime" ]]; then
 360         ## T: Time argument empty, use current time
 361         timeInput="$timeCurrent";
 362     else
 363         ## F: Time argument exists, validate time
 364         if date --date="$argTime" 1>/dev/null 2>&1; then
 365             ### T: Time argument is valid; use it
 366             timeInput="$argTime";
 367         else
 368             ### F: Time argument not valid; exit
 369             yell "ERROR:Invalid time argument supplied. Exiting."; exit 1;
 370         fi;
 371     fi;
 372     # Construct and deliver separator-les date string
 373     dateCurrentShort="$(date -d "$timeInput" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day.
 374     echo "$dateCurrentShort";
 375 } # Get YYYYmmdd
 376 appendFileTar(){
 377     # Desc: Appends [processed] file to tar
 378     # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd])
 379     # Version: 2.0.1
 380     # Input: arg1: path of file to be (processed and) written
 381     #        arg2: name to use for file inserted into tar
 382     #        arg3: tar archive path (must exist first)
 383     #        arg4: temporary working dir
 384     #        arg5: (optional) command string to process file (ex: "gpsbabel -i nmea -f - -o kml -F - ")
 385     # Output: file written to disk
 386     # Example: decrypt multiple large files in parallel
 387     #          appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" &
 388     #          appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" &
 389     #          appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" &
 390     # Depends: bash 5.0.3, tar 1.30, cat 8.30, yell()
 391     local fn fileName tarPath tmpDir
 392
 393     # Save function name
 394     fn="${FUNCNAME[0]}";
 395     #yell "DEBUG:STATUS:$fn:Started appendFileTar()."
 396
 397     # Set file name
 398     if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi
 399     # Check tar path is a file
 400     if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file:$3"; exit 1; fi
 401     # Check temp dir arg
 402     if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi
 403     # Set command strings
 404     if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string
 405
 406     # Input command string
 407     cmd0="cat \"\$1\"";
 408
 409     # Write to temporary working dir
 410     eval "$cmd0 | $cmd1" > "$tmpDir"/"$fileName";
 411
 412     # Append to tar
 413     try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName";
 414     #yell "DEBUG:STATUS:$fn:Finished appendFileTar()."
 415 } # Append [processed] file to Tar archive
 416 magicParseRecipientArgs() {
 417     # Desc: Parses recipient arguments specified by '-r' option
 418     # Input:  vars: optionEncrypt, optionRecipients
 419     #         arry: argRecPubKeys from processArguments()
 420     # Output: vars: cmd_encrypt, cmd_encrypt_suffix
 421     #         arry: recPubKeysValid, recPubKeysValidStatic
 422     # Depends: processArguments(), yell(), vbm(), checkapp(), checkAgePubkey(), validateInput()
 423     local recipients
 424
 425     # Check if encryption option active.
 426     if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then
 427         if checkapp age; then # Check that age is available.
 428             for pubkey in "${argRecPubKeys[@]}"; do # Validate recipient pubkey strings by forming test message
 429                 vbm "DEBUG:Testing pubkey string:$pubkey";
 430                 if checkAgePubkey "$pubkey" && \
 431                         ( validateInput "$pubkey" "ssh_pubkey" || validateInput "$pubkey" "age_pubkey"); then
 432                     #### Form age recipient string
 433                     recipients="$recipients""-r '$pubkey' ";
 434                     vbm "STATUS:Added pubkey for forming age recipient string:""$pubkey";
 435                     vbm "DEBUG:recipients:""$recipients";
 436                     #### Add validated pubkey to recPubKeysValid array
 437                     recPubKeysValid+=("$pubkey") && vbm "DEBUG:recPubkeysValid:pubkey added:$pubkey";
 438                 else
 439                     yell "ERROR:Exit code ""$?"". Invalid recipient pubkey string. Exiting."; exit 1;
 440                 fi;
 441             done
 442             vbm "DEBUG:Finished processing argRecPubKeys array";
 443             vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[*]}";
 444             recPubKeysValidStatic=("${recPubKeysValid[@]}"); # Save static image of pubkeys validated by this function
 445
 446             ##  Form age command string
 447             cmd_encrypt="age ""$recipients " && vbm "cmd_encrypt:$cmd_encrypt";
 448             cmd_encrypt_suffix=".age" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix";
 449         else
 450             yell "ERROR:Encryption enabled but \"age\" not found. Exiting."; exit 1;
 451         fi;
 452     else
 453         cmd_encrypt="tee /dev/null " && vbm "cmd_encrypt:$cmd_encrypt";
 454         cmd_encrypt_suffix="" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix";
 455         vbm "DEBUG:Encryption not enabled."
 456     fi;
 457     # Catch case if '-e' is set but '-r' or '-R' is not
 458     if [[ "$optionEncrypt" = "true" ]] && [[ ! "$optionRecipients" = "true" ]]; then
 459         yell "ERROR:\\'-e\\' set but no \\'-r\\' or \\'-R\\' set."; exit 1; fi;
 460     # Catch case if '-r' or '-R' set but '-e' is not
 461     if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then
 462         yell "ERROR:\\'-r\\' or \\'-R\\' set but \\'-e\\' is not set."; exit 1; fi;
 463 } # Populate recPubKeysValid with argRecPubKeys; form encryption cmd string and filename suffix
 464 magicParseRecipientDir() {
 465     # Desc: Updates recPubKeysValid with pubkeys in dir specified by '-R' option ("recipient directory")
 466     # Inputs:  vars: optionEncrypt, optionRecDir, argRecDir,
 467     #          arry: recPubKeysValid
 468     # Outputs: arry: recPubKeysValid
 469     # Depends: processArguments(), yell(), vbm(), validateInput(),  checkAgePubkey()
 470     local recipientDir recFileLine updateRecipients
 471     declare -a candRecPubKeysValid
 472
 473     # Check that '-e' and '-R' set
 474     if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then
 475         ### Check that argRecDir is a directory.
 476         if [[ -d "$argRecDir" ]]; then
 477             recipientDir="$argRecDir" && vbm "STATUS:Recipient watch directory detected:\"$recipientDir\"";
 478             #### Initialize variable indicating outcome of pubkey review
 479             unset updateRecipients
 480             #### Add existing recipients
 481             candRecPubKeysValid=("${recPubKeysValidStatic[@]}");
 482             #### Parse files in recipientDir
 483             for file in "$recipientDir"/*; do
 484                 ##### Read first line of each file
 485                 recFileLine="$(head -n1 "$file")" && vbm "STATUS:Checking if pubkey:\"$recFileLine\"";
 486                 ##### check if first line is a valid pubkey
 487                 if checkAgePubkey "$recFileLine" && \
 488                         ( validateInput "$recFileLine" "ssh_pubkey" || validateInput "$recFileLine" "age_pubkey"); then
 489                     ###### T: add candidate pubkey to candRecPubKeysValid
 490                     candRecPubKeysValid+=("$recFileLine") && vbm "STATUS:RecDir pubkey is valid pubkey:\"$recFileLine\"";
 491                 else
 492                     ###### F: throw warning;
 493                     yell "ERROR:Invalid recipient file detected. Not modifying recipient list."
 494                     updateRecipients="false";
 495                 fi;
 496             done
 497             #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected
 498             if ! [[ "$updateRecipients" = "false" ]]; then
 499                 recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\"";
 500             fi;
 501         else
 502             yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1;
 503         fi;
 504     fi;
 505     # Handle case if '-R' set but '-e' not set
 506     if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then
 507         yell "ERROR: \\'-R\\' is set but \\'-e\\' is not set."; fi;
 508 } # Update recPubKeysValid with argRecDir
 509 magicParseCompressionArg() {
 510     # Desc: Parses compression arguments specified by '-c' option
 511     # Input:  vars: optionCompress
 512     # Output: cmd_compress, cmd_compress_suffix
 513     # Depends: processArguments(), vbm(), checkapp(), gzip 1.9
 514     if [[ "$optionCompress" = "true" ]]; then # Check if compression option active
 515         if checkapp gzip; then # Check if gzip available
 516             cmd_compress="gzip " && vbm "cmd_compress:$cmd_compress";
 517             cmd_compress_suffix=".gz" && vbm "cmd_compress_suffix:$cmd_compress_suffix";
 518         else
 519             yell "ERROR:Compression enabled but \"gzip\" not found. Exiting."; exit 1;
 520         fi
 521     else
 522         cmd_compress="tee /dev/null " && vbm "cmd_compress:$cmd_compress";
 523         cmd_compress_suffix="" && vbm "cmd_compress_suffix:$cmd_compress_suffix";
 524         vbm "DEBUG:Compression not enabled.";
 525     fi
 526 } # Form compression cmd string and filename suffix
 527 magicInitWorkingDir() {
 528     # Desc: Determine temporary working directory from defaults or user input
 529     # Usage: magicInitWorkingDir
 530     # Input:  vars: optionTmpDir, argTempDirPriority, dirTmpDefault
 531     # Input:  vars: scriptTimeStart
 532     # Output: vars: dir_tmp
 533     # Depends: bash 5.0.3, processArguments(), vbm(), yell()
 534     # Parse '-t' option (user-specified temporary working dir)
 535     ## Set dir_tmp_parent to user-specified value if specified
 536     local dir_tmp_parent
 537
 538     if [[ "$optionTmpDir" = "true" ]]; then
 539         if [[ -d "$argTempDirPriority" ]]; then
 540             dir_tmp_parent="$argTempDirPriority";
 541         else
 542             yell "WARNING:Specified temporary working directory not valid:$argTempDirPriority";
 543             exit 1; # Exit since user requires a specific temp dir and it is not available.
 544         fi;
 545     else
 546     ## Set dir_tmp_parent to default or fallback otherwise
 547         if [[ -d "$dirTmpDefault" ]]; then
 548             dir_tmp_parent="$dirTmpDefault";
 549         elif [[ -d /tmp ]]; then
 550             yell "WARNING:$dirTmpDefault not available. Falling back to /tmp .";
 551             dir_tmp_parent="/tmp";
 552         else
 553             yell "ERROR:No valid working directory available. Exiting.";
 554             exit 1;
 555         fi;
 556     fi;
 557     ## Set dir_tmp using dir_tmp_parent and nonce (scriptTimeStart)
 558     dir_tmp="$dir_tmp_parent"/"$scriptTimeStart""..bkgpslog" && vbm "DEBUG:Set dir_tmp to:$dir_tmp"; # Note: removed at end of main().
 559 } # Sets working dir
 560 magicParseCustomTTL() {
 561     # Desc: Set user-specified TTLs for buffer and script
 562     # Usage: magicParseCustomTTL
 563     # Input: vars: argCustomBufferTTL (integer), argCustomScriptTTL_TE (string)
 564     # Input: vars: optionCustomBufferTTL, optionCustomScriptTTL_TE
 565     # Input: vars: bufferTTL (integer), scriptTTL_TE (string)
 566     # Output: bufferTTL (integer), scriptTTL_TE (string)
 567     # Depends: Bash 5.0.3, yell(), vbm(), validateInput(), showUsage()
 568
 569     # React to '-b, --buffer-ttl' option
 570     if [[ "$optionCustomBufferTTL" = "true" ]]; then
 571         ## T: Check if argCustomBufferTTL is an integer
 572         if validateInput "$argCustomBufferTTL" "integer"; then
 573             ### T: argCustomBufferTTL is an integer
 574             bufferTTL="$argCustomBufferTTL" && vbm "Custom bufferTTL from -b:$bufferTTL";
 575         else
 576             ### F: argcustomBufferTTL is not an integer
 577             yell "ERROR:Invalid integer argument for custom buffer time-to-live."; showUsage; exit 1;
 578         fi;
 579         ## F: do not change bufferTTL
 580     fi;
 581
 582     # React to '-B, --script-ttl' option
 583     if [[ "$optionCustomScriptTTL_TE" = "true" ]]; then
 584         ## T: Check if argCustomScriptTTL is a time element (ex: "day", "hour")
 585         if validateInput "$argCustomScriptTTL_TE" "time_element"; then
 586             ### T: argCustomScriptTTL is a time element
 587             scriptTTL_TE="$argCustomScriptTTL_TE" && vbm "Custom scriptTTL_TE from -B:$scriptTTL_TE";
 588         else
 589             ### F: argcustomScriptTTL is not a time element
 590             yell "ERROR:Invalid time element argument for custom script time-to-live."; showUsage; exit 1;
 591         fi;
 592         ## F: do not change scriptTTL_TE
 593     fi;
 594 } # Sets custom script or buffer TTL if specified
 595 magicSetScriptTTL() {
 596     #Desc: Sets script_TTL seconds from provided time_element string argument
 597     #Usage: magicSetScriptTTL [str time_element]
 598     #Input: arg1: string (Ex: scriptTTL_TE; "day" or "hour")
 599     #Output: var: scriptTTL (integer seconds)
 600     #Depends: timeUntilNextHour, timeUntilNextDay
 601     local argTimeElement
 602
 603     argTimeElement="$1";
 604     if [[ "$argTimeElement" = "day" ]]; then
 605             # Set script lifespan to end at start of next day
 606         if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code
 607             if [[ "$scriptTTL" -eq 0 ]]; then
 608             ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout.
 609             else
 610             yell "ERROR: timeUntilNextDay exit code $?"; exit 1;
 611             fi;
 612         fi;
 613     elif [[ "$argTimeElement" = "hour" ]]; then
 614         # Set script lifespan to end at start of next hour
 615         if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code
 616             if [[ "$scriptTTL" -eq 0 ]]; then
 617                 ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout.
 618             else
 619                 yell "ERROR: timeUntilNextHour exit code $?"; exit 1;
 620             fi;
 621         fi;
 622     else
 623         yell "ERROR:Invalid argument for setScriptTTL function:$argTimeElement"; exit 1;
 624     fi;
 625 } # Set scriptTTL in seconds until next (day|hour).
 626 magicInitCheckTar() {
 627     # Desc: Initializes or checks output tar
 628     # input: vars: dirOut, bufferTTL, cmd_encrypt_suffix, cmd_compress_suffix
 629     # input: vars: scriptHostname
 630     # output: vars: pathout_tar
 631     # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion()
 632
 633     # Form pathout_tar
 634     pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \
 635         vbm "STATUS:Set pathout_tar to:$pathout_tar";
 636     # Validate pathout_tar as tar.
 637     checkMakeTar "$pathout_tar";
 638     ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2)
 639     vbm "exit status before magicWriteVersion:$?"
 640     if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi
 641 } # Initialize tar, set pathout_tar
 642 magicWriteVersion() {
 643     # Desc: Appends time-stamped VERSION to pathout_tar
 644     # Usage: magicWriteVersion
 645     # Input: vars: pathout_tar, dir_tmp
 646     # Input: vars: scriptVersion, scriptURL, ageVersion, ageURL, scriptHostname
 647     # Input: array: recPubKeysValid
 648     # Output: appends tar (pathout_tar)
 649     # Depends: bash 5.0.3, dateTimeShort(), appendArgTar()
 650     local fileoutVersion contentVersion pubKeyIndex pubKeyIndex
 651
 652     # Set VERSION file name
 653     fileoutVersion="$(dateTimeShort)..VERSION";
 654
 655     # Gather VERSION data in contentVersion
 656     contentVersion="scriptVersion=$scriptVersion";
 657     #contentVersion="$contentVersion""\\n";
 658     contentVersion="$contentVersion""\\n""scriptName=$scriptName";
 659     contentVersion="$contentVersion""\\n""scriptURL=$scriptURL";
 660     contentVersion="$contentVersion""\\n""ageVersion=$ageVersion";
 661     contentVersion="$contentVersion""\\n""ageURL=$ageURL";
 662     contentVersion="$contentVersion""\\n""date=$(date --iso-8601=seconds)";
 663     contentVersion="$contentVersion""\\n""hostname=$scriptHostname";
 664     ## Add list of recipient pubkeys
 665     for pubkey in "${recPubKeysValid[@]}"; do
 666         ((pubKeyIndex++))
 667         contentVersion="$contentVersion""\\n""PUBKEY_$pubKeyIndex=$pubkey";
 668     done
 669     ## Process newline escapes
 670     contentVersion="$(echo -e "$contentVersion")"
 671
 672     # Write contentVersion as file fileoutVersion and write-append to pathout_tar
 673     appendArgTar "$contentVersion" "$fileoutVersion" "$pathout_tar" "$dir_tmp";
 674 } # write version data to pathout_tar via appendArgTar()
 675 magicParseProcessStrings() {
 676     # Desc: Processes user-supplied process strings into process commands for appendFileTar().
 677     # Usage: magicParseProcessStrings
 678     # In : vars: optionProcString optionNoStoreRaw optionStoreRaw argRawFileExt
 679     #      arry: argProcStrings, argProcFileExts
 680     # Out: arry: procStrings, procFileExts
 681     # Depends Bash 5.0.3, yell(), vbm()
 682     local rawFileExt
 683
 684     vbm "STATUS:Starting magicParseProcessStrings() function.";
 685     # Validate input
 686     ## Validate argRawFileExt
 687     if [[ "$argRawFileExt" =~ ^[.][[:alnum:]]*$ ]]; then
 688         rawFileExt="$argRawFileExt";
 689     fi;
 690
 691     # Add default stdin output file entries for procStrings, procFileExts
 692     ## Check if user specified that no raw stdin be saved.
 693     if [[ ! "$optionNoStoreRaw" = "true" ]]; then
 694         ### T: --no-store-raw not set. Store raw. Append procStrings with cat.
 695         #### Append procStrings array
 696         procStrings+=("cat ");
 697         #### Check if --store-raw set.
 698         if [[ "$optionStoreRaw" = "true" ]]; then
 699             ##### T: --store-raw set. Append procFileExts with user-specified file ext
 700             procFileExts+=("$rawFileExt");
 701         else
 702             ##### F: --store-raw not set. Append procFileExts with default ".stdin" file ext
 703             ###### Append procFileExts array
 704             procFileExts+=(".stdin");
 705         fi;
 706     else
 707         ### F: --no-store-raw set. Do not store raw.
 708         #### Do not append procStrings or procFileExts arrays.
 709         :
 710     fi;
 711
 712     # Do nothing more if optionProcString not set to true.
 713     if [[ ! "$optionProcString" = "true" ]]; then
 714         vbm "STATUS:optionProcString not set to 'true'. Returning early.";
 715         return; fi;
 716     # Validate input array indices
 717     ## Make sure that argProcStrings and argProcFileExts have same index counts
 718     if ! [[ "${#argProcStrings[@]}" -eq "${#argProcFileExts[@]}" ]]; then
 719         yell "ERROR:Mismatch in number of elements in arrays argProcStrings and argProcFileExts:${#argProcStrings[@]} DNE ${#argProcFileExts[@]}";
 720         yell "argProcStrings:${argProcStrings[*]}"; yell "argProcFileExts:${argProcFileExts[*]}"; exit 1; fi;
 721     ## Make sure that no array elements are blank
 722     for element in "${argProcStrings[@]}"; do
 723         if [[ -z "$element" ]]; then yell "ERROR:Empty process string specified. Exiting."; exit 1; fi; done
 724     for element in "${argProcFileExts[@]}"; do
 725         if [[ -z "$element" ]]; then yell "ERROR:Empty output file extension specified. Exiting."; exit 1; fi; done
 726     ## Make sure that no process string starts with '-' (ex: if only one arg supplied after '-p' option)
 727     for element in "${argProcStrings[@]}"; do
 728         if [[ ! "$element" =~ ^[-][[:print:]]*$ ]] && [[ "$element" =~ ^[[:print:]]*$ ]]; then
 729             yell "ERROR:Illegal character '-' at start of process string element. Option syntax error?";
 730             exit 1; fi; done;
 731     vbm "STATUS:Quick check shows argProcStrings and argProcFileExts appear to have valid contents.";
 732     procStrings=("${argProcStrings[@]}"); # Export process command strings
 733     procFileExts=("${argProcFileExts[@]}"); # Export process command strings
 734     vbm "STATUS:Finished magicParseProcessStrings() function.";
 735 } # Validate and save process strings and file extensions to arrays procStrings, procFileExts
 736 magicParseLabel() {
 737     # Desc: Parses -l option to set label
 738     # In : optionLabel, argLabel
 739     # Out: vars: label
 740     # Depends: Bash 5.0.3, vbm(), yell()
 741
 742     vbm "STATUS:Started magicParseLabel() function.";
 743     # Do nothing if optionLabel not set to true.
 744     if [[ ! "$optionLabel" = "true" ]]; then
 745         vbm "STATUS:optionlabel not set to 'true'. Returning early.";
 746         return;
 747     fi;
 748     # Set label if optionLabel is true
 749     if [[ "$optionLabel" = "true" ]]; then
 750         label="_""$argLabel";
 751         vbm "STATUS:Set label:$label";
 752     fi;
 753     vbm "STATUS:Finished magicParseLabel() function.";
 754 }
 755 magicProcessWriteBuffer() {
 756     # Desc: process and write buffer
 757     # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS
 758     #    : arry: buffer
 759     # Out: file:(pathout_tar)
 760     # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(),
 761     ### Note: These arrays should all have the same number of elements:
 762     ###       pathouts, fileouts, procFileExts, procStrings
 763
 764     local fn timeBufferStartLong timeBufferStart fileoutBasename
 765     local -a fileouts pathouts
 766     local writeCmd1 writeCmd2 writeCmd3 writeCmd4
 767
 768     vbm "DEBUG:STATUS:$fn:Started magicProcessWriteBuffer().";
 769     # Debug:Get function name
 770     fn="${FUNCNAME[0]}";
 771
 772     # Determine file paths (time is start of buffer period)
 773     ## Calculate start time
 774     timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \
 775         vbm "timeBufferStartLong:$timeBufferStartLong";
 776     timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \
 777         vbm "timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators)
 778     ## Set common basename
 779     fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \
 780         vbm "STATUS:Set fileoutBasename to:$fileoutBasename";
 781     ## Determine output file name array
 782     ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts
 783     for fileExt in "${procFileExts[@]}"; do
 784         fileouts+=("$fileoutBasename""$fileExt""$cmd_compress_suffix""$cmd_encrypt_suffix") && \
 785             vbm "STATUS:Added $fileExt to fileouts:${fileouts[*]}";
 786     done;
 787     for fileName in "${fileouts[@]}"; do
 788         pathouts+=("$dir_tmp"/"$fileName") && \
 789             vbm "STATUS:Added $fileName to pathouts:${pathouts[*]}";
 790     done;
 791     ## Update pathout_tar
 792     magicInitCheckTar;
 793
 794     # Process and write buffers to dir_tmp
 795     ## Prepare command strings
 796     writeCmd1="printf \"%s\\\\n\" \"\${buffer[@]}\""; # printf "%s\\n" "${buffer[@]}"
 797     #writeCmd2="" # NOTE: Specified by parsing array procStrings
 798     writeCmd3="$cmd_compress";
 799     writeCmd4="$cmd_encrypt";
 800
 801     ## Process buffer and write to dir_tmp
 802     for index in "${!pathouts[@]}"; do
 803         writeCmd2="${procStrings[$index]}"
 804         eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" >> "${pathouts[$index]}";
 805     done;
 806
 807     # Append dir_tmp files to pathout_tar
 808     wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
 809     for index in "${!pathouts[@]}"; do
 810         appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp";
 811     done;
 812
 813     # Remove secured chunks from dir_tmp
 814     for path in "${pathouts[@]}"; do
 815         rm "$path";
 816     done;
 817
 818     vbm "DEBUG:STATUS:$fn:Finished magicProcessWriteBuffer().";
 819 } # Process and Write buffer
 820 main() {
 821     # Process arguments
 822     processArguments "$@";
 823     ## Determine working directory
 824     magicInitWorkingDir; # Sets dir_tmp from argTempDirPriority
 825     ## Set output encryption and compression option strings
 826     ### React to "-e" and "-r" ("encryption recipients") options
 827     magicParseRecipientArgs; # Updates recPubKeysValid, cmd_encrypt[_suffix] from argRecPubKeys
 828     ### React to "-R" ("recipient directory") option
 829     magicParseRecipientDir; # Updates recPubKeysValid
 830     ### React to "-c" ("compression") option
 831     magicParseCompressionArg; # Updates cmd_compress[_suffix]
 832     ## React to "-b" and "-B" (custom buffer and script TTL) options
 833     magicParseCustomTTL; # Sets custom scriptTTL_TE and/or bufferTTL if specified
 834     ## React to "-p" (user-supplied process command and file extension strings) options
 835     magicParseProcessStrings; # Sets arrays: procStrings, procFileExts
 836     ## React to "-l" (output file label) option
 837     magicParseLabel; # sets label (ex: "_location")
 838     ## React to "-w" (how to name raw stdin file) option
 839     magicParseStoreRaw; # sets raw_suffix
 840
 841     # Perform secondary setup operations
 842     ## Set script lifespan (scriptTTL from scriptTTL_TE)
 843     magicSetScriptTTL "$scriptTTL_TE";
 844     ## File name substring (ISO-8601 duration from bufferTTL)
 845     bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG:bufferTTL_STR:$bufferTTL_STR";
 846     ## Init temp working dir
 847     try mkdir "$dir_tmp" && vbm "DEBUG:Working dir created at dir_tmp:$dir_tmp";
 848     ## Initialize output tar (set pathout_tar)
 849     magicInitCheckTar;
 850
 851     # Check vital apps, files, dirs
 852     if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then
 853         yell "ERROR:Critical components missing.";
 854         displayMissing; yell "Exiting."; exit 1; fi
 855
 856     # MAIN LOOP: Run until script TTL seconds pass
 857     bufferRound=0;
 858     while [[ $SECONDS -lt "scriptTTL" ]]; do
 859         bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death
 860         lineCount=0; # Debug counter
 861         # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives
 862         while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do
 863             # Append line to buffer array
 864             buffer+=("$line");
 865             echo "DEBUG:Processing line:$lineCount";
 866             echo "DEBUG:Current line   :$line";
 867             echo "DEBUG:buf elem count :${#buffer[@]}";
 868             ((lineCount++));
 869         done;
 870         # Create dir_tmp if missing
 871         if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:dir_tmp existence failure:$dir_tmp"; try mkdir "$dir_tmp" && vbm "DEBUG:Working dir recreated dir_tmp:$dir_tmp"; fi
 872         # Update encryption recipient array
 873         magicParseRecipientDir; # Update recPubKeysValid with argRecDir
 874         # Export buffer to asynchronous processing.
 875         magicProcessWriteBuffer &
 876         unset buffer; # Clear buffer array for next bufferRound
 877         # Increment buffer round
 878         ((bufferRound++));
 879     done;
 880
 881     # Cleanup
 882     ## Remove dir_tmp
 883     try rm -r "$dir_tmp" && vbm "Removed dir_tmp:$dir_tmp";
 884
 885     vbm "STATUS:Main function finished.";
 886 } # Main function
 887
 888 #===END Declare local script functions===
 889 #==END Define script parameters==
 890
 891 #==BEGIN Perform work and exit==
 892 main "$@" # Run main function.
 893 exit 0;
 894 #==END Perform work and exit==
 895
 896 # Author: Steven Baltakatei Sandoval;
 897 # License: GPLv3+