[EVA-2020-02.git] / exec / bklog

#!/bin/bash
# Desc: Compresses, encrypts, and writes stdin every 5 seconds

#==BEGIN Define script parameters==
#===BEGIN Initialize variables===

# Logging Behavior parameters
bufferTTL="300"; # Time-to-live (seconds) for each buffer round
scriptTTL_TE="day"; # Time element at the end of which script terminates
dirTmpDefault="/dev/shm"; # Default parent of working directory

# Script Metadata
scriptName="bklog";             # Define basename of script file.
scriptVersion="0.1.2";          # Define version of script.
scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN
scriptHostname=$(hostname);     # Save hostname of system running this script.
PATH="$HOME/.local/bin:$PATH";  # Add "$(systemd-path user-binaries)" path in case user apps saved there
ageVersion="1.0.0-beta2";       # Define version of age (encryption program)
ageURL="https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2"; # Define website hosting age.

# Arrays
declare -a buffer          # array for storing while read buffer
declare -a argRecPubKeys   # array for processArguments function
declare -a recPubKeysValid # array for storing both '-r' and '-R' recipient pubkeys
declare -a recPubKeysValidStatic # for storing '-r' recipient pubkeys
declare -a argProcStrings argProcFileExts # for storing buffer processing strings (ex: "gpsbabel -i nmea -f - -o gpx -F - ")
declare -Ag appRollCall    # Associative array for storing app status
declare -Ag fileRollCall   # Associative array for storing file status
declare -Ag dirRollCall    # Associative array for storing dir status
declare -a procStrings procFileExts # Arrays for storing processing commands and resulting output file extensions

# Variables
optionVerbose=""; optionEncrypt=""; dirOut=""; optionEncrypt=""; dir_tmp="";
cmd_compress="";cmd_compress_suffix=""; cmd_encrypt=""; cmd_encrypt_suffix="";

#===END Initialize variables===

#===BEGIN Declare local script functions===
yell() { echo "$0: $*" >&2; }      #o Yell, Die, Try Three-Fingered Claw technique
die() { yell "$*"; exit 111; }     #o Ref/Attrib: https://stackoverflow.com/a/25515370
try() { "$@" || die "cannot $*"; } #o
processArguments() {
    while [ ! $# -eq 0 ]; do   # While number of arguments ($#) is not (!) equal to (-eq) zero (0).
	case "$1" in
	    -v | --verbose) optionVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode.
	    -h | --help) showUsage; exit 1;; # Display usage.
	    --version) showVersion; exit 1;; # Show version
	    -o | --output) if [ -d "$2" ]; then dirOut="$2"; vbm "DEBUG:dirOut:$dirOut"; shift; fi ;; # Define output directory.
	    -e | --encrypt) optionEncrypt="true"; vbm "DEBUG:Encrypted output mode enabled.";; # Enable encryption
	    -r | --recipient) optionRecipients="true"; argRecPubKeys+=("$2"); vbm "STATUS:pubkey added:""$2"; shift;; # Add recipients
	    -c | --compress) optionCompress="true"; vbm "DEBUG:Compressed output mode enabled.";; # Enable compression
	    -z | --time-zone) try setTimeZoneEV "$2"; shift;; # Set timestamp timezone
	    -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone
	    -R | --recipient-dir) optionRecipients="true"; optionRecDir="true" && argRecDir="$2"; shift;; # Add recipient watch dir
	    -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds)
	    -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day")
	    -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;;
	    -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG:Custom label received:$argLabel"; shift;;
	    -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG:Raw stdin file extension received:$argRawFileExt"; shift;;
	    -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG:Option selected to not store raw stdin data."; shift;;
	    *) yell "ERROR: Unrecognized argument: $1"; yell "STATUS:All arguments:$*"; exit 1;; # Handle unrecognized options.
	esac
	shift
    done
} # Argument Processing
vbm() {
    # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true".
    # Usage: vbm "DEBUG:verbose message here"
    # Version 0.1.2
    # Input: arg1: string
    #        vars: optionVerbose
    # Output: stderr
    # Depends: bash 5.0.3, echo 8.30, date 8.30

    if [ "$optionVerbose" = "true" ]; then
	functionTime=$(date --iso-8601=ns); # Save current time in nano seconds.
	echo "[$functionTime] ""$*" 1>&2; # Display argument text.
    fi

    # End function
    return 0; # Function finished.
} # Displays message if optionVerbose true
checkapp() {
    # Desc: If arg is a command, save result in assoc array 'appRollCall'
    # Usage: checkapp arg1 arg2 arg3 ...
    # Version: 0.1.1
    # Input: global assoc. array 'appRollCall'
    # Output: adds/updates key(value) to global assoc array 'appRollCall'
    # Depends: bash 5.0.3
    local returnState    

    #===Process Args===
    for arg in "$@"; do
	if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
	    appRollCall[$arg]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
	else
	    appRollCall[$arg]="false"; returnState="false";
	fi;
    done;

    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi;
} # Check that app exists
checkfile() {
    # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
    # Usage: checkfile arg1 arg2 arg3 ...
    # Version: 0.1.1
    # Input: global assoc. array 'fileRollCall'
    # Output: adds/updates key(value) to global assoc array 'fileRollCall';
    # Output: returns 0 if app found, 1 otherwise
    # Depends: bash 5.0.3
    local returnState

    #===Process Args===
    for arg in "$@"; do
	if [ -f "$arg" ]; then
	    fileRollCall["$arg"]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
	else
	    fileRollCall["$arg"]="false"; returnState="false";
	fi;
    done;
    
    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi;
} # Check that file exists
checkdir() {
    # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
    # Usage: checkdir arg1 arg2 arg3 ...
    # Version 0.1.1
    # Input: global assoc. array 'dirRollCall'
    # Output: adds/updates key(value) to global assoc array 'dirRollCall';
    # Output: returns 0 if app found, 1 otherwise
    # Depends: Bash 5.0.3
    local returnState

    #===Process Args===
    for arg in "$@"; do
	if [ -d "$arg" ]; then
	    dirRollCall["$arg"]="true";
	    if ! [ "$returnState" = "false" ]; then returnState="true"; fi
	else
	    dirRollCall["$arg"]="false"; returnState="false";
	fi
    done
    
    #===Determine function return code===
    if [ "$returnState" = "true" ]; then
	return 0;
    else
	return 1;
    fi
} # Check that dir exists
displayMissing() {
    # Desc: Displays missing apps, files, and dirs
    # Usage: displayMissing
    # Version 0.1.1
    # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
    # Output: stderr: messages indicating missing apps, file, or dirs
    # Depends: bash 5, checkAppFileDir()
    local missingApps value appMissing missingFiles fileMissing
    local missingDirs dirMissing
    
    #==BEGIN Display errors==
    #===BEGIN Display Missing Apps===
    missingApps="Missing apps  :";
    #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
    for key in "${!appRollCall[@]}"; do
	value="${appRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing apps: $key => $value";
	    missingApps="$missingApps""$key ";
	    appMissing="true";
	fi;
    done;
    if [ "$appMissing" = "true" ]; then  # Only indicate if an app is missing.
	echo "$missingApps" 1>&2;
    fi;
    unset value;
    #===END Display Missing Apps===

    #===BEGIN Display Missing Files===
    missingFiles="Missing files:";
    #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
    for key in "${!fileRollCall[@]}"; do
	value="${fileRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing files: $key => $value";
	    missingFiles="$missingFiles""$key ";
	    fileMissing="true";
	fi;
    done;
    if [ "$fileMissing" = "true" ]; then  # Only indicate if an app is missing.
	echo "$missingFiles" 1>&2;
    fi;
    unset value;
    #===END Display Missing Files===

    #===BEGIN Display Missing Directories===
    missingDirs="Missing dirs:";
    #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
    for key in "${!dirRollCall[@]}"; do
	value="${dirRollCall[$key]}";
	if [ "$value" = "false" ]; then
	    #echo "DEBUG:Missing dirs: $key => $value";
	    missingDirs="$missingDirs""$key ";
	    dirMissing="true";
	fi;
    done;
    if [ "$dirMissing" = "true" ]; then  # Only indicate if an dir is missing.
	echo "$missingDirs" 1>&2;
    fi;
    unset value;
    #===END Display Missing Directories===

    #==END Display errors==
} # Display missing apps, files, dirs
showUsage() {
    cat <<'EOF'
    USAGE:
        cmd | bklog [ options ]
    echoerr
    OPTIONS:
        -h, --help
                Display help information.
        --version
                Display script version.
        -v, --verbose
                Display debugging info.
        -e, --encrypt
                Encrypt output.
        -r, --recipient [ string pubkey ]
                Specify recipient. May be age or ssh pubkey.
                May be specified multiple times for multiple pubkeys.
                See https://github.com/FiloSottile/age
        -o, --output [ path dir ]
                Specify output directory to save logs. This option is required
                to save log data.
        -p, --process-string [ filter command ] [ output file extension] 
                Specify how to create and name a processed version of the stdin.
                For example, if stdin is 'nmea' location data:
                
                -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx"

                This option would cause the stdin to 'bklog' to be piped into
                the 'gpsbabel' command, interpreted as 'nmea' data, converted
                into 'gpx' format, and then appended to the output tar file
                as a file with a '.gpx' extension.
                This option may be specified multiple times in order to output
                results of multiple different processing methods.
        -l, --label [ string ]
                Specify a label to be included in all output file names.
                Ex: 'location' if stdin is location data.
        -w, --store-raw [ file extension ]
                Specify file extension of file within output tar that contains
                raw stdin data. The default behavior is to always save raw stdin
                data in a '.stdin' file. Example usage when 'bklog' receives
                'nmea' data from 'gpspipe -r':

                -w ".nmea"

                Stdin data is saved in a '.nmea' file within the output tar.
        -W, --no-store-raw
                Do not store raw stdin in output tar.
        -c, --compress
                Compress output with gzip (before encryption if enabled).
        -z, --time-zone
                Specify time zone. (ex: "America/New_York")
        -t, --temp-dir [path dir]
                Specify parent directory for temporary working directory.
                Default: "/dev/shm"
        -R, --recipient-dir [path dir]
                Specify directory containing files whose first lines are
                to be interpreted as pubkey strings (see '-r' option).
        -b, --buffer-ttl [integer]
                Specify custom buffer period in seconds (default: 300 seconds)
        -B, --script-ttl [time element string]
                Specify custom script time-to-live in seconds (default: "day")
                Valid values: "day", "hour"
    echoerr
    EXAMPLE: (bash script lines)
    $ gpspipe -r | /bin/bash bklog -v -e -c -z "UTC" -t "/dev/shm" \
    -r age1mrmfnwhtlprn4jquex0ukmwcm7y2nxlphuzgsgv8ew2k9mewy3rs8u7su5 \
    -r age1ala848kqrvxc88rzaauc6vc5v0fqrvef9dxyk79m0vjea3hagclswu0lgq \
    -R ~/.config/bklog/recipients -w ".nmea" -b 300 -B "day" \
    -o ~/Sync/Logs -l "location" \
    -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" \
    -p "gpsbabel -i nmea -f - -o kml -F - " ".gpx"
EOF
} # Display information on how to use this script.
showVersion() {
    yell "$scriptVersion"
} # Display script version.
setTimeZoneEV(){
    # Desc: Set time zone environment variable TZ
    # Usage: setTimeZoneEV arg1
    # Version 0.1.2
    # Input: arg1: 'date'-compatible timezone string (ex: "America/New_York")
    #        TZDIR env var (optional; default: "/usr/share/zoneinfo")
    # Output: exports TZ
    #         exit code 0 on success
    #         exit code 1 on incorrect number of arguments
    #         exit code 2 if unable to validate arg1
    # Depends: yell(), printenv 8.30, bash 5.0.3
    # Tested on: Debian 10
    local tzDir returnState argTimeZone

    argTimeZone="$1"
    if ! [[ $# -eq 1 ]]; then
	yell "ERROR:Invalid argument count.";
	return 1;
    fi

    # Read TZDIR env var if available
    if printenv TZDIR 1>/dev/null 2>&1; then
	tzDir="$(printenv TZDIR)";
    else
	tzDir="/usr/share/zoneinfo";
    fi
    
    # Validate TZ string
    if ! [[ -f "$tzDir"/"$argTimeZone" ]]; then
	yell "ERROR:Invalid time zone argument.";
	return 2;
    else
    # Export ARG1 as TZ environment variable
	TZ="$argTimeZone" && export TZ && returnState="true";
    fi

    # Determine function return code
    if [ "$returnState" = "true" ]; then
	return 0;
    fi
} # Exports TZ environment variable
dateShort(){
    # Desc: Date without separators (YYYYmmdd)
    # Usage: dateShort ([str date])
    # Version: 1.1.2
    # Input: arg1: 'date'-parsable timestamp string (optional)
    # Output: stdout: date (ISO-8601, no separators)
    # Depends: bash 5.0.3, date 8.30, yell()
    local argTime timeCurrent timeInput dateCurrentShort

    argTime="$1";
    # Get Current Time
    timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second.
    # Decide to parse current or supplied date
    ## Check if time argument empty
    if [[ -z "$argTime" ]]; then
	## T: Time argument empty, use current time
	timeInput="$timeCurrent";
    else
	## F: Time argument exists, validate time
	if date --date="$argTime" 1>/dev/null 2>&1; then
	    ### T: Time argument is valid; use it
	    timeInput="$argTime";
	else
	    ### F: Time argument not valid; exit
	    yell "ERROR:Invalid time argument supplied. Exiting."; exit 1;
	fi;
    fi;
    # Construct and deliver separator-les date string    
    dateCurrentShort="$(date -d "$timeInput" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day.
    echo "$dateCurrentShort";
} # Get YYYYmmdd
appendFileTar(){
    # Desc: Appends [processed] file to tar
    # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd])
    # Version: 2.0.1
    # Input: arg1: path of file to be (processed and) written
    #        arg2: name to use for file inserted into tar
    #        arg3: tar archive path (must exist first)
    #        arg4: temporary working dir
    #        arg5: (optional) command string to process file (ex: "gpsbabel -i nmea -f - -o kml -F - ")
    # Output: file written to disk
    # Example: decrypt multiple large files in parallel
    #          appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" &
    #          appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" &
    #          appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" &
    # Depends: bash 5.0.3, tar 1.30, cat 8.30, yell()
    local fn fileName tarPath tmpDir

    # Save function name
    fn="${FUNCNAME[0]}";
    #yell "DEBUG:STATUS:$fn:Started appendFileTar()."
    
    # Set file name
    if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi
    # Check tar path is a file
    if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file:$3"; exit 1; fi
    # Check temp dir arg
    if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi
    # Set command strings
    if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string

    # Input command string
    cmd0="cat \"\$1\"";
    
    # Write to temporary working dir
    eval "$cmd0 | $cmd1" > "$tmpDir"/"$fileName";

    # Append to tar
    try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName";
    #yell "DEBUG:STATUS:$fn:Finished appendFileTar()."
} # Append [processed] file to Tar archive
magicParseRecipientArgs() {
    # Desc: Parses recipient arguments specified by '-r' option
    # Input:  vars: optionEncrypt, optionRecipients
    #         arry: argRecPubKeys from processArguments()
    # Output: vars: cmd_encrypt, cmd_encrypt_suffix
    #         arry: recPubKeysValid, recPubKeysValidStatic
    # Depends: processArguments(), yell(), vbm(), checkapp(), checkAgePubkey(), validateInput()
    local recipients

    # Check if encryption option active.
    if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then 
	if checkapp age; then # Check that age is available.
	    for pubkey in "${argRecPubKeys[@]}"; do # Validate recipient pubkey strings by forming test message
		vbm "DEBUG:Testing pubkey string:$pubkey";
		if checkAgePubkey "$pubkey" && \
			( validateInput "$pubkey" "ssh_pubkey" || validateInput "$pubkey" "age_pubkey"); then
		    #### Form age recipient string
		    recipients="$recipients""-r '$pubkey' ";
		    vbm "STATUS:Added pubkey for forming age recipient string:""$pubkey";
		    vbm "DEBUG:recipients:""$recipients";
		    #### Add validated pubkey to recPubKeysValid array
		    recPubKeysValid+=("$pubkey") && vbm "DEBUG:recPubkeysValid:pubkey added:$pubkey";
		else
		    yell "ERROR:Exit code ""$?"". Invalid recipient pubkey string. Exiting."; exit 1;
		fi;
	    done
	    vbm "DEBUG:Finished processing argRecPubKeys array";
	    vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[*]}";
	    recPubKeysValidStatic=("${recPubKeysValid[@]}"); # Save static image of pubkeys validated by this function

	    ##  Form age command string
	    cmd_encrypt="age ""$recipients " && vbm "cmd_encrypt:$cmd_encrypt";
	    cmd_encrypt_suffix=".age" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix";
	else
	    yell "ERROR:Encryption enabled but \"age\" not found. Exiting."; exit 1;
	fi;
    else
	cmd_encrypt="tee /dev/null " && vbm "cmd_encrypt:$cmd_encrypt";
	cmd_encrypt_suffix="" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix";
	vbm "DEBUG:Encryption not enabled."
    fi;
    # Catch case if '-e' is set but '-r' or '-R' is not
    if [[ "$optionEncrypt" = "true" ]] && [[ ! "$optionRecipients" = "true" ]]; then
	yell "ERROR:\\'-e\\' set but no \\'-r\\' or \\'-R\\' set."; exit 1; fi;
    # Catch case if '-r' or '-R' set but '-e' is not
    if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then
	yell "ERROR:\\'-r\\' or \\'-R\\' set but \\'-e\\' is not set."; exit 1; fi; 
} # Populate recPubKeysValid with argRecPubKeys; form encryption cmd string and filename suffix
magicParseRecipientDir() {
    # Desc: Updates recPubKeysValid with pubkeys in dir specified by '-R' option ("recipient directory")
    # Inputs:  vars: optionEncrypt, optionRecDir, argRecDir, 
    #          arry: recPubKeysValid
    # Outputs: arry: recPubKeysValid
    # Depends: processArguments(), yell(), vbm(), validateInput(),  checkAgePubkey()
    local recipientDir recFileLine updateRecipients 
    declare -a candRecPubKeysValid

    # Check that '-e' and '-R' set
    if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then
	### Check that argRecDir is a directory.
	if [[ -d "$argRecDir" ]]; then
	    recipientDir="$argRecDir" && vbm "STATUS:Recipient watch directory detected:\"$recipientDir\"";
	    #### Initialize variable indicating outcome of pubkey review
	    unset updateRecipients
	    #### Add existing recipients
	    candRecPubKeysValid=("${recPubKeysValidStatic[@]}");
	    #### Parse files in recipientDir
	    for file in "$recipientDir"/*; do
		##### Read first line of each file
		recFileLine="$(head -n1 "$file")" && vbm "STATUS:Checking if pubkey:\"$recFileLine\"";
		##### check if first line is a valid pubkey
		if checkAgePubkey "$recFileLine" && \
			( validateInput "$recFileLine" "ssh_pubkey" || validateInput "$recFileLine" "age_pubkey"); then
		    ###### T: add candidate pubkey to candRecPubKeysValid
		    candRecPubKeysValid+=("$recFileLine") && vbm "STATUS:RecDir pubkey is valid pubkey:\"$recFileLine\"";
		else
		    ###### F: throw warning;
		    yell "ERROR:Invalid recipient file detected. Not modifying recipient list."
		    updateRecipients="false";
		fi;
	    done
	    #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected
	    if ! [[ "$updateRecipients" = "false" ]]; then
		recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\"";
	    fi;
	else
	    yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1;
	fi;
    fi;
    # Handle case if '-R' set but '-e' not set
    if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then
	yell "ERROR: \\'-R\\' is set but \\'-e\\' is not set."; fi;
} # Update recPubKeysValid with argRecDir
magicParseCompressionArg() {
    # Desc: Parses compression arguments specified by '-c' option
    # Input:  vars: optionCompress
    # Output: cmd_compress, cmd_compress_suffix
    # Depends: processArguments(), vbm(), checkapp(), gzip 1.9
    if [[ "$optionCompress" = "true" ]]; then # Check if compression option active
	if checkapp gzip; then # Check if gzip available
	    cmd_compress="gzip " && vbm "cmd_compress:$cmd_compress";
	    cmd_compress_suffix=".gz" && vbm "cmd_compress_suffix:$cmd_compress_suffix";
	else
	    yell "ERROR:Compression enabled but \"gzip\" not found. Exiting."; exit 1;
	fi
    else
	cmd_compress="tee /dev/null " && vbm "cmd_compress:$cmd_compress";
	cmd_compress_suffix="" && vbm "cmd_compress_suffix:$cmd_compress_suffix";
	vbm "DEBUG:Compression not enabled.";
    fi    
} # Form compression cmd string and filename suffix
magicInitWorkingDir() {
    # Desc: Determine temporary working directory from defaults or user input
    # Usage: magicInitWorkingDir
    # Input:  vars: optionTmpDir, argTempDirPriority, dirTmpDefault
    # Input:  vars: scriptTimeStart
    # Output: vars: dir_tmp
    # Depends: bash 5.0.3, processArguments(), vbm(), yell()
    # Parse '-t' option (user-specified temporary working dir)
    ## Set dir_tmp_parent to user-specified value if specified
    local dir_tmp_parent
    
    if [[ "$optionTmpDir" = "true" ]]; then
	if [[ -d "$argTempDirPriority" ]]; then
	    dir_tmp_parent="$argTempDirPriority"; 
	else
	    yell "WARNING:Specified temporary working directory not valid:$argTempDirPriority";
	    exit 1; # Exit since user requires a specific temp dir and it is not available.
	fi;
    else
    ## Set dir_tmp_parent to default or fallback otherwise
	if [[ -d "$dirTmpDefault" ]]; then
	    dir_tmp_parent="$dirTmpDefault";
	elif [[ -d /tmp ]]; then
	    yell "WARNING:$dirTmpDefault not available. Falling back to /tmp .";
	    dir_tmp_parent="/tmp";
	else
	    yell "ERROR:No valid working directory available. Exiting.";
	    exit 1;
	fi;
    fi;
    ## Set dir_tmp using dir_tmp_parent and nonce (scriptTimeStart)
    dir_tmp="$dir_tmp_parent"/"$scriptTimeStart""..bkgpslog" && vbm "DEBUG:Set dir_tmp to:$dir_tmp"; # Note: removed at end of main().
} # Sets working dir
magicParseCustomTTL() {
    # Desc: Set user-specified TTLs for buffer and script
    # Usage: magicParseCustomTTL
    # Input: vars: argCustomBufferTTL (integer), argCustomScriptTTL_TE (string)
    # Input: vars: optionCustomBufferTTL, optionCustomScriptTTL_TE
    # Input: vars: bufferTTL (integer), scriptTTL_TE (string)
    # Output: bufferTTL (integer), scriptTTL_TE (string)
    # Depends: Bash 5.0.3, yell(), vbm(), validateInput(), showUsage()

    # React to '-b, --buffer-ttl' option
    if [[ "$optionCustomBufferTTL" = "true" ]]; then
	## T: Check if argCustomBufferTTL is an integer
	if validateInput "$argCustomBufferTTL" "integer"; then
	    ### T: argCustomBufferTTL is an integer
	    bufferTTL="$argCustomBufferTTL" && vbm "Custom bufferTTL from -b:$bufferTTL";
	else
	    ### F: argcustomBufferTTL is not an integer
	    yell "ERROR:Invalid integer argument for custom buffer time-to-live."; showUsage; exit 1;
	fi;
	## F: do not change bufferTTL
    fi;
    
    # React to '-B, --script-ttl' option
    if [[ "$optionCustomScriptTTL_TE" = "true" ]]; then
	## T: Check if argCustomScriptTTL is a time element (ex: "day", "hour")
	if validateInput "$argCustomScriptTTL_TE" "time_element"; then
	    ### T: argCustomScriptTTL is a time element
	    scriptTTL_TE="$argCustomScriptTTL_TE" && vbm "Custom scriptTTL_TE from -B:$scriptTTL_TE";
	else
	    ### F: argcustomScriptTTL is not a time element
	    yell "ERROR:Invalid time element argument for custom script time-to-live."; showUsage; exit 1;
	fi;
	## F: do not change scriptTTL_TE
    fi;    
} # Sets custom script or buffer TTL if specified
magicSetScriptTTL() {
    #Desc: Sets script_TTL seconds from provided time_element string argument
    #Usage: magicSetScriptTTL [str time_element]
    #Input: arg1: string (Ex: scriptTTL_TE; "day" or "hour")
    #Output: var: scriptTTL (integer seconds)
    #Depends: timeUntilNextHour, timeUntilNextDay
    local argTimeElement
    
    argTimeElement="$1";
    if [[ "$argTimeElement" = "day" ]]; then
	    # Set script lifespan to end at start of next day
	if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code
	    if [[ "$scriptTTL" -eq 0 ]]; then
	    ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout.
	    else
	    yell "ERROR: timeUntilNextDay exit code $?"; exit 1;
	    fi;
	fi;
    elif [[ "$argTimeElement" = "hour" ]]; then
	# Set script lifespan to end at start of next hour
	if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code
	    if [[ "$scriptTTL" -eq 0 ]]; then
		((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout.
	    else
		yell "ERROR: timeUntilNextHour exit code $?"; exit 1;
	    fi;
	fi;
    else
	yell "ERROR:Invalid argument for setScriptTTL function:$argTimeElement"; exit 1;
    fi;
} # Set scriptTTL in seconds until next (day|hour).
magicInitCheckTar() {
    # Desc: Initializes or checks output tar
    # input: vars: dirOut, bufferTTL, cmd_encrypt_suffix, cmd_compress_suffix
    # input: vars: scriptHostname
    # output: vars: pathout_tar
    # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion()

    # Form pathout_tar
    pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \
	vbm "STATUS:Set pathout_tar to:$pathout_tar";
    # Validate pathout_tar as tar.
    checkMakeTar "$pathout_tar";
    ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2)
    vbm "exit status before magicWriteVersion:$?"
    if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi    
} # Initialize tar, set pathout_tar
magicWriteVersion() {
    # Desc: Appends time-stamped VERSION to pathout_tar
    # Usage: magicWriteVersion
    # Input: vars: pathout_tar, dir_tmp
    # Input: vars: scriptVersion, scriptURL, ageVersion, ageURL, scriptHostname
    # Input: array: recPubKeysValid
    # Output: appends tar (pathout_tar)
    # Depends: bash 5.0.3, dateTimeShort(), appendArgTar()
    local fileoutVersion contentVersion pubKeyIndex pubKeyIndex
    
    # Set VERSION file name
    fileoutVersion="$(dateTimeShort)..VERSION";

    # Gather VERSION data in contentVersion
    contentVersion="scriptVersion=$scriptVersion";
    #contentVersion="$contentVersion""\\n";
    contentVersion="$contentVersion""\\n""scriptName=$scriptName";
    contentVersion="$contentVersion""\\n""scriptURL=$scriptURL";
    contentVersion="$contentVersion""\\n""ageVersion=$ageVersion";
    contentVersion="$contentVersion""\\n""ageURL=$ageURL";
    contentVersion="$contentVersion""\\n""date=$(date --iso-8601=seconds)";
    contentVersion="$contentVersion""\\n""hostname=$scriptHostname";
    ## Add list of recipient pubkeys
    for pubkey in "${recPubKeysValid[@]}"; do
	((pubKeyIndex++))
	contentVersion="$contentVersion""\\n""PUBKEY_$pubKeyIndex=$pubkey";
    done
    ## Process newline escapes
    contentVersion="$(echo -e "$contentVersion")"

    # Write contentVersion as file fileoutVersion and write-append to pathout_tar
    appendArgTar "$contentVersion" "$fileoutVersion" "$pathout_tar" "$dir_tmp";
} # write version data to pathout_tar via appendArgTar()
magicParseProcessStrings() {
    # Desc: Processes user-supplied process strings into process commands for appendFileTar().
    # Usage: magicParseProcessStrings
    # In : vars: optionProcString optionNoStoreRaw optionStoreRaw argRawFileExt
    #      arry: argProcStrings, argProcFileExts
    # Out: arry: procStrings, procFileExts
    # Depends Bash 5.0.3, yell(), vbm()
    local rawFileExt

    vbm "STATUS:Starting magicParseProcessStrings() function.";
    # Validate input
    ## Validate argRawFileExt
    if [[ "$argRawFileExt" =~ ^[.][[:alnum:]]*$ ]]; then
	rawFileExt="$argRawFileExt";
    fi;
    
    # Add default stdin output file entries for procStrings, procFileExts
    ## Check if user specified that no raw stdin be saved.
    if [[ ! "$optionNoStoreRaw" = "true" ]]; then
	### T: --no-store-raw not set. Store raw. Append procStrings with cat.
	#### Append procStrings array
	procStrings+=("cat ");
	#### Check if --store-raw set.
	if [[ "$optionStoreRaw" = "true" ]]; then
	    ##### T: --store-raw set. Append procFileExts with user-specified file ext
	    procFileExts+=("$rawFileExt");
	else
	    ##### F: --store-raw not set. Append procFileExts with default ".stdin" file ext
	    ###### Append procFileExts array
	    procFileExts+=(".stdin");
	fi;
    else
	### F: --no-store-raw set. Do not store raw.
	#### Do not append procStrings or procFileExts arrays.
	:
    fi;

    # Do nothing more if optionProcString not set to true.
    if [[ ! "$optionProcString" = "true" ]]; then
	vbm "STATUS:optionProcString not set to 'true'. Returning early.";
	return; fi;
    # Validate input array indices
    ## Make sure that argProcStrings and argProcFileExts have same index counts
    if ! [[ "${#argProcStrings[@]}" -eq "${#argProcFileExts[@]}" ]]; then
	yell "ERROR:Mismatch in number of elements in arrays argProcStrings and argProcFileExts:${#argProcStrings[@]} DNE ${#argProcFileExts[@]}";
	yell "argProcStrings:${argProcStrings[*]}"; yell "argProcFileExts:${argProcFileExts[*]}"; exit 1; fi;
    ## Make sure that no array elements are blank
    for element in "${argProcStrings[@]}"; do
	if [[ -z "$element" ]]; then yell "ERROR:Empty process string specified. Exiting."; exit 1; fi; done
    for element in "${argProcFileExts[@]}"; do
	if [[ -z "$element" ]]; then yell "ERROR:Empty output file extension specified. Exiting."; exit 1; fi; done
    ## Make sure that no process string starts with '-' (ex: if only one arg supplied after '-p' option)
    for element in "${argProcStrings[@]}"; do
	if [[ ! "$element" =~ ^[-][[:print:]]*$ ]] && [[ "$element" =~ ^[[:print:]]*$ ]]; then
	    yell "ERROR:Illegal character '-' at start of process string element. Option syntax error?";
	    exit 1; fi; done;
    vbm "STATUS:Quick check shows argProcStrings and argProcFileExts appear to have valid contents.";
    procStrings=("${argProcStrings[@]}"); # Export process command strings
    procFileExts=("${argProcFileExts[@]}"); # Export process command strings
    vbm "STATUS:Finished magicParseProcessStrings() function.";
} # Validate and save process strings and file extensions to arrays procStrings, procFileExts
magicParseLabel() {
    # Desc: Parses -l option to set label
    # In : optionLabel, argLabel
    # Out: vars: label
    # Depends: Bash 5.0.3, vbm(), yell()
    
    vbm "STATUS:Started magicParseLabel() function.";
    # Do nothing if optionLabel not set to true.
    if [[ ! "$optionLabel" = "true" ]]; then
	vbm "STATUS:optionlabel not set to 'true'. Returning early.";
	return;
    fi;
    # Set label if optionLabel is true
    if [[ "$optionLabel" = "true" ]]; then
	label="_""$argLabel";
	vbm "STATUS:Set label:$label";
    fi;
    vbm "STATUS:Finished magicParseLabel() function.";
}
magicProcessWriteBuffer() {
    # Desc: process and write buffer
    # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS
    #    : arry: buffer
    # Out: file:(pathout_tar)
    # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(),
    ### Note: These arrays should all have the same number of elements:
    ###       pathouts, fileouts, procFileExts, procStrings
    
    local fn timeBufferStartLong timeBufferStart fileoutBasename
    local -a fileouts pathouts
    local writeCmd1 writeCmd2 writeCmd3 writeCmd4

    vbm "DEBUG:STATUS:$fn:Started magicProcessWriteBuffer().";
    # Debug:Get function name
    fn="${FUNCNAME[0]}";

    # Determine file paths (time is start of buffer period)
    ## Calculate start time
    timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \
	vbm "timeBufferStartLong:$timeBufferStartLong";
    timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \
	vbm "timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators)
    ## Set common basename
    fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \
	vbm "STATUS:Set fileoutBasename to:$fileoutBasename";
    ## Determine output file name array
    ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts
    for fileExt in "${procFileExts[@]}"; do
	fileouts+=("$fileoutBasename""$fileExt""$cmd_compress_suffix""$cmd_encrypt_suffix") && \
	    vbm "STATUS:Added $fileExt to fileouts:${fileouts[*]}";
    done;
    for fileName in "${fileouts[@]}"; do
	pathouts+=("$dir_tmp"/"$fileName") && \
	    vbm "STATUS:Added $fileName to pathouts:${pathouts[*]}";
    done;
    ## Update pathout_tar
    magicInitCheckTar;
    
    # Process and write buffers to dir_tmp
    ## Prepare command strings
    writeCmd1="printf \"%s\\\\n\" \"\${buffer[@]}\""; # printf "%s\\n" "${buffer[@]}"
    #writeCmd2="" # NOTE: Specified by parsing array procStrings
    writeCmd3="$cmd_compress";
    writeCmd4="$cmd_encrypt";
    
    ## Process buffer and write to dir_tmp
    for index in "${!pathouts[@]}"; do
	writeCmd2="${procStrings[$index]}"
	eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" >> "${pathouts[$index]}";
    done;
    
    # Append dir_tmp files to pathout_tar
    wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
    for index in "${!pathouts[@]}"; do
	appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp";
    done;
    
    # Remove secured chunks from dir_tmp
    for path in "${pathouts[@]}"; do
	rm "$path";
    done;
    
    vbm "DEBUG:STATUS:$fn:Finished magicProcessWriteBuffer().";
} # Process and Write buffer
main() {
    # Process arguments
    processArguments "$@";
    ## Determine working directory
    magicInitWorkingDir; # Sets dir_tmp from argTempDirPriority
    ## Set output encryption and compression option strings
    ### React to "-e" and "-r" ("encryption recipients") options
    magicParseRecipientArgs; # Updates recPubKeysValid, cmd_encrypt[_suffix] from argRecPubKeys
    ### React to "-R" ("recipient directory") option
    magicParseRecipientDir; # Updates recPubKeysValid
    ### React to "-c" ("compression") option
    magicParseCompressionArg; # Updates cmd_compress[_suffix]
    ## React to "-b" and "-B" (custom buffer and script TTL) options
    magicParseCustomTTL; # Sets custom scriptTTL_TE and/or bufferTTL if specified
    ## React to "-p" (user-supplied process command and file extension strings) options
    magicParseProcessStrings; # Sets arrays: procStrings, procFileExts
    ## React to "-l" (output file label) option
    magicParseLabel; # sets label (ex: "_location")
    ## React to "-w" (how to name raw stdin file) option
    magicParseStoreRaw; # sets raw_suffix

    # Perform secondary setup operations
    ## Set script lifespan (scriptTTL from scriptTTL_TE)
    magicSetScriptTTL "$scriptTTL_TE";
    ## File name substring (ISO-8601 duration from bufferTTL)
    bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG:bufferTTL_STR:$bufferTTL_STR";
    ## Init temp working dir
    try mkdir "$dir_tmp" && vbm "DEBUG:Working dir created at dir_tmp:$dir_tmp";
    ## Initialize output tar (set pathout_tar)
    magicInitCheckTar; 
    
    # Check vital apps, files, dirs
    if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then
	yell "ERROR:Critical components missing.";
	displayMissing; yell "Exiting."; exit 1; fi

    # MAIN LOOP: Run until script TTL seconds pass
    bufferRound=0;
    while [[ $SECONDS -lt "scriptTTL" ]]; do
	bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death
	lineCount=0; # Debug counter
	# Consume stdin to fill buffer until buffer time-of-death (TOD) arrives
	while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do
	    # Append line to buffer array
	    buffer+=("$line");
	    echo "DEBUG:Processing line:$lineCount";
	    echo "DEBUG:Current line   :$line";
	    echo "DEBUG:buf elem count :${#buffer[@]}";
	    ((lineCount++));
	done;
	# Create dir_tmp if missing
	if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:dir_tmp existence failure:$dir_tmp"; try mkdir "$dir_tmp" && vbm "DEBUG:Working dir recreated dir_tmp:$dir_tmp"; fi
	# Update encryption recipient array 
	magicParseRecipientDir; # Update recPubKeysValid with argRecDir
	# Export buffer to asynchronous processing.
	magicProcessWriteBuffer &
	unset buffer; # Clear buffer array for next bufferRound
	# Increment buffer round
	((bufferRound++));
    done;

    # Cleanup
    ## Remove dir_tmp
    try rm -r "$dir_tmp" && vbm "Removed dir_tmp:$dir_tmp";

    vbm "STATUS:Main function finished.";
} # Main function

#===END Declare local script functions===
#==END Define script parameters==

#==BEGIN Perform work and exit==
main "$@" # Run main function.
exit 0;
#==END Perform work and exit==

# Author: Steven Baltakatei Sandoval;
# License: GPLv3+