#!/bin/bash # Desc: Compresses, encrypts, and writes stdin every 5 seconds #==BEGIN Define script parameters== #===BEGIN Initialize variables=== # Logging Behavior parameters bufferTTL="300"; # Time-to-live (seconds) for each buffer round scriptTTL_TE="day"; # Time element at the end of which script terminates dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. scriptVersion="0.1.28"; # Define version of script. scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case user apps saved there ageVersion="1.0.0-beta2"; # Define version of age (encryption program) ageURL="https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2"; # Define website hosting age. # Arrays declare -a buffer # array for storing while read buffer declare -a argRecPubKeys # array for processArguments function declare -a recPubKeysValid # array for storing both '-r' and '-R' recipient pubkeys declare -a argProcStrings argProcFileExts # for storing buffer processing strings (ex: "gpsbabel -i nmea -f - -o gpx -F - ") declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status declare -a procStrings procFileExts # Arrays for storing processing commands and resulting output file extensions # Variables optionVerbose=""; optionEncrypt=""; dirOut=""; optionEncrypt=""; dir_tmp=""; cmd_compress="";cmd_compress_suffix=""; cmd_encrypt=""; cmd_encrypt_suffix=""; #===END Initialize variables=== #===BEGIN Declare local script functions=== yell() { echo "$0: $*" >&2; } #o Yell, Die, Try Three-Fingered Claw technique die() { yell "$*"; exit 111; } #o Ref/Attrib: https://stackoverflow.com/a/25515370 try() { "$@" || die "cannot $*"; } #o processArguments() { while [ ! $# -eq 0 ]; do # While number of arguments ($#) is not (!) equal to (-eq) zero (0). case "$1" in -v | --verbose) optionVerbose="true"; vbm "DEBUG :Verbose mode enabled.";; # Enable verbose mode. -h | --help) showUsage; exit 1;; # Display usage. --version) showVersion; exit 1;; # Show version -o | --output) if [ -d "$2" ]; then dirOut="$2"; vbm "DEBUG :dirOut:$dirOut"; shift; fi ;; # Define output directory. -e | --encrypt) optionEncrypt="true"; vbm "DEBUG :Encrypted output mode enabled.";; # Enable encryption -r | --recipient) optionRecArg="true"; argRecPubKeys+=("$2"); vbm "STATUS:pubkey added:""$2"; shift;; # Add recipients -R | --recipient-dir) optionRecDir="true" && argRecDir="$2"; shift;; # Add recipient watch dir -c | --compress) optionCompress="true"; vbm "DEBUG :Compressed output mode enabled.";; # Enable compression -z | --time-zone) try setTimeZoneEV "$2"; shift;; # Set timestamp timezone -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds) -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day") -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$3\" for output of processing string added:\"$2\""; shift; shift;; -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG :Custom label received:$argLabel"; shift;; -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG :Raw stdin file extension received:$argRawFileExt"; shift;; -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG :Option selected to not store raw stdin data."; shift;; *) yell "ERROR: Unrecognized argument: $1"; yell "STATUS:All arguments:$*"; exit 1;; # Handle unrecognized options. esac shift done } # Argument Processing vbm() { # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true". # Usage: vbm "DEBUG :verbose message here" # Version 0.1.3 # Input: arg1: string # vars: optionVerbose # Output: stderr # Depends: bash 5.0.3, echo 8.30, date 8.30 if [ "$optionVerbose" = "true" ]; then functionTime=$(date --iso-8601=ns); # Save current time in nano seconds. echo "[$functionTime]:$0:""$*" 1>&2; # Display argument text. fi # End function return 0; # Function finished. } # Displays message if optionVerbose true checkapp() { # Desc: If arg is a command, save result in assoc array 'appRollCall' # Usage: checkapp arg1 arg2 arg3 ... # Version: 0.1.1 # Input: global assoc. array 'appRollCall' # Output: adds/updates key(value) to global assoc array 'appRollCall' # Depends: bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command appRollCall[$arg]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi; else appRollCall[$arg]="false"; returnState="false"; fi; done; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi; } # Check that app exists checkfile() { # Desc: If arg is a file path, save result in assoc array 'fileRollCall' # Usage: checkfile arg1 arg2 arg3 ... # Version: 0.1.1 # Input: global assoc. array 'fileRollCall' # Output: adds/updates key(value) to global assoc array 'fileRollCall'; # Output: returns 0 if app found, 1 otherwise # Depends: bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if [ -f "$arg" ]; then fileRollCall["$arg"]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi; else fileRollCall["$arg"]="false"; returnState="false"; fi; done; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi; } # Check that file exists checkdir() { # Desc: If arg is a dir path, save result in assoc array 'dirRollCall' # Usage: checkdir arg1 arg2 arg3 ... # Version 0.1.1 # Input: global assoc. array 'dirRollCall' # Output: adds/updates key(value) to global assoc array 'dirRollCall'; # Output: returns 0 if app found, 1 otherwise # Depends: Bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if [ -d "$arg" ]; then dirRollCall["$arg"]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi else dirRollCall["$arg"]="false"; returnState="false"; fi done #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi } # Check that dir exists displayMissing() { # Desc: Displays missing apps, files, and dirs # Usage: displayMissing # Version 0.1.1 # Input: associative arrays: appRollCall, fileRollCall, dirRollCall # Output: stderr: messages indicating missing apps, file, or dirs # Depends: bash 5, checkAppFileDir() local missingApps value appMissing missingFiles fileMissing local missingDirs dirMissing #==BEGIN Display errors== #===BEGIN Display Missing Apps=== missingApps="Missing apps :"; #for key in "${!appRollCall[@]}"; do echo "DEBUG :$key => ${appRollCall[$key]}"; done for key in "${!appRollCall[@]}"; do value="${appRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG :Missing apps: $key => $value"; missingApps="$missingApps""$key "; appMissing="true"; fi; done; if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing. echo "$missingApps" 1>&2; fi; unset value; #===END Display Missing Apps=== #===BEGIN Display Missing Files=== missingFiles="Missing files:"; #for key in "${!fileRollCall[@]}"; do echo "DEBUG :$key => ${fileRollCall[$key]}"; done for key in "${!fileRollCall[@]}"; do value="${fileRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG :Missing files: $key => $value"; missingFiles="$missingFiles""$key "; fileMissing="true"; fi; done; if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing. echo "$missingFiles" 1>&2; fi; unset value; #===END Display Missing Files=== #===BEGIN Display Missing Directories=== missingDirs="Missing dirs:"; #for key in "${!dirRollCall[@]}"; do echo "DEBUG :$key => ${dirRollCall[$key]}"; done for key in "${!dirRollCall[@]}"; do value="${dirRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG :Missing dirs: $key => $value"; missingDirs="$missingDirs""$key "; dirMissing="true"; fi; done; if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing. echo "$missingDirs" 1>&2; fi; unset value; #===END Display Missing Directories=== #==END Display errors== } # Display missing apps, files, dirs appendArgTar(){ # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) # Version: 1.0.6 # Input: arg1: data to be written # arg2: file name of file to be inserted into tar # arg3: tar archive path (must exist first) # arg4: temporary working dir # arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ") # Output: file written to disk # Example: decrypt multiple large files in parallel # appendArgTar "$(cat /tmp/largefile1.gpg)" "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & # appendArgTar "$(cat /tmp/largefile2.gpg)" "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & # appendArgTar "$(cat /tmp/largefile3.gpg)" "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & # Depends: bash 5, tar 1, yell() # Ref/Attrib: Using 'eval' to construct command strings https://askubuntu.com/a/476533 local fn fileName tarPath tmpDir cmd0 cmd1 cmd2 cmd3 cmd4 # Save function name fn="${FUNCNAME[0]}"; #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." # Set file name if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi # Check tar path is a file if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file."; exit 1; fi # Check temp dir arg if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi # Set command strings if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string 1 if ! [ -z "$6" ]; then cmd2="$6"; else cmd2="cat "; fi # command string 2 if ! [ -z "$7" ]; then cmd3="$7"; else cmd3="cat "; fi # command string 3 if ! [ -z "$8" ]; then cmd4="$8"; else cmd4="cat "; fi # command string 4 # Input command cmd0="echo \"\$1\"" # # Debug # yell "DEBUG:STATUS:$fn:cmd0:$cmd0" # yell "DEBUG:STATUS:$fn:cmd1:$cmd1" # yell "DEBUG:STATUS:$fn:cmd2:$cmd2" # yell "DEBUG:STATUS:$fn:cmd3:$cmd3" # yell "DEBUG:STATUS:$fn:cmd4:$cmd4" # yell "DEBUG:STATUS:$fn:fileName:$fileName" # yell "DEBUG:STATUS:$fn:tarPath:$tarPath" # yell "DEBUG:STATUS:$fn:tmpDir:$tmpDir" # Write to temporary working dir eval "$cmd0 | $cmd1 | $cmd2 | $cmd3 | $cmd4" > "$tmpDir"/"$fileName"; # Append to tar try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName"; #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." } # Append Bash var to file appended to Tar archive appendFileTar(){ # Desc: Appends [processed] file to tar # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd]) # Version: 2.0.1 # Input: arg1: path of file to be (processed and) written # arg2: name to use for file inserted into tar # arg3: tar archive path (must exist first) # arg4: temporary working dir # arg5: (optional) command string to process file (ex: "gpsbabel -i nmea -f - -o kml -F - ") # Output: file written to disk # Example: decrypt multiple large files in parallel # appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & # appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & # appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & # Depends: bash 5.0.3, tar 1.30, cat 8.30, yell() local fn fileName tarPath tmpDir # Save function name fn="${FUNCNAME[0]}"; #yell "DEBUG :STATUS:$fn:Started appendFileTar()." # Set file name if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi # Check tar path is a file if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file:$3"; exit 1; fi # Check temp dir arg if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi # Set command strings if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string # Input command string cmd0="cat \"\$1\""; # Write to temporary working dir eval "$cmd0 | $cmd1" > "$tmpDir"/"$fileName"; # Append to tar try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName"; #yell "DEBUG :STATUS:$fn:Finished appendFileTar()." } # Append [processed] file to Tar archive checkAgePubkey() { # Desc: Checks if string is an age-compatible pubkey # Usage: checkAgePubkey [str pubkey] # Version: 0.1.2 # Input: arg1: string # Output: return code 0: string is age-compatible pubkey # return code 1: string is NOT an age-compatible pubkey # age stderr (ex: there is stderr if invalid string provided) # Depends: age (v0.1.0-beta2; https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2 ) argPubkey="$1"; if echo "test" | age -a -r "$argPubkey" 1>/dev/null; then return 0; else return 1; fi; } # Check age pubkey checkMakeTar() { # Desc: Checks that a valid tar archive exists, creates one otherwise # Usage: checkMakeTar [ path ] # Version: 1.0.2 # Input: arg1: path of tar archive # Output: exit code 0 : tar readable # exit code 1 : tar missing; created # exit code 2 : tar not readable; moved; replaced # Depends: bash 5, date 8, tar 1, try() local pathTar returnFlag0 returnFlag1 returnFlag2 pathTar="$1"; # Check if file is a valid tar archive if tar --list --file="$pathTar" 1>/dev/null 2>&1; then ## T1: return success returnFlag0="tar valid"; elif { sleep 2; tar --list --file="$pathTar" 1>/dev/null 2>&1; }; then ## F1: Check tar archive again after 2-second sleep returnFlag0="tar valid"; else ## F2-1: Check if file exists if [[ -f "$pathTar" ]]; then ### T: Rename file try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S%z)" && \ returnFlag1="tar moved"; else ### F: - : fi; ## F2-1: Create tar archive, return 0 try tar --create --file="$pathTar" --files-from=/dev/null && \ returnFlag2="tar created"; fi; # Determine function return code if [[ "$returnFlag0" = "tar valid" ]]; then return 0; elif [[ "$returnFlag2" = "tar created" ]] && ! [[ "$returnFlag1" = "tar moved" ]]; then return 1; # tar missing so created elif [[ "$returnFlag2" = "tar created" ]] && [[ "$returnFlag1" = "tar moved" ]]; then return 2; # tar not readable so moved; replaced fi; } # checks if arg1 is tar; creates one otherwise dateShort(){ # Desc: Date without separators (YYYYmmdd) # Usage: dateShort ([str date]) # Version: 1.1.2 # Input: arg1: 'date'-parsable timestamp string (optional) # Output: stdout: date (ISO-8601, no separators) # Depends: bash 5.0.3, date 8.30, yell() local argTime timeCurrent timeInput dateCurrentShort argTime="$1"; # Get Current Time timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. # Decide to parse current or supplied date ## Check if time argument empty if [[ -z "$argTime" ]]; then ## T: Time argument empty, use current time timeInput="$timeCurrent"; else ## F: Time argument exists, validate time if date --date="$argTime" 1>/dev/null 2>&1; then ### T: Time argument is valid; use it timeInput="$argTime"; else ### F: Time argument not valid; exit yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; fi; fi; # Construct and deliver separator-les date string dateCurrentShort="$(date -d "$timeInput" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day. echo "$dateCurrentShort"; } # Get YYYYmmdd dateTimeShort(){ # Desc: Timestamp without separators (YYYYmmddTHHMMSS+zzzz) # Usage: dateTimeShort ([str date]) # Version 1.1.1 # Input: arg1: 'date'-parsable timestamp string (optional) # Output: stdout: timestamp (ISO-8601, no separators) # Depends: yell local argTime timeCurrent timeInput timeCurrentShort argTime="$1"; # Get Current Time timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. # Decide to parse current or supplied date ## Check if time argument empty if [[ -z "$argTime" ]]; then ## T: Time argument empty, use current time timeInput="$timeCurrent"; else ## F: Time argument exists, validate time if date --date="$argTime" 1>/dev/null 2>&1; then ### T: Time argument is valid; use it timeInput="$argTime"; else ### F: Time argument not valid; exit yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; fi fi # Construct and deliver separator-les date string timeCurrentShort="$(date -d "$timeInput" +%Y%m%dT%H%M%S%z)"; echo "$timeCurrentShort"; } # Get YYYYmmddTHHMMSS±zzzz setTimeZoneEV(){ # Desc: Set time zone environment variable TZ # Usage: setTimeZoneEV arg1 # Version 0.1.2 # Input: arg1: 'date'-compatible timezone string (ex: "America/New_York") # TZDIR env var (optional; default: "/usr/share/zoneinfo") # Output: exports TZ # exit code 0 on success # exit code 1 on incorrect number of arguments # exit code 2 if unable to validate arg1 # Depends: yell(), printenv 8.30, bash 5.0.3 # Tested on: Debian 10 local tzDir returnState argTimeZone argTimeZone="$1" if ! [[ $# -eq 1 ]]; then yell "ERROR:Invalid argument count."; return 1; fi # Read TZDIR env var if available if printenv TZDIR 1>/dev/null 2>&1; then tzDir="$(printenv TZDIR)"; else tzDir="/usr/share/zoneinfo"; fi # Validate TZ string if ! [[ -f "$tzDir"/"$argTimeZone" ]]; then yell "ERROR:Invalid time zone argument."; return 2; else # Export ARG1 as TZ environment variable TZ="$argTimeZone" && export TZ && returnState="true"; fi # Determine function return code if [ "$returnState" = "true" ]; then return 0; fi } # Exports TZ environment variable showVersion() { yell "$scriptVersion" } # Display script version. timeDuration(){ # Desc: Given seconds, output ISO-8601 duration string # Ref/Attrib: ISO-8601:2004(E), §4.4.4.2 Representations of time intervals by duration and context information # Note: "1 month" ("P1M") is assumed to be "30 days" (see ISO-8601:2004(E), §2.2.1.2) # Usage: timeDuration [1:seconds] ([2:precision]) # Version: 1.0.5 # Input: arg1: seconds as base 10 integer >= 0 (ex: 3601) # arg2: precision level (optional; default=2) # Output: stdout: ISO-8601 duration string (ex: "P1H1S", "P2Y10M15DT10H30M20S") # exit code 0: success # exit code 1: error_input # exit code 2: error_unknown # Example: 'timeDuration 111111 3' yields 'P1DT6H51M' # Depends: date 8, bash 5, yell, local argSeconds argPrecision precision returnState remainder local fullYears fullMonths fullDays fullHours fullMinutes fullSeconds local hasYears hasMonths hasDays hasHours hasMinutes hasSeconds local witherPrecision output local displayYears displayMonths displayDays displayHours displayMinutes displaySeconds argSeconds="$1"; # read arg1 (seconds) argPrecision="$2"; # read arg2 (precision) precision=2; # set default precision # Check that between one and two arguments is supplied if ! { [[ $# -ge 1 ]] && [[ $# -le 2 ]]; }; then yell "ERROR:Invalid number of arguments:$# . Exiting."; returnState="error_input"; fi # Check that argSeconds provided if [[ $# -ge 1 ]]; then ## Check that argSeconds is a positive integer if [[ "$argSeconds" =~ ^[[:digit:]]+$ ]]; then : else yell "ERROR:argSeconds not a digit."; returnState="error_input"; fi else yell "ERROR:No argument provided. Exiting."; exit 1; fi # Consider whether argPrecision was provided if [[ $# -eq 2 ]]; then # Check that argPrecision is a positive integer if [[ "$argPrecision" =~ ^[[:digit:]]+$ ]] && [[ "$argPrecision" -gt 0 ]]; then precision="$argPrecision"; else yell "ERROR:argPrecision not a positive integer. (is $argPrecision ). Leaving early."; returnState="error_input"; fi; else : fi; remainder="$argSeconds" ; # seconds ## Calculate full years Y, update remainder fullYears=$(( remainder / (365*24*60*60) )); remainder=$(( remainder - (fullYears*365*24*60*60) )); ## Calculate full months M, update remainder fullMonths=$(( remainder / (30*24*60*60) )); remainder=$(( remainder - (fullMonths*30*24*60*60) )); ## Calculate full days D, update remainder fullDays=$(( remainder / (24*60*60) )); remainder=$(( remainder - (fullDays*24*60*60) )); ## Calculate full hours H, update remainder fullHours=$(( remainder / (60*60) )); remainder=$(( remainder - (fullHours*60*60) )); ## Calculate full minutes M, update remainder fullMinutes=$(( remainder / (60) )); remainder=$(( remainder - (fullMinutes*60) )); ## Calculate full seconds S, update remainder fullSeconds=$(( remainder / (1) )); remainder=$(( remainder - (remainder*1) )); ## Check which fields filled if [[ $fullYears -gt 0 ]]; then hasYears="true"; else hasYears="false"; fi if [[ $fullMonths -gt 0 ]]; then hasMonths="true"; else hasMonths="false"; fi if [[ $fullDays -gt 0 ]]; then hasDays="true"; else hasDays="false"; fi if [[ $fullHours -gt 0 ]]; then hasHours="true"; else hasHours="false"; fi if [[ $fullMinutes -gt 0 ]]; then hasMinutes="true"; else hasMinutes="false"; fi if [[ $fullSeconds -ge 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi ## Determine which fields to display (see ISO-8601:2004 §4.4.3.2) witherPrecision="false" ### Years if $hasYears && [[ $precision -gt 0 ]]; then displayYears="true"; witherPrecision="true"; else displayYears="false"; fi; if $witherPrecision; then ((precision--)); fi; ### Months if $hasMonths && [[ $precision -gt 0 ]]; then displayMonths="true"; witherPrecision="true"; else displayMonths="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayMonths="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Days if $hasDays && [[ $precision -gt 0 ]]; then displayDays="true"; witherPrecision="true"; else displayDays="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayDays="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Hours if $hasHours && [[ $precision -gt 0 ]]; then displayHours="true"; witherPrecision="true"; else displayHours="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayHours="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Minutes if $hasMinutes && [[ $precision -gt 0 ]]; then displayMinutes="true"; witherPrecision="true"; else displayMinutes="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayMinutes="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Seconds if $hasSeconds && [[ $precision -gt 0 ]]; then displaySeconds="true"; witherPrecision="true"; else displaySeconds="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displaySeconds="true"; fi; if $witherPrecision; then ((precision--)); fi; ## Determine whether or not the "T" separator is needed to separate date and time elements if ( $displayHours || $displayMinutes || $displaySeconds); then displayDateTime="true"; else displayDateTime="false"; fi ## Construct duration output string output="P" if $displayYears; then output=$output$fullYears"Y"; fi if $displayMonths; then output=$output$fullMonths"M"; fi if $displayDays; then output=$output$fullDays"D"; fi if $displayDateTime; then output=$output"T"; fi if $displayHours; then output=$output$fullHours"H"; fi if $displayMinutes; then output=$output$fullMinutes"M"; fi if $displaySeconds; then output=$output$fullSeconds"S"; fi ## Output duration string to stdout echo "$output" && returnState="true"; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; elif [ "$returnState" = "error_input" ]; then yell "ERROR:input"; return 1; else yell "ERROR:Unknown"; return 2; fi } # Get duration (ex: PT10M4S ) timeUntilNextDay(){ # Desc: Report seconds until next day. # Version: 1.0.2 # Output: stdout: integer seconds until next day # Output: exit code 0 if stdout > 0; 1 if stdout = 0; 2 if stdout < 0 # Usage: timeUntilNextDay # Usage: if ! myTTL="$(timeUntilNextDay)"; then yell "ERROR in if statement"; exit 1; fi # Depends: date 8, echo 8, yell, try local returnState timeCurrent timeNextDay secondsUntilNextDay returnState timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. timeNextDay="$(date -d "$timeCurrent next day" --iso-8601=date)"; # Produce timestamp of beginning of tomorrow with resolution of 1 second. secondsUntilNextDay="$(( $(date +%s -d "$timeNextDay") - $(date +%s -d "$timeCurrent") ))" ; # Calculate seconds until closest future midnight (res. 1 second). if [[ "$secondsUntilNextDay" -gt 0 ]]; then returnState="true"; elif [[ "$secondsUntilNextDay" -eq 0 ]]; then returnState="warning_zero"; yell "WARNING:Reported time until next day exactly zero."; elif [[ "$secondsUntilNextDay" -lt 0 ]]; then returnState="warning_negative"; yell "WARNING:Reported time until next day is negative."; fi try echo "$secondsUntilNextDay"; # Report # Determine function return code if [[ "$returnState" = "true" ]]; then return 0; elif [[ "$returnState" = "warning_zero" ]]; then return 1; elif [[ "$returnState" = "warning_negative" ]]; then return 2; fi } # Report seconds until next day timeUntilNextHour(){ # Desc: Report seconds until next hour # Version 1.0.1 # Output: stdout: integer seconds until next hour # Output: exit code 0 if stdout > 0; 1 if stdout = 0; 2 if stdout < 0 # Usage: timeUntilNextHour # Usage: if ! myTTL="$(timeUntilNextHour)"; then yell "ERROR in if statement"; exit 1; fi local returnState timeCurrent timeNextHour secondsUntilNextHour timeCurrent="$(date --iso-8601=seconds)"; # Produce `date`-parsable current timestamp with resolution of 1 second. timeNextHour="$(date -d "$timeCurrent next hour" --iso-8601=hours)"; # Produce `date`-parsable current time stamp with resolution of 1 second. secondsUntilNextHour="$(( $(date +%s -d "$timeNextHour") - $(date +%s -d "$timeCurrent") ))"; # Calculate seconds until next hour (res. 1 second). if [[ "$secondsUntilNextHour" -gt 0 ]]; then returnState="true"; elif [[ "$secondsUntilNextHour" -eq 0 ]]; then returnState="warning_zero"; yell "WARNING:Reported time until next hour exactly zero."; elif [[ "$secondsUntilNextHour" -lt 0 ]]; then returnState="warning_negative"; yell "WARNING:Reported time until next hour is negative."; fi; try echo "$secondsUntilNextHour"; # Report # Determine function return code if [[ "$returnState" = "true" ]]; then return 0; elif [[ "$returnState" = "warning_zero" ]]; then return 1; elif [[ "$returnState" = "warning_negative" ]]; then return 2; fi; } # Report seconds until next hour validateInput() { # Desc: Validates Input # Usage: validateInput [str input] [str input type] # Version: 0.3.1 # Input: arg1: string to validate # arg2: string specifying input type (ex:"ssh_pubkey") # Output: return code 0: if input string matched specified string type # Depends: bash 5, yell() local fn argInput argType # Save function name fn="${FUNCNAME[0]}"; # Process arguments argInput="$1"; argType="$2"; if [[ $# -gt 2 ]]; then yell "ERROR:$0:$fn:Too many arguments."; exit 1; fi; # Check for blank if [[ -z "$argInput" ]]; then return 1; fi # Define input types ## ssh_pubkey ### Check for alnum/dash base64 (ex: "ssh-rsa AAAAB3NzaC1yc2EAAA") if [[ "$argType" = "ssh_pubkey" ]]; then if [[ "$argInput" =~ ^[[:alnum:]-]*[\ ]*[[:alnum:]+/=]*$ ]]; then return 0; fi; fi; ## age_pubkey ### Check for age1[:bech32:] if [[ "$argType" = "age_pubkey" ]]; then if [[ "$argInput" =~ ^age1[qpzry9x8gf2tvdw0s3jn54khce6mua7l]*$ ]]; then return 0; fi; fi ## integer if [[ "$argType" = "integer" ]]; then if [[ "$argInput" =~ ^[[:digit:]]*$ ]]; then return 0; fi; fi; ## time element (year, month, week, day, hour, minute, second) if [[ "$argType" = "time_element" ]]; then if [[ "$argInput" = "year" ]] || \ [[ "$argInput" = "month" ]] || \ [[ "$argInput" = "week" ]] || \ [[ "$argInput" = "day" ]] || \ [[ "$argInput" = "hour" ]] || \ [[ "$argInput" = "minute" ]] || \ [[ "$argInput" = "second" ]]; then return 0; fi; fi; # Return error if no condition matched. return 1; } # Validates strings magicInitWorkingDir() { # Desc: Determine temporary working directory from defaults or user input # Usage: magicInitWorkingDir # Input: vars: optionTmpDir, argTempDirPriority, dirTmpDefault # Input: vars: scriptTimeStart # Output: vars: dir_tmp # Depends: bash 5.0.3, processArguments(), vbm(), yell() # Parse '-t' option (user-specified temporary working dir) ## Set dir_tmp_parent to user-specified value if specified local fn dir_tmp_parent # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicInitWorkingDir() function."; if [[ "$optionTmpDir" = "true" ]]; then if [[ -d "$argTempDirPriority" ]]; then dir_tmp_parent="$argTempDirPriority"; else yell "WARNING:$fn:Specified temporary working directory not valid:$argTempDirPriority"; exit 1; # Exit since user requires a specific temp dir and it is not available. fi; else ## Set dir_tmp_parent to default or fallback otherwise if [[ -d "$dirTmpDefault" ]]; then dir_tmp_parent="$dirTmpDefault"; elif [[ -d /tmp ]]; then yell "WARNING:$fn:$dirTmpDefault not available. Falling back to /tmp ."; dir_tmp_parent="/tmp"; else yell "ERROR:$fn:No valid working directory available. Exiting."; exit 1; fi; fi; ## Set dir_tmp using dir_tmp_parent and nonce (scriptTimeStart) dir_tmp="$dir_tmp_parent"/"$scriptTimeStart""..bkgpslog" && vbm "DEBUG :$fn:Set dir_tmp to:$dir_tmp"; # Note: removed at end of main(). vbm "STATUS:$fn:Finished magicInitWorkingDir() function."; } # Sets working dir magicInitCheckTar() { # Desc: Initializes or checks output tar # input: vars: dirOut, bufferTTL, cmd_encrypt_suffix, cmd_compress_suffix # input: vars: scriptHostname # output: vars: pathout_tar # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion() local fn checkMakeTarES # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicInitCheckTar() function."; # Form pathout_tar pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \ vbm "STATUS:$fn:Set pathout_tar to:$pathout_tar"; # Validate pathout_tar as tar. checkMakeTar "$pathout_tar"; checkMakeTarES="$?"; ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2) vbm "STATUS:$fn:exit status before magicWriteVersion:$checkMakeTarES" if [[ "$checkMakeTarES" -eq 1 ]] || [[ "$checkMakeTarES" -eq 2 ]]; then magicWriteVersion; fi vbm "STATUS:$fn:Finished magicInitCheckTar() function."; } # Initialize tar, set pathout_tar magicParseCompressionArg() { # Desc: Parses compression arguments specified by '-c' option # Input: vars: optionCompress # Output: cmd_compress, cmd_compress_suffix # Depends: processArguments(), vbm(), checkapp(), gzip 1.9 local fn # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicParseCompressionArg() function."; if [[ "$optionCompress" = "true" ]]; then # Check if compression option active if checkapp gzip; then # Check if gzip available cmd_compress="gzip " && vbm "STATUS:$fn:cmd_compress:$cmd_compress"; cmd_compress_suffix=".gz" && vbm "STATUS:$fn:cmd_compress_suffix:$cmd_compress_suffix"; else yell "ERROR:$fn:Compression enabled but \"gzip\" not found. Exiting."; exit 1; fi; else cmd_compress="tee /dev/null " && vbm "STATUS:$fn:cmd_compress:$cmd_compress"; cmd_compress_suffix="" && vbm "STATUS:$fn:cmd_compress_suffix:$cmd_compress_suffix"; vbm "DEBUG :$fn:Compression not enabled."; fi; vbm "STATUS:$fn:Finished magicParseCompressionArg() function."; } # Form compression cmd string and filename suffix magicParseCustomTTL() { # Desc: Set user-specified TTLs for buffer and script # Usage: magicParseCustomTTL # Input: vars: argCustomBufferTTL (integer), argCustomScriptTTL_TE (string) # Input: vars: optionCustomBufferTTL, optionCustomScriptTTL_TE # Input: vars: bufferTTL (integer), scriptTTL_TE (string) # Output: bufferTTL (integer), scriptTTL_TE (string) # Depends: Bash 5.0.3, yell(), vbm(), validateInput(), showUsage() local fn # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicParseCustomTTL() function."; # React to '-b, --buffer-ttl' option if [[ "$optionCustomBufferTTL" = "true" ]]; then ## T: Check if argCustomBufferTTL is an integer if validateInput "$argCustomBufferTTL" "integer"; then ### T: argCustomBufferTTL is an integer bufferTTL="$argCustomBufferTTL" && vbm "STATUS:$fn:Custom bufferTTL from -b:$bufferTTL"; else ### F: argcustomBufferTTL is not an integer yell "ERROR:$fn:Invalid integer argument for custom buffer time-to-live."; showUsage; exit 1; fi; ## F: do not change bufferTTL fi; # React to '-B, --script-ttl' option if [[ "$optionCustomScriptTTL_TE" = "true" ]]; then ## T: Check if argCustomScriptTTL is a time element (ex: "day", "hour") if validateInput "$argCustomScriptTTL_TE" "time_element"; then ### T: argCustomScriptTTL is a time element scriptTTL_TE="$argCustomScriptTTL_TE" && vbm "STATUS:$fn:Custom scriptTTL_TE from -B:$scriptTTL_TE"; else ### F: argcustomScriptTTL is not a time element yell "ERROR:$fn:Invalid time element argument for custom script time-to-live."; showUsage; exit 1; fi; ## F: do not change scriptTTL_TE fi; vbm "STATUS:$fn:Finished magicParseCustomTTL() function."; } # Sets custom script or buffer TTL if specified magicParseLabel() { # Desc: Parses -l option to set label # In : optionLabel, argLabel # Out: vars: label # Depends: Bash 5.0.3, vbm(), yell() local fn # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Started magicParseLabel() function."; # Do nothing if optionLabel not set to true. if [[ ! "$optionLabel" = "true" ]]; then vbm "STATUS:$fn:optionlabel not set to 'true'. Returning early."; return; fi; # Set label if optionLabel is true if [[ "$optionLabel" = "true" ]]; then label="_""$argLabel"; vbm "STATUS:$fn:Set label:$label"; fi; vbm "STATUS:$fn:Finished magicParseLabel() function."; } # Set label used in output file name magicParseProcessStrings() { # Desc: Processes user-supplied process strings into process commands for appendFileTar(). # Usage: magicParseProcessStrings # In : vars: optionProcString optionNoStoreRaw optionStoreRaw argRawFileExt # arry: argProcStrings, argProcFileExts # Out: arry: procStrings, procFileExts # Depends Bash 5.0.3, yell(), vbm() local fn rawFileExt # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicParseProcessStrings() function."; vbm "STATUS:$fn:var:optionProcString:$optionProcString"; vbm "STATUS:$fn:var:optionNoStoreRaw:$optionNoStoreRaw"; vbm "STATUS:$fn:var:optionStoreRaw:$optionStoreRaw"; vbm "STATUS:$fn:var:argRawFileExt:$argRawFileExt"; vbm "STATUS:$fn:ary:argProcStrings:${argProcStrings[*]}"; vbm "STATUS:$fn:ary:argProcFileExts:${argProcFileExts[*]}" # Validate input ## Validate argRawFileExt if [[ "$argRawFileExt" =~ ^[.][[:alnum:]]*$ ]]; then rawFileExt="$argRawFileExt" && \ vbm "DEBUG :$fn:Set rawFileExt to \"$argRawFileExt\""; else vbm "DEBUG :$fn:Validation failure for $argRawFileExt . Not set to rawFileExt."; fi; # Add default stdin output file entries for procStrings, procFileExts ## Check if user specified that no raw stdin be saved. if [[ ! "$optionNoStoreRaw" = "true" ]]; then ### T: --no-store-raw not set. Store raw. Append procStrings with cat. vbm "DEBUG :$fn:--no-store-raw not set. Storing raw."; #### Append procStrings array procStrings+=("cat ") && \ vbm "DEBUG :$fn:Appended \"cat \" to procStrings"; vbm "DEBUG :$fn:procStrings array:${procStrings[*]}"; #### Check if --store-raw set. if [[ "$optionStoreRaw" = "true" ]]; then ##### T: --store-raw set. Append procFileExts with user-specified file ext vbm "DEBUG :$fn:--store-raw set."; procFileExts+=("$rawFileExt") && \ vbm "DEBUG :$fn:Appended $rawFileExt to procFileExts"; vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; else ##### F: --store-raw not set. Append procFileExts with default ".stdin" file ext ###### Append procFileExts array procFileExts+=(".stdin") && \ vbm "DEBUG :$fn:Appended \".stdin\" to procFileExts"; vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; fi; else ### F: --no-store-raw set. Do not store raw. #### Do not append procStrings or procFileExts arrays. vbm "STATUS:$fn:--no-store-raw set. Not storing raw."; vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; fi; # Do nothing more if optionProcString not set to true. if [[ ! "$optionProcString" = "true" ]]; then vbm "STATUS:$fn:optionProcString not set to 'true'. Returning early."; return; fi; # Validate input array indices ## Make sure that argProcStrings and argProcFileExts have same index counts if ! [[ "${#argProcStrings[@]}" -eq "${#argProcFileExts[@]}" ]]; then yell "ERROR:$fn:Mismatch in number of elements in arrays argProcStrings and argProcFileExts:${#argProcStrings[@]} DNE ${#argProcFileExts[@]}"; yell "STATUS:$fn:argProcStrings:${argProcStrings[*]}"; yell "STATUS:$fn:argProcFileExts:${argProcFileExts[*]}"; exit 1; fi; ## Make sure that no array elements are blank for element in "${argProcStrings[@]}"; do if [[ -z "$element" ]]; then yell "ERROR:$fn:Empty process string specified. Exiting."; exit 1; fi; done for element in "${argProcFileExts[@]}"; do if [[ -z "$element" ]]; then yell "ERROR:$fn:Empty output file extension specified. Exiting."; exit 1; fi; done ## Make sure that no process string starts with '-' (ex: if only one arg supplied after '-p' option) for element in "${argProcStrings[@]}"; do if [[ "$element" =~ ^[-][[:print:]]*$ ]] && [[ ! "$element" =~ ^[[:print:]]*$ ]]; then yell "ERROR:$fn:Illegal character '-' at start of process string element:\"$element\""; exit 1; fi; done; vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents."; vbm "STATUS:$fn:argProcStrings:${argProcStrings[*]}" vbm "STATUS:$fn:argProcFileExts:${argProcFileExts[*]}" procStrings+=("${argProcStrings[@]}"); # Export process command strings procFileExts+=("${argProcFileExts[@]}"); # Export process command strings vbm "STATUS:$fn:procStrings:${procStrings[*]}" vbm "STATUS:$fn:procFileExts:${procFileExts[*]}" vbm "STATUS:$fn:Finished magicParseProcessStrings() function."; } # Validate and save process strings and file extensions to arrays procStrings, procFileExts magicParseRecipients() { # Desc: Parses recipient arguments specified by '-r' or '-R' options # Usage: magicParseRecipients # In : vars: optionEncrypt, optionRecArg, optionRecDir # arry: argRecPubKeys (-r), argRecDir (-R) # Out: vars: cmd_encrypt, cmd_encrypt_suffix # Depends: head 8.30, checkapp(), checkAgePubkey(), validateInput() local fn recipients recipientDir recFileLine updateRecipients local -a recPubKeysValid candRecPubKeysValid # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicParseRecipients() function."; # Catch illegal option combinations ## Catch case if '-e' is set but neither '-r' nor '-R' is set if [[ "$optionEncrypt" = "true" ]] && \ ! { [[ "$optionRecArg" = "true" ]] || [[ "$optionRecDir" = "true" ]]; }; then yell "ERROR:$fn:\\'-e\\' set but no \\'-r\\' or \\'-R\\' set."; exit 1; fi; ## Catch case if '-r' or '-R' set but '-e' is not if [[ ! "$optionEncrypt" = "true" ]] && \ { [[ "$optionRecArg" = "true" ]] || [[ "$optionRecDir" = "true" ]]; }; then yell "ERROR:$fn:\\'-r\\' or \\'-R\\' set but \\'-e\\' is not set."; exit 1; fi; # Handle no encryption cases if [[ ! "$optionEncrypt" = "true" ]]; then cmd_encrypt="cat " && vbm "STATUS:$fn:cmd_encrypt:$cmd_encrypt"; cmd_encrypt_suffix="" && vbm "STATUS:$fn:cmd_encrypt_suffix:$cmd_encrypt_suffix"; vbm "DEBUG :$fn:Encryption not enabled."; return; fi; # Handle encryption cases ## Check age availability if ! checkapp age; then yell "ERROR:$fn:age not available. Exiting."; exit 1; fi ## Parse '-r' options: validate and append pubkeys from argRecPubKeys to recPubKeysValid if [[ "$optionRecArg" = "true" ]]; then for pubkey in "${argRecPubKeys[@]}"; do # Validate recipient pubkey strings by forming test message vbm "DEBUG :$fn:Testing pubkey string:$pubkey"; if checkAgePubkey "$pubkey" && \ ( validateInput "$pubkey" "ssh_pubkey" || validateInput "$pubkey" "age_pubkey"); then #### Add validated pubkey to recPubKeysValid array recPubKeysValid+=("$pubkey") && \ vbm "DEBUG :$fn:recPubkeysValid:pubkey added:$pubkey"; else yell "ERROR:$fn:Exit code ""$?"". Invalid recipient pubkey string. Exiting."; exit 1; fi; done; vbm "STATUS:$fn:Finished processing argRecPubKeys array"; vbm "DEBUG :$fn:Array of validated pubkeys:${recPubKeysValid[*]}"; fi; ## Parse '-R' options: validate and append pubkeys in argRecDir to recPubKeysValid if [[ "$optionRecDir" = "true" ]]; then ### Check that argRecDir is a directory if [[ -d "$argRecDir" ]]; then recipientDir="$argRecDir" && \ vbm "STATUS:$fn:Recipient watch directory detected:\"$recipientDir\""; #### Initialize variable indicating outcome of pubkey review unset updateRecipients #### Add existing recipients from '-r' option candRecPubKeysValid=("${recPubKeysValid[@]}"); #### Parse files in recipientDir for file in "$recipientDir"/*; do ##### Read first line of each file recFileLine="$(head -n1 "$file")" && \ vbm "STATUS:$fn:Checking if pubkey:\"$recFileLine\""; ##### check if first line is a valid pubkey if checkAgePubkey "$recFileLine" && \ ( validateInput "$recFileLine" "ssh_pubkey" || validateInput "$recFileLine" "age_pubkey"); then ###### T: add candidate pubkey to candRecPubKeysValid candRecPubKeysValid+=("$recFileLine") && \ vbm "STATUS:$fn:RecDir pubkey is valid pubkey:\"$recFileLine\""; else ###### F: throw warning; yell "ERROR:$fn:Invalid recipient file detected. Not modifying recipient list:$recFileLine"; updateRecipients="false"; fi; done #### Write candRecPubKeysValid array to recPubKeysValid if no invalid key detected if ! [[ "$updateRecipients" = "false" ]]; then recPubKeysValid=("${candRecPubKeysValid[@]}") && \ vbm "STATUS:$fn:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\""; fi; fi; fi; ## Form age recipient string from recPubKeysValid for pubkey in "${recPubKeysValid[@]}"; do recipients="$recipients""-r '$pubkey' "; vbm "STATUS:$fn:Added pubkey for forming age recipient string:""$pubkey"; vbm "DEBUG :$fn:recipients:""$recipients"; done; ## Output cmd_encrypt, cmd_encrypt_suffix from recipients cmd_encrypt="age ""$recipients " && vbm "STATUS:$fn:cmd_encrypt:$cmd_encrypt"; cmd_encrypt_suffix=".age" && vbm "STATUS:$fn:cmd_encrypt_suffix:$cmd_encrypt_suffix"; vbm "STATUS:$fn:Finished magicParseRecipients() function."; } # Sets cmd_encrypt, cmd_encrypt_suffix from -r, -R args magicSetScriptTTL() { #Desc: Sets script_TTL seconds from provided time_element string argument #Usage: magicSetScriptTTL [str time_element] #Input: arg1: string (Ex: scriptTTL_TE; "day" or "hour") #Output: var: scriptTTL (integer seconds) #Depends: timeUntilNextHour, timeUntilNextDay local fn argTimeElement # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicSetScriptTTL() function."; argTimeElement="$1"; if [[ "$argTimeElement" = "day" ]]; then # Set script lifespan to end at start of next day vbm "STATUS:$fn:Setting script lifespan to end at start of next day. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextDay exit code $?"; exit 1; fi; fi; elif [[ "$argTimeElement" = "hour" ]]; then # Set script lifespan to end at start of next hour vbm "STATUS:$fn:Setting script lifespan to end at start of next hour. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextHour exit code $?"; exit 1; fi; fi; else yell "ERROR:$fn:Invalid argument for setScriptTTL function:$argTimeElement"; exit 1; fi; vbm "STATUS:$fn:Finished magicSetScriptTTL() function."; } # Set scriptTTL in seconds until next (day|hour). magicWriteVersion() { # Desc: Appends time-stamped VERSION to pathout_tar # Usage: magicWriteVersion # Input: vars: pathout_tar, dir_tmp # Input: vars: scriptVersion, scriptURL, ageVersion, ageURL, scriptHostname # Input: array: recPubKeysValid # Output: appends tar (pathout_tar) # Depends: bash 5.0.3, dateTimeShort(), appendArgTar() local fn fileoutVersion contentVersion pubKeyIndex pubKeyIndex # Save function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Starting magicWriteVersion() function."; # Set VERSION file name fileoutVersion="$(dateTimeShort)..VERSION"; # Gather VERSION data in contentVersion contentVersion="scriptVersion=$scriptVersion"; #contentVersion="$contentVersion""\\n"; contentVersion="$contentVersion""\\n""scriptName=$scriptName"; contentVersion="$contentVersion""\\n""scriptURL=$scriptURL"; contentVersion="$contentVersion""\\n""ageVersion=$ageVersion"; contentVersion="$contentVersion""\\n""ageURL=$ageURL"; contentVersion="$contentVersion""\\n""date=$(date --iso-8601=seconds)"; contentVersion="$contentVersion""\\n""hostname=$scriptHostname"; ## Add list of recipient pubkeys for pubkey in "${recPubKeysValid[@]}"; do ((pubKeyIndex++)) contentVersion="$contentVersion""\\n""PUBKEY_$pubKeyIndex=$pubkey"; done ## Process newline escapes contentVersion="$(echo -e "$contentVersion")" # Write contentVersion as file fileoutVersion and write-append to pathout_tar appendArgTar "$contentVersion" "$fileoutVersion" "$pathout_tar" "$dir_tmp" && \ vbm "STATUS:$fn:Appended $fileoutVersion to $pathout_tar"; vbm "STATUS:$fn:Finished magicWriteVersion() function."; } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer # In : vars: bufferTTL scriptHostname label dir_tmp SECONDS # : vars: timeBufferStartEpoch timeBufferEndEpoch # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings local fn timeBufferStartLong timeBufferStart bufferDuration bufferDurationStr fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 # Debug:Get function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Started magicProcessWriteBuffer()."; vbm "DEBUG :$fn:buffer array element count:${#buffer[@]}"; vbm "DEBUG :$fn:buffer array first element:${buffer[0]}"; vbm "DEBUG :$fn:buffer array last element :${buffer[-1]}"; # Determine file paths (time is start of buffer period) ## Calculate start time timeBufferStartLong="$(date --date="@$timeBufferStartEpoch" --iso-8601=seconds)" && \ vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; # Note start time in 'date' parsable ISO-8601 timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "DEBUG :$fn:timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) ## Calculate buffer duration string (ISO-8601 duration) bufferDuration="$((timeBufferEndEpoch - timeBufferStartEpoch))" && \ vbm "DEBUG :$fn:bufferDuration:$bufferDuration"; # length of time (seconds) stdin was read bufferDurationStr="$(timeDuration "$bufferDuration")" && \ vbm "DEBUG :$fn:bufferDurationStr:$bufferDurationStr"; # buffer duration (ISO-8601) ## Set common basename fileoutBasename="$timeBufferStart""--""$bufferDurationStr""..""$scriptHostname""$label" && \ vbm "STATUS:$fn:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts for fileExt in "${procFileExts[@]}"; do fileouts+=("$fileoutBasename""$fileExt""$cmd_compress_suffix""$cmd_encrypt_suffix") && \ vbm "STATUS:$fn:Added $fileExt to fileouts:${fileouts[*]}"; done; for fileName in "${fileouts[@]}"; do pathouts+=("$dir_tmp"/"$fileName") && \ vbm "STATUS:$fn:Added $fileName to pathouts:${pathouts[*]}"; done; ## Update pathout_tar magicInitCheckTar; # Process and write buffers to dir_tmp ## Prepare command strings writeCmd1="printf \"%s\\\\n\" \"\${buffer[@]}\""; # printf "%s\\n" "${buffer[@]}" #writeCmd2="" # NOTE: Specified by parsing array procStrings writeCmd3="$cmd_compress"; writeCmd4="$cmd_encrypt"; ## Process buffer and write to dir_tmp vbm "DEBUG :$fn:fileouts element count:${#fileouts[@]}"; vbm "DEBUG :$fn:pathouts element count:${#pathouts[@]}"; vbm "DEBUG :$fn:procStrings element count:${#pathouts[@]}"; vbm "DEBUG :$fn:fileouts contents:${fileouts[*]}"; vbm "DEBUG :$fn:pathouts contents:${pathouts[*]}"; vbm "DEBUG :$fn:procStrings contents:${pathouts[*]}"; for index in "${!pathouts[@]}"; do writeCmd2="${procStrings[$index]}"; writeCmdAll="$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" && vbm "STATUS:$fn:Assembled command:\"$writeCmdAll\""; eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" > "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}"; done; # Append dir_tmp files to pathout_tar wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) for index in "${!pathouts[@]}"; do tar --append --directory="$dir_tmp" --file="$pathout_tar" "${fileouts[$index]}" && \ vbm "STATUS:$fn:Appended ${pathouts[$index]} to $pathout_tar"; #appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \ done; # Remove secured chunks from dir_tmp for path in "${pathouts[@]}"; do rm "$path" && vbm "STATUS:$fn:Removed:$path"; done; vbm "STATUS:$fn:Finished magicProcessWriteBuffer()."; } # Process and Write buffer showUsage() { cat <<'EOF' USAGE: cmd | bklog [ options ] OPTIONS: -h, --help Display help information. --version Display script version. -v, --verbose Display debugging info. -e, --encrypt Encrypt output. -r, --recipient [ string pubkey ] Specify recipient. May be age or ssh pubkey. May be specified multiple times for multiple pubkeys. See https://github.com/FiloSottile/age -o, --output [ path dir ] Specify output directory to save logs. This option is required to save log data. -p, --process-string [ filter command ] [ output file extension] Specify how to create and name a processed version of the stdin. For example, if stdin is 'nmea' location data: -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" This option would cause the stdin to 'bklog' to be piped into the 'gpsbabel' command, interpreted as 'nmea' data, converted into 'gpx' format, and then appended to the output tar file as a file with a '.gpx' extension. This option may be specified multiple times in order to output results of multiple different processing methods. -l, --label [ string ] Specify a label to be included in all output file names. Ex: 'location' if stdin is location data. -w, --store-raw [ file extension ] Specify file extension of file within output tar that contains raw stdin data. The default behavior is to always save raw stdin data in a '.stdin' file. Example usage when 'bklog' receives 'nmea' data from 'gpspipe -r': -w ".nmea" Stdin data is saved in a '.nmea' file within the output tar. -W, --no-store-raw Do not store raw stdin in output tar. -c, --compress Compress output with gzip (before encryption if enabled). -z, --time-zone Specify time zone. (ex: "America/New_York") -t, --temp-dir [path dir] Specify parent directory for temporary working directory. Default: "/dev/shm" -R, --recipient-dir [path dir] Specify directory containing files whose first lines are to be interpreted as pubkey strings (see '-r' option). Only one directory may be specified. -b, --buffer-ttl [integer] Specify custom buffer period in seconds (default: 300 seconds) -B, --script-ttl [time element string] Specify custom script time-to-live in seconds (default: "day") Valid values: "day", "hour" EXAMPLE: (bash script lines) $ gpspipe -r | /bin/bash bklog -v -e -c -z "UTC" -t "/dev/shm" \ -r age1mrmfnwhtlprn4jquex0ukmwcm7y2nxlphuzgsgv8ew2k9mewy3rs8u7su5 \ -r age1ala848kqrvxc88rzaauc6vc5v0fqrvef9dxyk79m0vjea3hagclswu0lgq \ -R ~/.config/bklog/recipients -w ".nmea" -b 300 -B "day" \ -o ~/Sync/Logs -l "location" \ -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" \ -p "gpsbabel -i nmea -f - -o kml -F - " ".kml" EOF } # Display information on how to use this script. main() { # Desc: Main function # Usage: main "$@" # Inputs: many # Outputs: file (pathout_tar) # Depends: many local fn # Debug:Get function name fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Started function main()."; # Process arguments processArguments "$@"; ## Determine working directory magicInitWorkingDir; # Sets dir_tmp from argTempDirPriority ## Set output encryption and compression option strings ### React to "-e", "-r", and "-R" (encryption recipients) options magicParseRecipients; # Update cmd_encrypt, cmd_encrypt_suffix ### React to "-c" ("compression") option magicParseCompressionArg; # Updates cmd_compress[_suffix] ## React to "-b" and "-B" (custom buffer and script TTL) options magicParseCustomTTL; # Sets custom scriptTTL_TE and/or bufferTTL if specified ## React to "-p" (user-supplied process command and file extension strings) options magicParseProcessStrings; # Sets arrays: procStrings, procFileExts ## React to "-l" (output file label) option magicParseLabel; # sets label (ex: "_location") # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; ## Adjust SECONDS so buffer rounds align with time elements ### Advance SECONDS the remainder seconds for dividend timeUntilNextDay, divisor bufferTTL if [[ "$(timeUntilNextDay)" -gt "$bufferTTL" ]]; then vbm "DEBUG :$fn:SECONDS currently :$SECONDS"; SECONDS="$(( bufferTTL - ($(timeUntilNextDay) % bufferTTL) ))" && \ vbm "DEBUG :$fn:SECONDS advanced to:$SECONDS"; vbm "DEBUG :$fn:current time:$(date --iso-8601=seconds)"; fi; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) magicInitCheckTar; ## Append VERSION file to tar magicWriteVersion; # Check vital apps, files, dirs if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then yell "ERROR:$fn:Critical components missing."; displayMissing; yell "Exiting."; exit 1; fi # MAIN LOOP: Run until script TTL seconds pass bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do vbm "STATUS:$fn:Starting buffer round:$bufferRound"; bufferTOD="$(( (1+bufferRound)*bufferTTL ))" && vbm "DEBUG :$fn:bufferTOD:$bufferTOD"; # Set buffer round time-of-death # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); done; # Mark time for buffer timeBufferEndEpoch="$timeBufferStartEpoch"; timeBufferStartEpoch="$(date +%s)" && vbm "DEBUG :$fn:timeBufferStartEpoch"; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:$fn:dir_tmp existence failure:$dir_tmp"; try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir recreated dir_tmp:$dir_tmp"; fi # Update cmd_encrypt, cmd_encrypt_suffix magicParseRecipients; # Export buffer to asynchronous processing. magicProcessWriteBuffer & unset buffer; # Clear buffer array for next bufferRound # Increment buffer round ((bufferRound++)); done; # Cleanup ## Remove dir_tmp try rm -r "$dir_tmp" && vbm "STATUS:$fn:Removed dir_tmp:$dir_tmp"; vbm "STATUS:$fn:Finished function main()."; } # Main function #===END Declare local script functions=== #==END Define script parameters== #==BEGIN Perform work and exit== main "$@" # Run main function. exit 0; #==END Perform work and exit== # Author: Steven Baltakatei Sandoval; # License: GPLv3+