#!/bin/bash # Desc: Compresses, encrypts, and writes stdin every 5 seconds #==BEGIN Define script parameters== #===BEGIN Initialize variables=== # Logging Behavior parameters bufferTTL="300"; # Time-to-live (seconds) for each buffer round scriptTTL_TE="day"; # Time element at the end of which script terminates dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. scriptVersion="0.1.8"; # Define version of script. scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case user apps saved there ageVersion="1.0.0-beta2"; # Define version of age (encryption program) ageURL="https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2"; # Define website hosting age. # Arrays declare -a buffer # array for storing while read buffer declare -a argRecPubKeys # array for processArguments function declare -a recPubKeysValid # array for storing both '-r' and '-R' recipient pubkeys declare -a recPubKeysValidStatic # for storing '-r' recipient pubkeys declare -a argProcStrings argProcFileExts # for storing buffer processing strings (ex: "gpsbabel -i nmea -f - -o gpx -F - ") declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status declare -a procStrings procFileExts # Arrays for storing processing commands and resulting output file extensions # Variables optionVerbose=""; optionEncrypt=""; dirOut=""; optionEncrypt=""; dir_tmp=""; cmd_compress="";cmd_compress_suffix=""; cmd_encrypt=""; cmd_encrypt_suffix=""; #===END Initialize variables=== #===BEGIN Declare local script functions=== yell() { echo "$0: $*" >&2; } #o Yell, Die, Try Three-Fingered Claw technique die() { yell "$*"; exit 111; } #o Ref/Attrib: https://stackoverflow.com/a/25515370 try() { "$@" || die "cannot $*"; } #o processArguments() { while [ ! $# -eq 0 ]; do # While number of arguments ($#) is not (!) equal to (-eq) zero (0). case "$1" in -v | --verbose) optionVerbose="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. -h | --help) showUsage; exit 1;; # Display usage. --version) showVersion; exit 1;; # Show version -o | --output) if [ -d "$2" ]; then dirOut="$2"; vbm "DEBUG:dirOut:$dirOut"; shift; fi ;; # Define output directory. -e | --encrypt) optionEncrypt="true"; vbm "DEBUG:Encrypted output mode enabled.";; # Enable encryption -r | --recipient) optionRecipients="true"; argRecPubKeys+=("$2"); vbm "STATUS:pubkey added:""$2"; shift;; # Add recipients -c | --compress) optionCompress="true"; vbm "DEBUG:Compressed output mode enabled.";; # Enable compression -z | --time-zone) try setTimeZoneEV "$2"; shift;; # Set timestamp timezone -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone -R | --recipient-dir) optionRecipients="true"; optionRecDir="true" && argRecDir="$2"; shift;; # Add recipient watch dir -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds) -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day") -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;; -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG:Custom label received:$argLabel"; shift;; -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG:Raw stdin file extension received:$argRawFileExt"; shift;; -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG:Option selected to not store raw stdin data."; shift;; *) yell "ERROR: Unrecognized argument: $1"; yell "STATUS:All arguments:$*"; exit 1;; # Handle unrecognized options. esac shift done } # Argument Processing vbm() { # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true". # Usage: vbm "DEBUG:verbose message here" # Version 0.1.2 # Input: arg1: string # vars: optionVerbose # Output: stderr # Depends: bash 5.0.3, echo 8.30, date 8.30 if [ "$optionVerbose" = "true" ]; then functionTime=$(date --iso-8601=ns); # Save current time in nano seconds. echo "[$functionTime] ""$*" 1>&2; # Display argument text. fi # End function return 0; # Function finished. } # Displays message if optionVerbose true checkapp() { # Desc: If arg is a command, save result in assoc array 'appRollCall' # Usage: checkapp arg1 arg2 arg3 ... # Version: 0.1.1 # Input: global assoc. array 'appRollCall' # Output: adds/updates key(value) to global assoc array 'appRollCall' # Depends: bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command appRollCall[$arg]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi; else appRollCall[$arg]="false"; returnState="false"; fi; done; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi; } # Check that app exists checkfile() { # Desc: If arg is a file path, save result in assoc array 'fileRollCall' # Usage: checkfile arg1 arg2 arg3 ... # Version: 0.1.1 # Input: global assoc. array 'fileRollCall' # Output: adds/updates key(value) to global assoc array 'fileRollCall'; # Output: returns 0 if app found, 1 otherwise # Depends: bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if [ -f "$arg" ]; then fileRollCall["$arg"]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi; else fileRollCall["$arg"]="false"; returnState="false"; fi; done; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi; } # Check that file exists checkdir() { # Desc: If arg is a dir path, save result in assoc array 'dirRollCall' # Usage: checkdir arg1 arg2 arg3 ... # Version 0.1.1 # Input: global assoc. array 'dirRollCall' # Output: adds/updates key(value) to global assoc array 'dirRollCall'; # Output: returns 0 if app found, 1 otherwise # Depends: Bash 5.0.3 local returnState #===Process Args=== for arg in "$@"; do if [ -d "$arg" ]; then dirRollCall["$arg"]="true"; if ! [ "$returnState" = "false" ]; then returnState="true"; fi else dirRollCall["$arg"]="false"; returnState="false"; fi done #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; else return 1; fi } # Check that dir exists displayMissing() { # Desc: Displays missing apps, files, and dirs # Usage: displayMissing # Version 0.1.1 # Input: associative arrays: appRollCall, fileRollCall, dirRollCall # Output: stderr: messages indicating missing apps, file, or dirs # Depends: bash 5, checkAppFileDir() local missingApps value appMissing missingFiles fileMissing local missingDirs dirMissing #==BEGIN Display errors== #===BEGIN Display Missing Apps=== missingApps="Missing apps :"; #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done for key in "${!appRollCall[@]}"; do value="${appRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG:Missing apps: $key => $value"; missingApps="$missingApps""$key "; appMissing="true"; fi; done; if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing. echo "$missingApps" 1>&2; fi; unset value; #===END Display Missing Apps=== #===BEGIN Display Missing Files=== missingFiles="Missing files:"; #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done for key in "${!fileRollCall[@]}"; do value="${fileRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG:Missing files: $key => $value"; missingFiles="$missingFiles""$key "; fileMissing="true"; fi; done; if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing. echo "$missingFiles" 1>&2; fi; unset value; #===END Display Missing Files=== #===BEGIN Display Missing Directories=== missingDirs="Missing dirs:"; #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done for key in "${!dirRollCall[@]}"; do value="${dirRollCall[$key]}"; if [ "$value" = "false" ]; then #echo "DEBUG:Missing dirs: $key => $value"; missingDirs="$missingDirs""$key "; dirMissing="true"; fi; done; if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing. echo "$missingDirs" 1>&2; fi; unset value; #===END Display Missing Directories=== #==END Display errors== } # Display missing apps, files, dirs appendFileTar(){ # Desc: Appends [processed] file to tar # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd]) # Version: 2.0.1 # Input: arg1: path of file to be (processed and) written # arg2: name to use for file inserted into tar # arg3: tar archive path (must exist first) # arg4: temporary working dir # arg5: (optional) command string to process file (ex: "gpsbabel -i nmea -f - -o kml -F - ") # Output: file written to disk # Example: decrypt multiple large files in parallel # appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & # appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & # appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & # Depends: bash 5.0.3, tar 1.30, cat 8.30, yell() local fn fileName tarPath tmpDir # Save function name fn="${FUNCNAME[0]}"; #yell "DEBUG:STATUS:$fn:Started appendFileTar()." # Set file name if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi # Check tar path is a file if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file:$3"; exit 1; fi # Check temp dir arg if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi # Set command strings if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string # Input command string cmd0="cat \"\$1\""; # Write to temporary working dir eval "$cmd0 | $cmd1" > "$tmpDir"/"$fileName"; # Append to tar try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName"; #yell "DEBUG:STATUS:$fn:Finished appendFileTar()." } # Append [processed] file to Tar archive checkAgePubkey() { # Desc: Checks if string is an age-compatible pubkey # Usage: checkAgePubkey [str pubkey] # Version: 0.1.2 # Input: arg1: string # Output: return code 0: string is age-compatible pubkey # return code 1: string is NOT an age-compatible pubkey # age stderr (ex: there is stderr if invalid string provided) # Depends: age (v0.1.0-beta2; https://github.com/FiloSottile/age/releases/tag/v1.0.0-beta2 ) argPubkey="$1"; if echo "test" | age -a -r "$argPubkey" 1>/dev/null; then return 0; else return 1; fi; } # Check age pubkey dateShort(){ # Desc: Date without separators (YYYYmmdd) # Usage: dateShort ([str date]) # Version: 1.1.2 # Input: arg1: 'date'-parsable timestamp string (optional) # Output: stdout: date (ISO-8601, no separators) # Depends: bash 5.0.3, date 8.30, yell() local argTime timeCurrent timeInput dateCurrentShort argTime="$1"; # Get Current Time timeCurrent="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. # Decide to parse current or supplied date ## Check if time argument empty if [[ -z "$argTime" ]]; then ## T: Time argument empty, use current time timeInput="$timeCurrent"; else ## F: Time argument exists, validate time if date --date="$argTime" 1>/dev/null 2>&1; then ### T: Time argument is valid; use it timeInput="$argTime"; else ### F: Time argument not valid; exit yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; fi; fi; # Construct and deliver separator-les date string dateCurrentShort="$(date -d "$timeInput" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day. echo "$dateCurrentShort"; } # Get YYYYmmdd setTimeZoneEV(){ # Desc: Set time zone environment variable TZ # Usage: setTimeZoneEV arg1 # Version 0.1.2 # Input: arg1: 'date'-compatible timezone string (ex: "America/New_York") # TZDIR env var (optional; default: "/usr/share/zoneinfo") # Output: exports TZ # exit code 0 on success # exit code 1 on incorrect number of arguments # exit code 2 if unable to validate arg1 # Depends: yell(), printenv 8.30, bash 5.0.3 # Tested on: Debian 10 local tzDir returnState argTimeZone argTimeZone="$1" if ! [[ $# -eq 1 ]]; then yell "ERROR:Invalid argument count."; return 1; fi # Read TZDIR env var if available if printenv TZDIR 1>/dev/null 2>&1; then tzDir="$(printenv TZDIR)"; else tzDir="/usr/share/zoneinfo"; fi # Validate TZ string if ! [[ -f "$tzDir"/"$argTimeZone" ]]; then yell "ERROR:Invalid time zone argument."; return 2; else # Export ARG1 as TZ environment variable TZ="$argTimeZone" && export TZ && returnState="true"; fi # Determine function return code if [ "$returnState" = "true" ]; then return 0; fi } # Exports TZ environment variable showUsage() { cat <<'EOF' USAGE: cmd | bklog [ options ] OPTIONS: -h, --help Display help information. --version Display script version. -v, --verbose Display debugging info. -e, --encrypt Encrypt output. -r, --recipient [ string pubkey ] Specify recipient. May be age or ssh pubkey. May be specified multiple times for multiple pubkeys. See https://github.com/FiloSottile/age -o, --output [ path dir ] Specify output directory to save logs. This option is required to save log data. -p, --process-string [ filter command ] [ output file extension] Specify how to create and name a processed version of the stdin. For example, if stdin is 'nmea' location data: -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" This option would cause the stdin to 'bklog' to be piped into the 'gpsbabel' command, interpreted as 'nmea' data, converted into 'gpx' format, and then appended to the output tar file as a file with a '.gpx' extension. This option may be specified multiple times in order to output results of multiple different processing methods. -l, --label [ string ] Specify a label to be included in all output file names. Ex: 'location' if stdin is location data. -w, --store-raw [ file extension ] Specify file extension of file within output tar that contains raw stdin data. The default behavior is to always save raw stdin data in a '.stdin' file. Example usage when 'bklog' receives 'nmea' data from 'gpspipe -r': -w ".nmea" Stdin data is saved in a '.nmea' file within the output tar. -W, --no-store-raw Do not store raw stdin in output tar. -c, --compress Compress output with gzip (before encryption if enabled). -z, --time-zone Specify time zone. (ex: "America/New_York") -t, --temp-dir [path dir] Specify parent directory for temporary working directory. Default: "/dev/shm" -R, --recipient-dir [path dir] Specify directory containing files whose first lines are to be interpreted as pubkey strings (see '-r' option). -b, --buffer-ttl [integer] Specify custom buffer period in seconds (default: 300 seconds) -B, --script-ttl [time element string] Specify custom script time-to-live in seconds (default: "day") Valid values: "day", "hour" EXAMPLE: (bash script lines) $ gpspipe -r | /bin/bash bklog -v -e -c -z "UTC" -t "/dev/shm" \ -r age1mrmfnwhtlprn4jquex0ukmwcm7y2nxlphuzgsgv8ew2k9mewy3rs8u7su5 \ -r age1ala848kqrvxc88rzaauc6vc5v0fqrvef9dxyk79m0vjea3hagclswu0lgq \ -R ~/.config/bklog/recipients -w ".nmea" -b 300 -B "day" \ -o ~/Sync/Logs -l "location" \ -p "gpsbabel -i nmea -f - -o gpx -F - " ".gpx" \ -p "gpsbabel -i nmea -f - -o kml -F - " ".kml" EOF } # Display information on how to use this script. showVersion() { yell "$scriptVersion" } # Display script version. timeDuration(){ # Desc: Given seconds, output ISO-8601 duration string # Ref/Attrib: ISO-8601:2004(E), §4.4.4.2 Representations of time intervals by duration and context information # Note: "1 month" ("P1M") is assumed to be "30 days" (see ISO-8601:2004(E), §2.2.1.2) # Usage: timeDuration [1:seconds] ([2:precision]) # Version: 1.0.4 # Input: arg1: seconds as base 10 integer >= 0 (ex: 3601) # arg2: precision level (optional; default=2) # Output: stdout: ISO-8601 duration string (ex: "P1H1S", "P2Y10M15DT10H30M20S") # exit code 0: success # exit code 1: error_input # exit code 2: error_unknown # Example: 'timeDuration 111111 3' yields 'P1DT6H51M' # Depends: date 8, bash 5, yell, local argSeconds argPrecision precision returnState remainder local fullYears fullMonths fullDays fullHours fullMinutes fullSeconds local hasYears hasMonths hasDays hasHours hasMinutes hasSeconds local witherPrecision output local displayYears displayMonths displayDays displayHours displayMinutes displaySeconds argSeconds="$1"; # read arg1 (seconds) argPrecision="$2"; # read arg2 (precision) precision=2; # set default precision # Check that between one and two arguments is supplied if ! { [[ $# -ge 1 ]] && [[ $# -le 2 ]]; }; then yell "ERROR:Invalid number of arguments:$# . Exiting."; returnState="error_input"; fi # Check that argSeconds provided if [[ $# -ge 1 ]]; then ## Check that argSeconds is a positive integer if [[ "$argSeconds" =~ ^[[:digit:]]+$ ]]; then : else yell "ERROR:argSeconds not a digit."; returnState="error_input"; fi else yell "ERROR:No argument provided. Exiting."; exit 1; fi # Consider whether argPrecision was provided if [[ $# -eq 2 ]]; then # Check that argPrecision is a positive integer if [[ "$argPrecision" =~ ^[[:digit:]]+$ ]] && [[ "$argPrecision" -gt 0 ]]; then precision="$argPrecision"; else yell "ERROR:argPrecision not a positive integer. (is $argPrecision ). Leaving early."; returnState="error_input"; fi; else : fi; remainder="$argSeconds" ; # seconds ## Calculate full years Y, update remainder fullYears=$(( remainder / (365*24*60*60) )); remainder=$(( remainder - (fullYears*365*24*60*60) )); ## Calculate full months M, update remainder fullMonths=$(( remainder / (30*24*60*60) )); remainder=$(( remainder - (fullMonths*30*24*60*60) )); ## Calculate full days D, update remainder fullDays=$(( remainder / (24*60*60) )); remainder=$(( remainder - (fullDays*24*60*60) )); ## Calculate full hours H, update remainder fullHours=$(( remainder / (60*60) )); remainder=$(( remainder - (fullHours*60*60) )); ## Calculate full minutes M, update remainder fullMinutes=$(( remainder / (60) )); remainder=$(( remainder - (fullMinutes*60) )); ## Calculate full seconds S, update remainder fullSeconds=$(( remainder / (1) )); remainder=$(( remainder - (remainder*1) )); ## Check which fields filled if [[ $fullYears -gt 0 ]]; then hasYears="true"; else hasYears="false"; fi if [[ $fullMonths -gt 0 ]]; then hasMonths="true"; else hasMonths="false"; fi if [[ $fullDays -gt 0 ]]; then hasDays="true"; else hasDays="false"; fi if [[ $fullHours -gt 0 ]]; then hasHours="true"; else hasHours="false"; fi if [[ $fullMinutes -gt 0 ]]; then hasMinutes="true"; else hasMinutes="false"; fi if [[ $fullSeconds -gt 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi ## Determine which fields to display (see ISO-8601:2004 §4.4.3.2) witherPrecision="false" ### Years if $hasYears && [[ $precision -gt 0 ]]; then displayYears="true"; witherPrecision="true"; else displayYears="false"; fi; if $witherPrecision; then ((precision--)); fi; ### Months if $hasMonths && [[ $precision -gt 0 ]]; then displayMonths="true"; witherPrecision="true"; else displayMonths="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayMonths="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Days if $hasDays && [[ $precision -gt 0 ]]; then displayDays="true"; witherPrecision="true"; else displayDays="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayDays="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Hours if $hasHours && [[ $precision -gt 0 ]]; then displayHours="true"; witherPrecision="true"; else displayHours="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayHours="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Minutes if $hasMinutes && [[ $precision -gt 0 ]]; then displayMinutes="true"; witherPrecision="true"; else displayMinutes="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displayMinutes="true"; fi; if $witherPrecision; then ((precision--)); fi; ### Seconds if $hasSeconds && [[ $precision -gt 0 ]]; then displaySeconds="true"; witherPrecision="true"; else displaySeconds="false"; fi; if $witherPrecision && [[ $precision -gt 0 ]]; then displaySeconds="true"; fi; if $witherPrecision; then ((precision--)); fi; ## Determine whether or not the "T" separator is needed to separate date and time elements if ( $displayHours || $displayMinutes || $displaySeconds); then displayDateTime="true"; else displayDateTime="false"; fi ## Construct duration output string output="P" if $displayYears; then output=$output$fullYears"Y"; fi if $displayMonths; then output=$output$fullMonths"M"; fi if $displayDays; then output=$output$fullDays"D"; fi if $displayDateTime; then output=$output"T"; fi if $displayHours; then output=$output$fullHours"H"; fi if $displayMinutes; then output=$output$fullMinutes"M"; fi if $displaySeconds; then output=$output$fullSeconds"S"; fi ## Output duration string to stdout echo "$output" && returnState="true"; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; elif [ "$returnState" = "error_input" ]; then yell "ERROR:input"; return 1; else yell "ERROR:Unknown"; return 2; fi } # Get duration (ex: PT10M4S ) validateInput() { # Desc: Validates Input # Usage: validateInput [str input] [str input type] # Version: 0.3.1 # Input: arg1: string to validate # arg2: string specifying input type (ex:"ssh_pubkey") # Output: return code 0: if input string matched specified string type # Depends: bash 5, yell() local fn argInput argType # Save function name fn="${FUNCNAME[0]}"; # Process arguments argInput="$1"; argType="$2"; if [[ $# -gt 2 ]]; then yell "ERROR:$0:$fn:Too many arguments."; exit 1; fi; # Check for blank if [[ -z "$argInput" ]]; then return 1; fi # Define input types ## ssh_pubkey ### Check for alnum/dash base64 (ex: "ssh-rsa AAAAB3NzaC1yc2EAAA") if [[ "$argType" = "ssh_pubkey" ]]; then if [[ "$argInput" =~ ^[[:alnum:]-]*[\ ]*[[:alnum:]+/=]*$ ]]; then return 0; fi; fi; ## age_pubkey ### Check for age1[:bech32:] if [[ "$argType" = "age_pubkey" ]]; then if [[ "$argInput" =~ ^age1[qpzry9x8gf2tvdw0s3jn54khce6mua7l]*$ ]]; then return 0; fi; fi ## integer if [[ "$argType" = "integer" ]]; then if [[ "$argInput" =~ ^[[:digit:]]*$ ]]; then return 0; fi; fi; ## time element (year, month, week, day, hour, minute, second) if [[ "$argType" = "time_element" ]]; then if [[ "$argInput" = "year" ]] || \ [[ "$argInput" = "month" ]] || \ [[ "$argInput" = "week" ]] || \ [[ "$argInput" = "day" ]] || \ [[ "$argInput" = "hour" ]] || \ [[ "$argInput" = "minute" ]] || \ [[ "$argInput" = "second" ]]; then return 0; fi; fi; # Return error if no condition matched. return 1; } # Validates strings magicInitWorkingDir() { # Desc: Determine temporary working directory from defaults or user input # Usage: magicInitWorkingDir # Input: vars: optionTmpDir, argTempDirPriority, dirTmpDefault # Input: vars: scriptTimeStart # Output: vars: dir_tmp # Depends: bash 5.0.3, processArguments(), vbm(), yell() # Parse '-t' option (user-specified temporary working dir) ## Set dir_tmp_parent to user-specified value if specified local dir_tmp_parent if [[ "$optionTmpDir" = "true" ]]; then if [[ -d "$argTempDirPriority" ]]; then dir_tmp_parent="$argTempDirPriority"; else yell "WARNING:Specified temporary working directory not valid:$argTempDirPriority"; exit 1; # Exit since user requires a specific temp dir and it is not available. fi; else ## Set dir_tmp_parent to default or fallback otherwise if [[ -d "$dirTmpDefault" ]]; then dir_tmp_parent="$dirTmpDefault"; elif [[ -d /tmp ]]; then yell "WARNING:$dirTmpDefault not available. Falling back to /tmp ."; dir_tmp_parent="/tmp"; else yell "ERROR:No valid working directory available. Exiting."; exit 1; fi; fi; ## Set dir_tmp using dir_tmp_parent and nonce (scriptTimeStart) dir_tmp="$dir_tmp_parent"/"$scriptTimeStart""..bkgpslog" && vbm "DEBUG:Set dir_tmp to:$dir_tmp"; # Note: removed at end of main(). } # Sets working dir magicInitCheckTar() { # Desc: Initializes or checks output tar # input: vars: dirOut, bufferTTL, cmd_encrypt_suffix, cmd_compress_suffix # input: vars: scriptHostname # output: vars: pathout_tar # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion() # Form pathout_tar pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \ vbm "STATUS:Set pathout_tar to:$pathout_tar"; # Validate pathout_tar as tar. checkMakeTar "$pathout_tar"; ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2) vbm "exit status before magicWriteVersion:$?" if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi } # Initialize tar, set pathout_tar magicParseCompressionArg() { # Desc: Parses compression arguments specified by '-c' option # Input: vars: optionCompress # Output: cmd_compress, cmd_compress_suffix # Depends: processArguments(), vbm(), checkapp(), gzip 1.9 if [[ "$optionCompress" = "true" ]]; then # Check if compression option active if checkapp gzip; then # Check if gzip available cmd_compress="gzip " && vbm "cmd_compress:$cmd_compress"; cmd_compress_suffix=".gz" && vbm "cmd_compress_suffix:$cmd_compress_suffix"; else yell "ERROR:Compression enabled but \"gzip\" not found. Exiting."; exit 1; fi else cmd_compress="tee /dev/null " && vbm "cmd_compress:$cmd_compress"; cmd_compress_suffix="" && vbm "cmd_compress_suffix:$cmd_compress_suffix"; vbm "DEBUG:Compression not enabled."; fi } # Form compression cmd string and filename suffix magicParseCustomTTL() { # Desc: Set user-specified TTLs for buffer and script # Usage: magicParseCustomTTL # Input: vars: argCustomBufferTTL (integer), argCustomScriptTTL_TE (string) # Input: vars: optionCustomBufferTTL, optionCustomScriptTTL_TE # Input: vars: bufferTTL (integer), scriptTTL_TE (string) # Output: bufferTTL (integer), scriptTTL_TE (string) # Depends: Bash 5.0.3, yell(), vbm(), validateInput(), showUsage() # React to '-b, --buffer-ttl' option if [[ "$optionCustomBufferTTL" = "true" ]]; then ## T: Check if argCustomBufferTTL is an integer if validateInput "$argCustomBufferTTL" "integer"; then ### T: argCustomBufferTTL is an integer bufferTTL="$argCustomBufferTTL" && vbm "Custom bufferTTL from -b:$bufferTTL"; else ### F: argcustomBufferTTL is not an integer yell "ERROR:Invalid integer argument for custom buffer time-to-live."; showUsage; exit 1; fi; ## F: do not change bufferTTL fi; # React to '-B, --script-ttl' option if [[ "$optionCustomScriptTTL_TE" = "true" ]]; then ## T: Check if argCustomScriptTTL is a time element (ex: "day", "hour") if validateInput "$argCustomScriptTTL_TE" "time_element"; then ### T: argCustomScriptTTL is a time element scriptTTL_TE="$argCustomScriptTTL_TE" && vbm "Custom scriptTTL_TE from -B:$scriptTTL_TE"; else ### F: argcustomScriptTTL is not a time element yell "ERROR:Invalid time element argument for custom script time-to-live."; showUsage; exit 1; fi; ## F: do not change scriptTTL_TE fi; } # Sets custom script or buffer TTL if specified magicParseLabel() { # Desc: Parses -l option to set label # In : optionLabel, argLabel # Out: vars: label # Depends: Bash 5.0.3, vbm(), yell() vbm "STATUS:Started magicParseLabel() function."; # Do nothing if optionLabel not set to true. if [[ ! "$optionLabel" = "true" ]]; then vbm "STATUS:optionlabel not set to 'true'. Returning early."; return; fi; # Set label if optionLabel is true if [[ "$optionLabel" = "true" ]]; then label="_""$argLabel"; vbm "STATUS:Set label:$label"; fi; vbm "STATUS:Finished magicParseLabel() function."; } # Set label used in output file name magicParseProcessStrings() { # Desc: Processes user-supplied process strings into process commands for appendFileTar(). # Usage: magicParseProcessStrings # In : vars: optionProcString optionNoStoreRaw optionStoreRaw argRawFileExt # arry: argProcStrings, argProcFileExts # Out: arry: procStrings, procFileExts # Depends Bash 5.0.3, yell(), vbm() local rawFileExt vbm "STATUS:Starting magicParseProcessStrings() function."; vbm "var:optionProcString:$optionProcString"; vbm "var:optionNoStoreRaw:$optionNoStoreRaw"; vbm "var:optionStoreRaw:$optionStoreRaw"; vbm "var:argRawFileExt:$argRawFileExt"; vbm "ary:argProcStrings:${argProcStrings[*]}"; vbm "ary:argProcFileExts:${argProcFileExts[*]}" # Validate input ## Validate argRawFileExt if [[ "$argRawFileExt" =~ ^[.][[:alnum:]]*$ ]]; then rawFileExt="$argRawFileExt"; fi; # Add default stdin output file entries for procStrings, procFileExts ## Check if user specified that no raw stdin be saved. if [[ ! "$optionNoStoreRaw" = "true" ]]; then ### T: --no-store-raw not set. Store raw. Append procStrings with cat. #### Append procStrings array procStrings+=("cat "); #### Check if --store-raw set. if [[ "$optionStoreRaw" = "true" ]]; then ##### T: --store-raw set. Append procFileExts with user-specified file ext procFileExts+=("$rawFileExt"); else ##### F: --store-raw not set. Append procFileExts with default ".stdin" file ext ###### Append procFileExts array procFileExts+=(".stdin"); fi; else ### F: --no-store-raw set. Do not store raw. #### Do not append procStrings or procFileExts arrays. : fi; # Do nothing more if optionProcString not set to true. if [[ ! "$optionProcString" = "true" ]]; then vbm "STATUS:optionProcString not set to 'true'. Returning early."; return; fi; # Validate input array indices ## Make sure that argProcStrings and argProcFileExts have same index counts if ! [[ "${#argProcStrings[@]}" -eq "${#argProcFileExts[@]}" ]]; then yell "ERROR:Mismatch in number of elements in arrays argProcStrings and argProcFileExts:${#argProcStrings[@]} DNE ${#argProcFileExts[@]}"; yell "argProcStrings:${argProcStrings[*]}"; yell "argProcFileExts:${argProcFileExts[*]}"; exit 1; fi; ## Make sure that no array elements are blank for element in "${argProcStrings[@]}"; do if [[ -z "$element" ]]; then yell "ERROR:Empty process string specified. Exiting."; exit 1; fi; done for element in "${argProcFileExts[@]}"; do if [[ -z "$element" ]]; then yell "ERROR:Empty output file extension specified. Exiting."; exit 1; fi; done ## Make sure that no process string starts with '-' (ex: if only one arg supplied after '-p' option) for element in "${argProcStrings[@]}"; do if [[ "$element" =~ ^[-][[:print:]]*$ ]] && [[ ! "$element" =~ ^[[:print:]]*$ ]]; then yell "ERROR:Illegal character '-' at start of process string element:\"$element\""; exit 1; fi; done; vbm "STATUS:Quick check shows argProcStrings and argProcFileExts appear to have valid contents."; procStrings=("${argProcStrings[@]}"); # Export process command strings procFileExts=("${argProcFileExts[@]}"); # Export process command strings vbm "STATUS:Finished magicParseProcessStrings() function."; } # Validate and save process strings and file extensions to arrays procStrings, procFileExts magicParseRecipientArgs() { # Desc: Parses recipient arguments specified by '-r' option # Input: vars: optionEncrypt, optionRecipients # arry: argRecPubKeys from processArguments() # Output: vars: cmd_encrypt, cmd_encrypt_suffix # arry: recPubKeysValid, recPubKeysValidStatic # Depends: processArguments(), yell(), vbm(), checkapp(), checkAgePubkey(), validateInput() local recipients # Check if encryption option active. if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then if checkapp age; then # Check that age is available. for pubkey in "${argRecPubKeys[@]}"; do # Validate recipient pubkey strings by forming test message vbm "DEBUG:Testing pubkey string:$pubkey"; if checkAgePubkey "$pubkey" && \ ( validateInput "$pubkey" "ssh_pubkey" || validateInput "$pubkey" "age_pubkey"); then #### Form age recipient string recipients="$recipients""-r '$pubkey' "; vbm "STATUS:Added pubkey for forming age recipient string:""$pubkey"; vbm "DEBUG:recipients:""$recipients"; #### Add validated pubkey to recPubKeysValid array recPubKeysValid+=("$pubkey") && vbm "DEBUG:recPubkeysValid:pubkey added:$pubkey"; else yell "ERROR:Exit code ""$?"". Invalid recipient pubkey string. Exiting."; exit 1; fi; done vbm "DEBUG:Finished processing argRecPubKeys array"; vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[*]}"; recPubKeysValidStatic=("${recPubKeysValid[@]}"); # Save static image of pubkeys validated by this function ## Form age command string cmd_encrypt="age ""$recipients " && vbm "cmd_encrypt:$cmd_encrypt"; cmd_encrypt_suffix=".age" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix"; else yell "ERROR:Encryption enabled but \"age\" not found. Exiting."; exit 1; fi; else cmd_encrypt="tee /dev/null " && vbm "cmd_encrypt:$cmd_encrypt"; cmd_encrypt_suffix="" && vbm "cmd_encrypt_suffix:$cmd_encrypt_suffix"; vbm "DEBUG:Encryption not enabled." fi; # Catch case if '-e' is set but '-r' or '-R' is not if [[ "$optionEncrypt" = "true" ]] && [[ ! "$optionRecipients" = "true" ]]; then yell "ERROR:\\'-e\\' set but no \\'-r\\' or \\'-R\\' set."; exit 1; fi; # Catch case if '-r' or '-R' set but '-e' is not if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecipients" = "true" ]]; then yell "ERROR:\\'-r\\' or \\'-R\\' set but \\'-e\\' is not set."; exit 1; fi; } # Populate recPubKeysValid with argRecPubKeys; form encryption cmd string and filename suffix magicParseRecipientDir() { # Desc: Updates recPubKeysValid with pubkeys in dir specified by '-R' option ("recipient directory") # Inputs: vars: optionEncrypt, optionRecDir, argRecDir, # arry: recPubKeysValid # Outputs: arry: recPubKeysValid # Depends: processArguments(), yell(), vbm(), validateInput(), checkAgePubkey() local recipientDir recFileLine updateRecipients declare -a candRecPubKeysValid # Check that '-e' and '-R' set if [[ "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then ### Check that argRecDir is a directory. if [[ -d "$argRecDir" ]]; then recipientDir="$argRecDir" && vbm "STATUS:Recipient watch directory detected:\"$recipientDir\""; #### Initialize variable indicating outcome of pubkey review unset updateRecipients #### Add existing recipients candRecPubKeysValid=("${recPubKeysValidStatic[@]}"); #### Parse files in recipientDir for file in "$recipientDir"/*; do ##### Read first line of each file recFileLine="$(head -n1 "$file")" && vbm "STATUS:Checking if pubkey:\"$recFileLine\""; ##### check if first line is a valid pubkey if checkAgePubkey "$recFileLine" && \ ( validateInput "$recFileLine" "ssh_pubkey" || validateInput "$recFileLine" "age_pubkey"); then ###### T: add candidate pubkey to candRecPubKeysValid candRecPubKeysValid+=("$recFileLine") && vbm "STATUS:RecDir pubkey is valid pubkey:\"$recFileLine\""; else ###### F: throw warning; yell "ERROR:Invalid recipient file detected. Not modifying recipient list." updateRecipients="false"; fi; done #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected if ! [[ "$updateRecipients" = "false" ]]; then recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\""; fi; else yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1; fi; fi; # Handle case if '-R' set but '-e' not set if [[ ! "$optionEncrypt" = "true" ]] && [[ "$optionRecDir" = "true" ]]; then yell "ERROR: \\'-R\\' is set but \\'-e\\' is not set."; fi; } # Update recPubKeysValid with argRecDir magicSetScriptTTL() { #Desc: Sets script_TTL seconds from provided time_element string argument #Usage: magicSetScriptTTL [str time_element] #Input: arg1: string (Ex: scriptTTL_TE; "day" or "hour") #Output: var: scriptTTL (integer seconds) #Depends: timeUntilNextHour, timeUntilNextDay local argTimeElement argTimeElement="$1"; if [[ "$argTimeElement" = "day" ]]; then # Set script lifespan to end at start of next day if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. else yell "ERROR: timeUntilNextDay exit code $?"; exit 1; fi; fi; elif [[ "$argTimeElement" = "hour" ]]; then # Set script lifespan to end at start of next hour if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. else yell "ERROR: timeUntilNextHour exit code $?"; exit 1; fi; fi; else yell "ERROR:Invalid argument for setScriptTTL function:$argTimeElement"; exit 1; fi; } # Set scriptTTL in seconds until next (day|hour). magicWriteVersion() { # Desc: Appends time-stamped VERSION to pathout_tar # Usage: magicWriteVersion # Input: vars: pathout_tar, dir_tmp # Input: vars: scriptVersion, scriptURL, ageVersion, ageURL, scriptHostname # Input: array: recPubKeysValid # Output: appends tar (pathout_tar) # Depends: bash 5.0.3, dateTimeShort(), appendArgTar() local fileoutVersion contentVersion pubKeyIndex pubKeyIndex # Set VERSION file name fileoutVersion="$(dateTimeShort)..VERSION"; # Gather VERSION data in contentVersion contentVersion="scriptVersion=$scriptVersion"; #contentVersion="$contentVersion""\\n"; contentVersion="$contentVersion""\\n""scriptName=$scriptName"; contentVersion="$contentVersion""\\n""scriptURL=$scriptURL"; contentVersion="$contentVersion""\\n""ageVersion=$ageVersion"; contentVersion="$contentVersion""\\n""ageURL=$ageURL"; contentVersion="$contentVersion""\\n""date=$(date --iso-8601=seconds)"; contentVersion="$contentVersion""\\n""hostname=$scriptHostname"; ## Add list of recipient pubkeys for pubkey in "${recPubKeysValid[@]}"; do ((pubKeyIndex++)) contentVersion="$contentVersion""\\n""PUBKEY_$pubKeyIndex=$pubkey"; done ## Process newline escapes contentVersion="$(echo -e "$contentVersion")" # Write contentVersion as file fileoutVersion and write-append to pathout_tar appendArgTar "$contentVersion" "$fileoutVersion" "$pathout_tar" "$dir_tmp"; } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings local fn timeBufferStartLong timeBufferStart fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 vbm "DEBUG:STATUS:$fn:Started magicProcessWriteBuffer()."; # Debug:Get function name fn="${FUNCNAME[0]}"; # Determine file paths (time is start of buffer period) ## Calculate start time timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \ vbm "timeBufferStartLong:$timeBufferStartLong"; timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) ## Set common basename fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \ vbm "STATUS:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts for fileExt in "${procFileExts[@]}"; do fileouts+=("$fileoutBasename""$fileExt""$cmd_compress_suffix""$cmd_encrypt_suffix") && \ vbm "STATUS:Added $fileExt to fileouts:${fileouts[*]}"; done; for fileName in "${fileouts[@]}"; do pathouts+=("$dir_tmp"/"$fileName") && \ vbm "STATUS:Added $fileName to pathouts:${pathouts[*]}"; done; ## Update pathout_tar magicInitCheckTar; # Process and write buffers to dir_tmp ## Prepare command strings writeCmd1="printf \"%s\\\\n\" \"\${buffer[@]}\""; # printf "%s\\n" "${buffer[@]}" #writeCmd2="" # NOTE: Specified by parsing array procStrings writeCmd3="$cmd_compress"; writeCmd4="$cmd_encrypt"; ## Process buffer and write to dir_tmp for index in "${!pathouts[@]}"; do writeCmd2="${procStrings[$index]}" eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" >> "${pathouts[$index]}"; done; # Append dir_tmp files to pathout_tar wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) for index in "${!pathouts[@]}"; do appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp"; done; # Remove secured chunks from dir_tmp for path in "${pathouts[@]}"; do rm "$path"; done; vbm "DEBUG:STATUS:$fn:Finished magicProcessWriteBuffer()."; } # Process and Write buffer main() { # Process arguments processArguments "$@"; ## Determine working directory magicInitWorkingDir; # Sets dir_tmp from argTempDirPriority ## Set output encryption and compression option strings ### React to "-e" and "-r" ("encryption recipients") options magicParseRecipientArgs; # Updates recPubKeysValid, cmd_encrypt[_suffix] from argRecPubKeys ### React to "-R" ("recipient directory") option magicParseRecipientDir; # Updates recPubKeysValid ### React to "-c" ("compression") option magicParseCompressionArg; # Updates cmd_compress[_suffix] ## React to "-b" and "-B" (custom buffer and script TTL) options magicParseCustomTTL; # Sets custom scriptTTL_TE and/or bufferTTL if specified ## React to "-p" (user-supplied process command and file extension strings) options magicParseProcessStrings; # Sets arrays: procStrings, procFileExts ## React to "-l" (output file label) option magicParseLabel; # sets label (ex: "_location") ## React to "-w" (how to name raw stdin file) option magicParseStoreRaw; # sets raw_suffix # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; ## File name substring (ISO-8601 duration from bufferTTL) bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG:bufferTTL_STR:$bufferTTL_STR"; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) magicInitCheckTar; # Check vital apps, files, dirs if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then yell "ERROR:Critical components missing."; displayMissing; yell "Exiting."; exit 1; fi # MAIN LOOP: Run until script TTL seconds pass bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death lineCount=0; # Debug counter # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); echo "DEBUG:Processing line:$lineCount"; echo "DEBUG:Current line :$line"; echo "DEBUG:buf elem count :${#buffer[@]}"; ((lineCount++)); done; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:dir_tmp existence failure:$dir_tmp"; try mkdir "$dir_tmp" && vbm "DEBUG:Working dir recreated dir_tmp:$dir_tmp"; fi # Update encryption recipient array magicParseRecipientDir; # Update recPubKeysValid with argRecDir # Export buffer to asynchronous processing. magicProcessWriteBuffer & unset buffer; # Clear buffer array for next bufferRound # Increment buffer round ((bufferRound++)); done; # Cleanup ## Remove dir_tmp try rm -r "$dir_tmp" && vbm "Removed dir_tmp:$dir_tmp"; vbm "STATUS:Main function finished."; } # Main function #===END Declare local script functions=== #==END Define script parameters== #==BEGIN Perform work and exit== main "$@" # Run main function. exit 0; #==END Perform work and exit== # Author: Steven Baltakatei Sandoval; # License: GPLv3+