X-Git-Url: https://zdv2.bktei.com/gitweb/EVA-2020-02.git/blobdiff_plain/236c8b97f828fa0d58474ef028592ff0ff435790..99e199b255630ca54defe2abbc245bd9e259759c:/exec/bklog diff --git a/exec/bklog b/exec/bklog index 5ab7fb6..188910d 100644 --- a/exec/bklog +++ b/exec/bklog @@ -11,7 +11,7 @@ dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. -scriptVersion="0.1.15"; # Define version of script. +scriptVersion="0.1.26"; # Define version of script. scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. @@ -54,7 +54,7 @@ processArguments() { -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds) -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day") - -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;; + -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$3\" for output of processing string added:\"$2\""; shift; shift;; -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG :Custom label received:$argLabel"; shift;; -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG :Raw stdin file extension received:$argRawFileExt"; shift;; -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG :Option selected to not store raw stdin data."; shift;; @@ -66,7 +66,7 @@ processArguments() { vbm() { # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true". # Usage: vbm "DEBUG :verbose message here" - # Version 0.1.2 + # Version 0.1.3 # Input: arg1: string # vars: optionVerbose # Output: stderr @@ -74,7 +74,7 @@ vbm() { if [ "$optionVerbose" = "true" ]; then functionTime=$(date --iso-8601=ns); # Save current time in nano seconds. - echo "[$functionTime] ""$*" 1>&2; # Display argument text. + echo "[$functionTime]:$0:""$*" 1>&2; # Display argument text. fi # End function @@ -225,6 +225,64 @@ displayMissing() { #==END Display errors== } # Display missing apps, files, dirs +appendArgTar(){ + # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar + # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) + # Version: 1.0.6 + # Input: arg1: data to be written + # arg2: file name of file to be inserted into tar + # arg3: tar archive path (must exist first) + # arg4: temporary working dir + # arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ") + # Output: file written to disk + # Example: decrypt multiple large files in parallel + # appendArgTar "$(cat /tmp/largefile1.gpg)" "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendArgTar "$(cat /tmp/largefile2.gpg)" "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendArgTar "$(cat /tmp/largefile3.gpg)" "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & + # Depends: bash 5, tar 1, yell() + # Ref/Attrib: Using 'eval' to construct command strings https://askubuntu.com/a/476533 + + local fn fileName tarPath tmpDir cmd0 cmd1 cmd2 cmd3 cmd4 + + # Save function name + fn="${FUNCNAME[0]}"; + #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." + + # Set file name + if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi + + # Check tar path is a file + if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file."; exit 1; fi + + # Check temp dir arg + if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi + + # Set command strings + if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string 1 + if ! [ -z "$6" ]; then cmd2="$6"; else cmd2="cat "; fi # command string 2 + if ! [ -z "$7" ]; then cmd3="$7"; else cmd3="cat "; fi # command string 3 + if ! [ -z "$8" ]; then cmd4="$8"; else cmd4="cat "; fi # command string 4 + + # Input command + cmd0="echo \"\$1\"" + + # # Debug + # yell "DEBUG:STATUS:$fn:cmd0:$cmd0" + # yell "DEBUG:STATUS:$fn:cmd1:$cmd1" + # yell "DEBUG:STATUS:$fn:cmd2:$cmd2" + # yell "DEBUG:STATUS:$fn:cmd3:$cmd3" + # yell "DEBUG:STATUS:$fn:cmd4:$cmd4" + # yell "DEBUG:STATUS:$fn:fileName:$fileName" + # yell "DEBUG:STATUS:$fn:tarPath:$tarPath" + # yell "DEBUG:STATUS:$fn:tmpDir:$tmpDir" + + # Write to temporary working dir + eval "$cmd0 | $cmd1 | $cmd2 | $cmd3 | $cmd4" > "$tmpDir"/"$fileName"; + + # Append to tar + try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName"; + #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." +} # Append Bash var to file appended to Tar archive appendFileTar(){ # Desc: Appends [processed] file to tar # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd]) @@ -299,17 +357,20 @@ checkMakeTar() { if tar --list --file="$pathTar" 1>/dev/null 2>&1; then ## T1: return success returnFlag0="tar valid"; + elif { sleep 2; tar --list --file="$pathTar" 1>/dev/null 2>&1; }; then + ## F1: Check tar archive again after 2-second sleep + returnFlag0="tar valid"; else - ## F1: Check if file exists + ## F2-1: Check if file exists if [[ -f "$pathTar" ]]; then ### T: Rename file - try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S)" && \ + try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S%z)" && \ returnFlag1="tar moved"; else ### F: - : fi; - ## F2: Create tar archive, return 0 + ## F2-1: Create tar archive, return 0 try tar --create --file="$pathTar" --files-from=/dev/null && \ returnFlag2="tar created"; fi; @@ -787,7 +848,7 @@ magicInitCheckTar() { # input: vars: scriptHostname # output: vars: pathout_tar # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion() - local fn + local fn checkMakeTarES # Save function name fn="${FUNCNAME[0]}"; @@ -797,10 +858,10 @@ magicInitCheckTar() { pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \ vbm "STATUS:$fn:Set pathout_tar to:$pathout_tar"; # Validate pathout_tar as tar. - checkMakeTar "$pathout_tar"; + checkMakeTar "$pathout_tar"; checkMakeTarES="$?"; ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2) - vbm "STATUS:$fn:exit status before magicWriteVersion:$?" - if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi + vbm "STATUS:$fn:exit status before magicWriteVersion:$checkMakeTarES" + if [[ "$checkMakeTarES" -eq 1 ]] || [[ "$checkMakeTarES" -eq 2 ]]; then magicWriteVersion; fi vbm "STATUS:$fn:Finished magicInitCheckTar() function."; } # Initialize tar, set pathout_tar magicParseCompressionArg() { @@ -826,7 +887,7 @@ magicParseCompressionArg() { cmd_compress_suffix="" && vbm "STATUS:$fn:cmd_compress_suffix:$cmd_compress_suffix"; vbm "DEBUG :$fn:Compression not enabled."; fi; - vbm "STATUS:$fn:Starting magicParseCompressionArg() function."; + vbm "STATUS:$fn:Finished magicParseCompressionArg() function."; } # Form compression cmd string and filename suffix magicParseCustomTTL() { # Desc: Set user-specified TTLs for buffer and script @@ -867,7 +928,7 @@ magicParseCustomTTL() { fi; ## F: do not change scriptTTL_TE fi; - vbm "STATUS:$fn:Starting magicParseCustomTTL() function."; + vbm "STATUS:$fn:Finished magicParseCustomTTL() function."; } # Sets custom script or buffer TTL if specified magicParseLabel() { # Desc: Parses -l option to set label @@ -914,28 +975,40 @@ magicParseProcessStrings() { # Validate input ## Validate argRawFileExt if [[ "$argRawFileExt" =~ ^[.][[:alnum:]]*$ ]]; then - rawFileExt="$argRawFileExt"; + rawFileExt="$argRawFileExt" && \ + vbm "DEBUG :$fn:Set rawFileExt to \"$argRawFileExt\""; + else + vbm "DEBUG :$fn:Validation failure for $argRawFileExt . Not set to rawFileExt."; fi; # Add default stdin output file entries for procStrings, procFileExts ## Check if user specified that no raw stdin be saved. if [[ ! "$optionNoStoreRaw" = "true" ]]; then ### T: --no-store-raw not set. Store raw. Append procStrings with cat. + vbm "DEBUG :$fn:--no-store-raw not set. Storing raw."; #### Append procStrings array - procStrings+=("cat "); + procStrings+=("cat ") && \ + vbm "DEBUG :$fn:Appended \"cat \" to procStrings"; + vbm "DEBUG :$fn:procStrings array:${procStrings[*]}"; #### Check if --store-raw set. if [[ "$optionStoreRaw" = "true" ]]; then ##### T: --store-raw set. Append procFileExts with user-specified file ext - procFileExts+=("$rawFileExt"); + vbm "DEBUG :$fn:--store-raw set."; + procFileExts+=("$rawFileExt") && \ + vbm "DEBUG :$fn:Appended $rawFileExt to procFileExts"; + vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; else ##### F: --store-raw not set. Append procFileExts with default ".stdin" file ext ###### Append procFileExts array - procFileExts+=(".stdin"); + procFileExts+=(".stdin") && \ + vbm "DEBUG :$fn:Appended \".stdin\" to procFileExts"; + vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; fi; else ### F: --no-store-raw set. Do not store raw. #### Do not append procStrings or procFileExts arrays. - : + vbm "STATUS:$fn:--no-store-raw set. Not storing raw."; + vbm "STATUS:$fn:procFileExts array:${procFileExts[*]}"; fi; # Do nothing more if optionProcString not set to true. @@ -958,8 +1031,12 @@ magicParseProcessStrings() { yell "ERROR:$fn:Illegal character '-' at start of process string element:\"$element\""; exit 1; fi; done; vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents."; - procStrings=("${argProcStrings[@]}"); # Export process command strings - procFileExts=("${argProcFileExts[@]}"); # Export process command strings + vbm "STATUS:$fn:argProcStrings:${argProcStrings[*]}" + vbm "STATUS:$fn:argProcFileExts:${argProcFileExts[*]}" + procStrings+=("${argProcStrings[@]}"); # Export process command strings + procFileExts+=("${argProcFileExts[@]}"); # Export process command strings + vbm "STATUS:$fn:procStrings:${procStrings[*]}" + vbm "STATUS:$fn:procFileExts:${procFileExts[*]}" vbm "STATUS:$fn:Finished magicParseProcessStrings() function."; } # Validate and save process strings and file extensions to arrays procStrings, procFileExts magicParseRecipients() { @@ -1074,19 +1151,23 @@ magicSetScriptTTL() { vbm "STATUS:$fn:Starting magicSetScriptTTL() function."; argTimeElement="$1"; if [[ "$argTimeElement" = "day" ]]; then - # Set script lifespan to end at start of next day + # Set script lifespan to end at start of next day + vbm "STATUS:$fn:Setting script lifespan to end at start of next day. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then - ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextDay exit code $?"; exit 1; fi; fi; elif [[ "$argTimeElement" = "hour" ]]; then # Set script lifespan to end at start of next hour + vbm "STATUS:$fn:Setting script lifespan to end at start of next hour. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextHour exit code $?"; exit 1; fi; @@ -1137,14 +1218,15 @@ magicWriteVersion() { } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer - # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS + # In : vars: bufferTTL scriptHostname label dir_tmp SECONDS + # : vars: timeBufferStartEpoch timeBufferEndEpoch # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings - local fn timeBufferStartLong timeBufferStart fileoutBasename + local fn timeBufferStartLong timeBufferStart bufferDuration bufferDurationStr fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 @@ -1152,14 +1234,23 @@ magicProcessWriteBuffer() { fn="${FUNCNAME[0]}"; vbm "STATUS:$fn:Started magicProcessWriteBuffer()."; + vbm "DEBUG :$fn:buffer array element count:${#buffer[@]}"; + vbm "DEBUG :$fn:buffer array first element:${buffer[0]}"; + vbm "DEBUG :$fn:buffer array last element :${buffer[-1]}"; + # Determine file paths (time is start of buffer period) ## Calculate start time - timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \ - vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; + timeBufferStartLong="$(date --date="@$timeBufferStartEpoch" --iso-8601=seconds)" && \ + vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; # Note start time in 'date' parsable ISO-8601 timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "DEBUG :$fn:timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) + ## Calculate buffer duration string (ISO-8601 duration) + bufferDuration="$((timeBufferEndEpoch - timeBufferStartEpoch))" && \ + vbm "DEBUG :$fn:bufferDuration:$bufferDuration"; # length of time (seconds) stdin was read + bufferDurationStr="$(timeDuration "$bufferDuration")" && \ + vbm "DEBUG :$fn:bufferDurationStr:$bufferDurationStr"; # buffer duration (ISO-8601) ## Set common basename - fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \ + fileoutBasename="$timeBufferStart""--""$bufferDurationStr""..""$scriptHostname""$label" && \ vbm "STATUS:$fn:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts @@ -1182,17 +1273,24 @@ magicProcessWriteBuffer() { writeCmd4="$cmd_encrypt"; ## Process buffer and write to dir_tmp + vbm "DEBUG :$fn:fileouts element count:${#fileouts[@]}"; + vbm "DEBUG :$fn:pathouts element count:${#pathouts[@]}"; + vbm "DEBUG :$fn:procStrings element count:${#pathouts[@]}"; + vbm "DEBUG :$fn:fileouts contents:${fileouts[*]}"; + vbm "DEBUG :$fn:pathouts contents:${pathouts[*]}"; + vbm "DEBUG :$fn:procStrings contents:${pathouts[*]}"; for index in "${!pathouts[@]}"; do writeCmd2="${procStrings[$index]}"; writeCmdAll="$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" && vbm "STATUS:$fn:Assembled command:\"$writeCmdAll\""; - eval "$writeCmdAll" >> "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}"; + eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" > "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}"; done; # Append dir_tmp files to pathout_tar wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) for index in "${!pathouts[@]}"; do - appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \ + tar --append --directory="$dir_tmp" --file="$pathout_tar" "${fileouts[$index]}" && \ vbm "STATUS:$fn:Appended ${pathouts[$index]} to $pathout_tar"; + #appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \ done; # Remove secured chunks from dir_tmp @@ -1308,12 +1406,20 @@ main() { # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; - ## File name substring (ISO-8601 duration from bufferTTL) - bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG :$fn:bufferTTL_STR:$bufferTTL_STR"; + ## Adjust SECONDS so buffer rounds align with time elements + ### Advance SECONDS the remainder seconds for dividend timeUntilNextDay, divisor bufferTTL + if [[ "$(timeUntilNextDay)" -gt "$bufferTTL" ]]; then + vbm "DEBUG :$fn:SECONDS currently :$SECONDS"; + SECONDS="$(( $(timeUntilNextDay) % bufferTTL ))" && \ + vbm "DEBUG :$fn:SECONDS advanced to:$SECONDS"; + vbm "DEBUG :$fn:current time:$(date --iso-8601=seconds)"; + fi; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) - magicInitCheckTar; + magicInitCheckTar; + ## Append VERSION file to tar + magicWriteVersion; # Check vital apps, files, dirs if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then @@ -1324,12 +1430,16 @@ main() { bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do vbm "STATUS:$fn:Starting buffer round:$bufferRound"; - bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death + bufferTOD="$(( (1+bufferRound)*bufferTTL ))" && vbm "DEBUG :$fn:bufferTOD:$bufferTOD"; # Set buffer round time-of-death + # Note start time of data collection + timeBufferStartEpoch="$(date +%s)" && vbm "DEBUG :$fn:timeBufferStartEpoch:$timeBufferStartEpoch"; # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); done; + # Note end time of data collection + timeBufferEndEpoch="$(date +%s)" && vbm "DEBUG :$fn:timeBufferEndEpoch:$timeBufferEndEpoch"; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:$fn:dir_tmp existence failure:$dir_tmp";