X-Git-Url: https://zdv2.bktei.com/gitweb/EVA-2020-02.git/blobdiff_plain/7177fe18e5da47f1e75840a2fac3eeb85b0c5395..9089b4f994436147e1f8df860bf7d47fc7a8a552:/exec/bklog?ds=inline diff --git a/exec/bklog b/exec/bklog index ecc4e85..cc807a6 100644 --- a/exec/bklog +++ b/exec/bklog @@ -11,8 +11,9 @@ dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. -scriptVersion="0.1.21-debug"; # Define version of script. -scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. +scriptVersion="0.1.33"; # Define version of script. +scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define website hosting this script. +scriptTimeStartEpoch="$(date +%s)"; # Save start time of script in epoch seconds scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case user apps saved there @@ -54,10 +55,10 @@ processArguments() { -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds) -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day") - -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;; + -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$3\" for output of processing string added:\"$2\""; shift; shift;; -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG :Custom label received:$argLabel"; shift;; -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG :Raw stdin file extension received:$argRawFileExt"; shift;; - -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG :Option selected to not store raw stdin data."; shift;; + -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG :Option selected to not store raw stdin data.";; *) yell "ERROR: Unrecognized argument: $1"; yell "STATUS:All arguments:$*"; exit 1;; # Handle unrecognized options. esac shift @@ -66,7 +67,7 @@ processArguments() { vbm() { # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true". # Usage: vbm "DEBUG :verbose message here" - # Version 0.1.2 + # Version 0.1.3 # Input: arg1: string # vars: optionVerbose # Output: stderr @@ -74,7 +75,7 @@ vbm() { if [ "$optionVerbose" = "true" ]; then functionTime=$(date --iso-8601=ns); # Save current time in nano seconds. - echo "[$functionTime] ""$*" 1>&2; # Display argument text. + echo "[$functionTime]:$0:""$*" 1>&2; # Display argument text. fi # End function @@ -357,17 +358,20 @@ checkMakeTar() { if tar --list --file="$pathTar" 1>/dev/null 2>&1; then ## T1: return success returnFlag0="tar valid"; + elif { sleep 2; tar --list --file="$pathTar" 1>/dev/null 2>&1; }; then + ## F1: Check tar archive again after 2-second sleep + returnFlag0="tar valid"; else - ## F1: Check if file exists + ## F2-1: Check if file exists if [[ -f "$pathTar" ]]; then ### T: Rename file - try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S)" && \ + try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S%z)" && \ returnFlag1="tar moved"; else ### F: - : fi; - ## F2: Create tar archive, return 0 + ## F2-1: Create tar archive, return 0 try tar --create --file="$pathTar" --files-from=/dev/null && \ returnFlag2="tar created"; fi; @@ -492,7 +496,7 @@ timeDuration(){ # Ref/Attrib: ISO-8601:2004(E), §4.4.4.2 Representations of time intervals by duration and context information # Note: "1 month" ("P1M") is assumed to be "30 days" (see ISO-8601:2004(E), §2.2.1.2) # Usage: timeDuration [1:seconds] ([2:precision]) - # Version: 1.0.4 + # Version: 1.0.5 # Input: arg1: seconds as base 10 integer >= 0 (ex: 3601) # arg2: precision level (optional; default=2) # Output: stdout: ISO-8601 duration string (ex: "P1H1S", "P2Y10M15DT10H30M20S") @@ -568,7 +572,7 @@ timeDuration(){ if [[ $fullDays -gt 0 ]]; then hasDays="true"; else hasDays="false"; fi if [[ $fullHours -gt 0 ]]; then hasHours="true"; else hasHours="false"; fi if [[ $fullMinutes -gt 0 ]]; then hasMinutes="true"; else hasMinutes="false"; fi - if [[ $fullSeconds -gt 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi + if [[ $fullSeconds -ge 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi ## Determine which fields to display (see ISO-8601:2004 §4.4.3.2) witherPrecision="false" @@ -884,7 +888,7 @@ magicParseCompressionArg() { cmd_compress_suffix="" && vbm "STATUS:$fn:cmd_compress_suffix:$cmd_compress_suffix"; vbm "DEBUG :$fn:Compression not enabled."; fi; - vbm "STATUS:$fn:Starting magicParseCompressionArg() function."; + vbm "STATUS:$fn:Finished magicParseCompressionArg() function."; } # Form compression cmd string and filename suffix magicParseCustomTTL() { # Desc: Set user-specified TTLs for buffer and script @@ -925,7 +929,7 @@ magicParseCustomTTL() { fi; ## F: do not change scriptTTL_TE fi; - vbm "STATUS:$fn:Starting magicParseCustomTTL() function."; + vbm "STATUS:$fn:Finished magicParseCustomTTL() function."; } # Sets custom script or buffer TTL if specified magicParseLabel() { # Desc: Parses -l option to set label @@ -1029,9 +1033,11 @@ magicParseProcessStrings() { exit 1; fi; done; vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents."; vbm "STATUS:$fn:argProcStrings:${argProcStrings[*]}" - vbm "STATUS:$fn:argProcStrings:${argProcFileExts[*]}" + vbm "STATUS:$fn:argProcFileExts:${argProcFileExts[*]}" procStrings+=("${argProcStrings[@]}"); # Export process command strings procFileExts+=("${argProcFileExts[@]}"); # Export process command strings + vbm "STATUS:$fn:procStrings:${procStrings[*]}" + vbm "STATUS:$fn:procFileExts:${procFileExts[*]}" vbm "STATUS:$fn:Finished magicParseProcessStrings() function."; } # Validate and save process strings and file extensions to arrays procStrings, procFileExts magicParseRecipients() { @@ -1146,19 +1152,23 @@ magicSetScriptTTL() { vbm "STATUS:$fn:Starting magicSetScriptTTL() function."; argTimeElement="$1"; if [[ "$argTimeElement" = "day" ]]; then - # Set script lifespan to end at start of next day + # Set script lifespan to end at start of next day + vbm "STATUS:$fn:Setting script lifespan to end at start of next day. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then - ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextDay exit code $?"; exit 1; fi; fi; elif [[ "$argTimeElement" = "hour" ]]; then # Set script lifespan to end at start of next hour + vbm "STATUS:$fn:Setting script lifespan to end at start of next hour. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextHour exit code $?"; exit 1; fi; @@ -1168,6 +1178,21 @@ magicSetScriptTTL() { fi; vbm "STATUS:$fn:Finished magicSetScriptTTL() function."; } # Set scriptTTL in seconds until next (day|hour). +magicVerboseReadout() { + vbm "$bufferTTL"; # Time-to-live (seconds) for each buffer round + vbm "$scriptTTL_TE"; # Time element at the end of which script terminates + vbm "$dirTmpDefault"; # Default parent of working directory + # Script Metadata + vbm "$scriptName"; # Basename of script file. + vbm "$scriptVersion"; # Version of script. + vbm "$scriptURL"; # Website hosting this script. + vbm "$scriptTimeStartEpoch"; # Start time of script in epoch seconds + vbm "$scriptTimeStart"; # YYYYmmddTHHMMSS.NNNNNNNNN + vbm "$scriptHostname" # Hostname of system running this script. + vbm "$PATH"; # PATH env. var. + vbm "$ageVersion"; # Version of age (encryption program) + vbm "$ageURL"; # Website hosting age. +} # Display script variables magicWriteVersion() { # Desc: Appends time-stamped VERSION to pathout_tar # Usage: magicWriteVersion @@ -1209,14 +1234,15 @@ magicWriteVersion() { } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer - # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS + # In : vars: bufferTTL scriptHostname label dir_tmp SECONDS + # : vars: timeBufferStartEpoch timeBufferEndEpoch # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings - local fn timeBufferStartLong timeBufferStart fileoutBasename + local fn timeBufferStartLong timeBufferStart bufferDuration bufferDurationStr fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 @@ -1230,12 +1256,17 @@ magicProcessWriteBuffer() { # Determine file paths (time is start of buffer period) ## Calculate start time - timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \ - vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; + timeBufferStartLong="$(date --date="@$timeBufferStartEpoch" --iso-8601=seconds)" && \ + vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; # Note start time in 'date' parsable ISO-8601 timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "DEBUG :$fn:timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) + ## Calculate buffer duration string (ISO-8601 duration) + bufferDuration="$((timeBufferEndEpoch - timeBufferStartEpoch))" && \ + vbm "DEBUG :$fn:bufferDuration:$bufferDuration"; # length of time (seconds) stdin was read + bufferDurationStr="$(timeDuration "$bufferDuration")" && \ + vbm "DEBUG :$fn:bufferDurationStr:$bufferDurationStr"; # buffer duration (ISO-8601) ## Set common basename - fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \ + fileoutBasename="$timeBufferStart""--""$bufferDurationStr""..""$scriptHostname""$label" && \ vbm "STATUS:$fn:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts @@ -1280,8 +1311,7 @@ magicProcessWriteBuffer() { # Remove secured chunks from dir_tmp for path in "${pathouts[@]}"; do - # rm "$path" && vbm "STATUS:$fn:Removed:$path"; - : + rm "$path" && vbm "STATUS:$fn:Removed:$path"; done; vbm "STATUS:$fn:Finished magicProcessWriteBuffer()."; @@ -1388,12 +1418,20 @@ main() { magicParseProcessStrings; # Sets arrays: procStrings, procFileExts ## React to "-l" (output file label) option magicParseLabel; # sets label (ex: "_location") + ## React to "-v" (verbose) option + magicVerboseReadout; # Display various script variables # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; - ## File name substring (ISO-8601 duration from bufferTTL) - bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG :$fn:bufferTTL_STR:$bufferTTL_STR"; + ## Adjust SECONDS so buffer rounds align with time elements + ### Advance SECONDS the remainder seconds for dividend timeUntilNextDay, divisor bufferTTL + if [[ "$(timeUntilNextDay)" -gt "$bufferTTL" ]]; then + vbm "DEBUG :$fn:SECONDS currently :$SECONDS"; + SECONDS="$(( bufferTTL - ($(timeUntilNextDay) % bufferTTL) ))" && \ + vbm "DEBUG :$fn:SECONDS advanced to:$SECONDS"; + vbm "DEBUG :$fn:current time:$(date --iso-8601=seconds)"; + fi; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) @@ -1410,12 +1448,29 @@ main() { bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do vbm "STATUS:$fn:Starting buffer round:$bufferRound"; - bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death + bufferTOD="$(( (1+bufferRound)*bufferTTL ))" && vbm "DEBUG :$fn:bufferTOD:$bufferTOD"; # Set buffer round time-of-death # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); - done; + done < /dev/stdin; + # End script if buffer array empty (ex: EOF found in stdin) + if [[ ${#buffer[@]} -eq 0 ]]; then + yell "ERROR:$fn:Empty array:buffer:${buffer[@]}"; exit 1; + fi; + # Mark time for buffer + ## Initial time + if [[ bufferRound -gt 0 ]]; then + ### Usual case + timeBufferStartEpoch="$timeBufferEndEpoch" && vbm "DEBUG :$fn:timeBufferStartEpoch:$timeBufferStartEpoch"; + elif [[ bufferRound -eq 0 ]]; then + ### Edge case: initial startup + timeBufferStartEpoch="$scriptTimeStartEpoch" && vbm "DEBUG :$fn:timeBufferStartEpoch:$timeBufferStartEpoch"; + else + yell "ERROR:$fn:Invalid bufferRound value."; exit 1; + fi; + ## End Time + timeBufferEndEpoch="$(date +%s)" && vbm "DEBUG :$fn:timeBufferEndEpoch:$timeBufferEndEpoch"; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:$fn:dir_tmp existence failure:$dir_tmp";