X-Git-Url: https://zdv2.bktei.com/gitweb/EVA-2020-02.git/blobdiff_plain/b3dc68f67500467655a44e8e8ec86b28df744093..8151f8ce6e2cfd2629e7a26b1ce94d6b860be50c:/exec/bklog?ds=inline diff --git a/exec/bklog b/exec/bklog index 45d0bd7..504ecbc 100644 --- a/exec/bklog +++ b/exec/bklog @@ -11,8 +11,9 @@ dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. -scriptVersion="0.1.18"; # Define version of script. +scriptVersion="0.1.29"; # Define version of script. scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. +scriptTimeStartEpoch="$(date +%s)"; # Save start time of script in epoch seconds scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case user apps saved there @@ -54,7 +55,7 @@ processArguments() { -t | --temp-dir) optionTmpDir="true" && argTempDirPriority="$2"; shift;; # Set time zone -b | --buffer-ttl) optionCustomBufferTTL="true" && argCustomBufferTTL="$2"; shift;; # Set custom buffer period (default: 300 seconds) -B | --script-ttl) optionCustomScriptTTL_TE="true" && argCustomScriptTTL_TE="$2"; shift;; # Set custom script TTL (default: "day") - -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$2\" for output of processing string added:\"$3\""; shift; shift;; + -p | --process-string) optionProcString="true" && argProcStrings+=("$2") && argProcFileExts+=("$3") && vbm "STATUS:file extension \"$3\" for output of processing string added:\"$2\""; shift; shift;; -l | --label) optionLabel="true" && argLabel="$2"; vbm "DEBUG :Custom label received:$argLabel"; shift;; -w | --store-raw) optionStoreRaw="true" && argRawFileExt="$2"; vbm "DEBUG :Raw stdin file extension received:$argRawFileExt"; shift;; -W | --no-store-raw) optionNoStoreRaw="true"; vbm "DEBUG :Option selected to not store raw stdin data."; shift;; @@ -66,7 +67,7 @@ processArguments() { vbm() { # Description: Prints verbose message ("vbm") to stderr if optionVerbose is set to "true". # Usage: vbm "DEBUG :verbose message here" - # Version 0.1.2 + # Version 0.1.3 # Input: arg1: string # vars: optionVerbose # Output: stderr @@ -74,7 +75,7 @@ vbm() { if [ "$optionVerbose" = "true" ]; then functionTime=$(date --iso-8601=ns); # Save current time in nano seconds. - echo "[$functionTime] ""$*" 1>&2; # Display argument text. + echo "[$functionTime]:$0:""$*" 1>&2; # Display argument text. fi # End function @@ -225,6 +226,64 @@ displayMissing() { #==END Display errors== } # Display missing apps, files, dirs +appendArgTar(){ + # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar + # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) + # Version: 1.0.6 + # Input: arg1: data to be written + # arg2: file name of file to be inserted into tar + # arg3: tar archive path (must exist first) + # arg4: temporary working dir + # arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ") + # Output: file written to disk + # Example: decrypt multiple large files in parallel + # appendArgTar "$(cat /tmp/largefile1.gpg)" "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendArgTar "$(cat /tmp/largefile2.gpg)" "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendArgTar "$(cat /tmp/largefile3.gpg)" "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & + # Depends: bash 5, tar 1, yell() + # Ref/Attrib: Using 'eval' to construct command strings https://askubuntu.com/a/476533 + + local fn fileName tarPath tmpDir cmd0 cmd1 cmd2 cmd3 cmd4 + + # Save function name + fn="${FUNCNAME[0]}"; + #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." + + # Set file name + if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi + + # Check tar path is a file + if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file."; exit 1; fi + + # Check temp dir arg + if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi + + # Set command strings + if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string 1 + if ! [ -z "$6" ]; then cmd2="$6"; else cmd2="cat "; fi # command string 2 + if ! [ -z "$7" ]; then cmd3="$7"; else cmd3="cat "; fi # command string 3 + if ! [ -z "$8" ]; then cmd4="$8"; else cmd4="cat "; fi # command string 4 + + # Input command + cmd0="echo \"\$1\"" + + # # Debug + # yell "DEBUG:STATUS:$fn:cmd0:$cmd0" + # yell "DEBUG:STATUS:$fn:cmd1:$cmd1" + # yell "DEBUG:STATUS:$fn:cmd2:$cmd2" + # yell "DEBUG:STATUS:$fn:cmd3:$cmd3" + # yell "DEBUG:STATUS:$fn:cmd4:$cmd4" + # yell "DEBUG:STATUS:$fn:fileName:$fileName" + # yell "DEBUG:STATUS:$fn:tarPath:$tarPath" + # yell "DEBUG:STATUS:$fn:tmpDir:$tmpDir" + + # Write to temporary working dir + eval "$cmd0 | $cmd1 | $cmd2 | $cmd3 | $cmd4" > "$tmpDir"/"$fileName"; + + # Append to tar + try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName"; + #yell "DEBUG:STATUS:$fn:Finished appendArgTar()." +} # Append Bash var to file appended to Tar archive appendFileTar(){ # Desc: Appends [processed] file to tar # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd]) @@ -299,17 +358,20 @@ checkMakeTar() { if tar --list --file="$pathTar" 1>/dev/null 2>&1; then ## T1: return success returnFlag0="tar valid"; + elif { sleep 2; tar --list --file="$pathTar" 1>/dev/null 2>&1; }; then + ## F1: Check tar archive again after 2-second sleep + returnFlag0="tar valid"; else - ## F1: Check if file exists + ## F2-1: Check if file exists if [[ -f "$pathTar" ]]; then ### T: Rename file - try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S)" && \ + try mv "$pathTar" "$pathTar""--broken--""$(date +%Y%m%dT%H%M%S%z)" && \ returnFlag1="tar moved"; else ### F: - : fi; - ## F2: Create tar archive, return 0 + ## F2-1: Create tar archive, return 0 try tar --create --file="$pathTar" --files-from=/dev/null && \ returnFlag2="tar created"; fi; @@ -434,7 +496,7 @@ timeDuration(){ # Ref/Attrib: ISO-8601:2004(E), §4.4.4.2 Representations of time intervals by duration and context information # Note: "1 month" ("P1M") is assumed to be "30 days" (see ISO-8601:2004(E), §2.2.1.2) # Usage: timeDuration [1:seconds] ([2:precision]) - # Version: 1.0.4 + # Version: 1.0.5 # Input: arg1: seconds as base 10 integer >= 0 (ex: 3601) # arg2: precision level (optional; default=2) # Output: stdout: ISO-8601 duration string (ex: "P1H1S", "P2Y10M15DT10H30M20S") @@ -510,7 +572,7 @@ timeDuration(){ if [[ $fullDays -gt 0 ]]; then hasDays="true"; else hasDays="false"; fi if [[ $fullHours -gt 0 ]]; then hasHours="true"; else hasHours="false"; fi if [[ $fullMinutes -gt 0 ]]; then hasMinutes="true"; else hasMinutes="false"; fi - if [[ $fullSeconds -gt 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi + if [[ $fullSeconds -ge 0 ]]; then hasSeconds="true"; else hasSeconds="false"; fi ## Determine which fields to display (see ISO-8601:2004 §4.4.3.2) witherPrecision="false" @@ -787,7 +849,7 @@ magicInitCheckTar() { # input: vars: scriptHostname # output: vars: pathout_tar # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion() - local fn + local fn checkMakeTarES # Save function name fn="${FUNCNAME[0]}"; @@ -797,10 +859,10 @@ magicInitCheckTar() { pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \ vbm "STATUS:$fn:Set pathout_tar to:$pathout_tar"; # Validate pathout_tar as tar. - checkMakeTar "$pathout_tar"; + checkMakeTar "$pathout_tar"; checkMakeTarES="$?"; ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2) - vbm "STATUS:$fn:exit status before magicWriteVersion:$?" - if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi + vbm "STATUS:$fn:exit status before magicWriteVersion:$checkMakeTarES" + if [[ "$checkMakeTarES" -eq 1 ]] || [[ "$checkMakeTarES" -eq 2 ]]; then magicWriteVersion; fi vbm "STATUS:$fn:Finished magicInitCheckTar() function."; } # Initialize tar, set pathout_tar magicParseCompressionArg() { @@ -826,7 +888,7 @@ magicParseCompressionArg() { cmd_compress_suffix="" && vbm "STATUS:$fn:cmd_compress_suffix:$cmd_compress_suffix"; vbm "DEBUG :$fn:Compression not enabled."; fi; - vbm "STATUS:$fn:Starting magicParseCompressionArg() function."; + vbm "STATUS:$fn:Finished magicParseCompressionArg() function."; } # Form compression cmd string and filename suffix magicParseCustomTTL() { # Desc: Set user-specified TTLs for buffer and script @@ -867,7 +929,7 @@ magicParseCustomTTL() { fi; ## F: do not change scriptTTL_TE fi; - vbm "STATUS:$fn:Starting magicParseCustomTTL() function."; + vbm "STATUS:$fn:Finished magicParseCustomTTL() function."; } # Sets custom script or buffer TTL if specified magicParseLabel() { # Desc: Parses -l option to set label @@ -971,9 +1033,11 @@ magicParseProcessStrings() { exit 1; fi; done; vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents."; vbm "STATUS:$fn:argProcStrings:${argProcStrings[*]}" - vbm "STATUS:$fn:argProcStrings:${argProcFileExts[*]}" + vbm "STATUS:$fn:argProcFileExts:${argProcFileExts[*]}" procStrings+=("${argProcStrings[@]}"); # Export process command strings procFileExts+=("${argProcFileExts[@]}"); # Export process command strings + vbm "STATUS:$fn:procStrings:${procStrings[*]}" + vbm "STATUS:$fn:procFileExts:${procFileExts[*]}" vbm "STATUS:$fn:Finished magicParseProcessStrings() function."; } # Validate and save process strings and file extensions to arrays procStrings, procFileExts magicParseRecipients() { @@ -1088,19 +1152,23 @@ magicSetScriptTTL() { vbm "STATUS:$fn:Starting magicSetScriptTTL() function."; argTimeElement="$1"; if [[ "$argTimeElement" = "day" ]]; then - # Set script lifespan to end at start of next day + # Set script lifespan to end at start of next day + vbm "STATUS:$fn:Setting script lifespan to end at start of next day. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextDay)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then - ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextDay exit code $?"; exit 1; fi; fi; elif [[ "$argTimeElement" = "hour" ]]; then # Set script lifespan to end at start of next hour + vbm "STATUS:$fn:Setting script lifespan to end at start of next hour. argTimeElement:$argTimeElement"; if ! scriptTTL="$(timeUntilNextHour)"; then # sets scriptTTL, then checks exit code if [[ "$scriptTTL" -eq 0 ]]; then ((scriptTTL++)); # Add 1 because 0 would cause 'timeout' to never timeout. + vbm "STATUS:$fn:scriptTTL:$scriptTTL"; else yell "ERROR:$fn:timeUntilNextHour exit code $?"; exit 1; fi; @@ -1151,14 +1219,15 @@ magicWriteVersion() { } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer - # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS + # In : vars: bufferTTL scriptHostname label dir_tmp SECONDS + # : vars: timeBufferStartEpoch timeBufferEndEpoch # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings - local fn timeBufferStartLong timeBufferStart fileoutBasename + local fn timeBufferStartLong timeBufferStart bufferDuration bufferDurationStr fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 @@ -1172,12 +1241,17 @@ magicProcessWriteBuffer() { # Determine file paths (time is start of buffer period) ## Calculate start time - timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \ - vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; + timeBufferStartLong="$(date --date="@$timeBufferStartEpoch" --iso-8601=seconds)" && \ + vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; # Note start time in 'date' parsable ISO-8601 timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "DEBUG :$fn:timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) + ## Calculate buffer duration string (ISO-8601 duration) + bufferDuration="$((timeBufferEndEpoch - timeBufferStartEpoch))" && \ + vbm "DEBUG :$fn:bufferDuration:$bufferDuration"; # length of time (seconds) stdin was read + bufferDurationStr="$(timeDuration "$bufferDuration")" && \ + vbm "DEBUG :$fn:bufferDurationStr:$bufferDurationStr"; # buffer duration (ISO-8601) ## Set common basename - fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \ + fileoutBasename="$timeBufferStart""--""$bufferDurationStr""..""$scriptHostname""$label" && \ vbm "STATUS:$fn:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts @@ -1200,17 +1274,24 @@ magicProcessWriteBuffer() { writeCmd4="$cmd_encrypt"; ## Process buffer and write to dir_tmp + vbm "DEBUG :$fn:fileouts element count:${#fileouts[@]}"; + vbm "DEBUG :$fn:pathouts element count:${#pathouts[@]}"; + vbm "DEBUG :$fn:procStrings element count:${#pathouts[@]}"; + vbm "DEBUG :$fn:fileouts contents:${fileouts[*]}"; + vbm "DEBUG :$fn:pathouts contents:${pathouts[*]}"; + vbm "DEBUG :$fn:procStrings contents:${pathouts[*]}"; for index in "${!pathouts[@]}"; do writeCmd2="${procStrings[$index]}"; writeCmdAll="$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" && vbm "STATUS:$fn:Assembled command:\"$writeCmdAll\""; - eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" >> "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}"; + eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" > "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}"; done; # Append dir_tmp files to pathout_tar wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) for index in "${!pathouts[@]}"; do - appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \ + tar --append --directory="$dir_tmp" --file="$pathout_tar" "${fileouts[$index]}" && \ vbm "STATUS:$fn:Appended ${pathouts[$index]} to $pathout_tar"; + #appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \ done; # Remove secured chunks from dir_tmp @@ -1326,12 +1407,20 @@ main() { # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; - ## File name substring (ISO-8601 duration from bufferTTL) - bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG :$fn:bufferTTL_STR:$bufferTTL_STR"; + ## Adjust SECONDS so buffer rounds align with time elements + ### Advance SECONDS the remainder seconds for dividend timeUntilNextDay, divisor bufferTTL + if [[ "$(timeUntilNextDay)" -gt "$bufferTTL" ]]; then + vbm "DEBUG :$fn:SECONDS currently :$SECONDS"; + SECONDS="$(( bufferTTL - ($(timeUntilNextDay) % bufferTTL) ))" && \ + vbm "DEBUG :$fn:SECONDS advanced to:$SECONDS"; + vbm "DEBUG :$fn:current time:$(date --iso-8601=seconds)"; + fi; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) - magicInitCheckTar; + magicInitCheckTar; + ## Append VERSION file to tar + magicWriteVersion; # Check vital apps, files, dirs if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then @@ -1342,12 +1431,23 @@ main() { bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do vbm "STATUS:$fn:Starting buffer round:$bufferRound"; - bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death + bufferTOD="$(( (1+bufferRound)*bufferTTL ))" && vbm "DEBUG :$fn:bufferTOD:$bufferTOD"; # Set buffer round time-of-death # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); done; + # Mark time for buffer + ## Initial time + if [[ ! -z "$timeBufferEndEpoch" ]]; then + ### Usual case + timeBufferStartEpoch="$timeBufferEndEpoch" && vbm "DEBUG :$fn:timeBufferStartEpoch:$timeBufferStartEpoch"; + else + ### Edge case: initial startup + timeBufferStartEpoch="$scriptTimeStartEpoch" && vbm "DEBUG :$fn:timeBufferStartEpoch:$timeBufferStartEpoch"; + fi; + ## End Time + timeBufferEndEpoch="$(date +%s)" && vbm "DEBUG :$fn:timeBufferEndEpoch:$timeBufferEndEpoch"; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:$fn:dir_tmp existence failure:$dir_tmp";