From f5024030c11cd53a1cfdb067f128e3047dff34a2 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Tue, 14 Jul 2020 02:08:49 +0000 Subject: [PATCH] fix(bklog):Fix buffer drift via SECONDS ==Change 1== Fix drift in when bklog starts reading data via `while read` by enforcing buffer termination when SECONDS (script variable that increments every second) rises above a number determined by the current bufferRound number and bufferTTL. ==Change 2== Also initially advance SECONDS so it aligns with the remainder of dividing the current seconds remaining in the day by bufferTTL. The result should be that, if the user specifies a bufferTTL value that is a clean fraction of a time element (ex: a bufferTTL of 5 minutes = 300 seconds), then each buffer round should start at a clean fraction of absolute time with respect to the day (ex: 02:05, 02:10, 02:15, etc.). This may assist searching through saved files since instead of files named: 20200714T000019+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T000521+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T001023+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T001525+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T002027+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T002529+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T003031+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T003533+0000--PT5M0S..hostname_location.nmea.gz.age they will be named something more like: 20200714T000000+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T000500+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T001000+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T001500+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T002000+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T002500+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T003000+0000--PT5M0S..hostname_location.nmea.gz.age 20200714T003500+0000--PT5M0S..hostname_location.nmea.gz.age ==Change 3== Fix how the ISO-8601 duration string is calculated. Instead of using a fixed value of `bufferTTL` seconds, record the epoch start and end times (in seconds since 1970-01-01) of the main data gathering loop and then calculate the difference. Now, actual deviations in buffer gathering durations (ex: the first buffer round will likely be shorter than a normal one) will be accurately represented. --- exec/bklog | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/exec/bklog b/exec/bklog index fbe28e0..9b7fd67 100644 --- a/exec/bklog +++ b/exec/bklog @@ -11,7 +11,7 @@ dirTmpDefault="/dev/shm"; # Default parent of working directory # Script Metadata scriptName="bklog"; # Define basename of script file. -scriptVersion="0.1.24"; # Define version of script. +scriptVersion="0.1.25"; # Define version of script. scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN scriptHostname=$(hostname); # Save hostname of system running this script. @@ -1218,14 +1218,15 @@ magicWriteVersion() { } # write version data to pathout_tar via appendArgTar() magicProcessWriteBuffer() { # Desc: process and write buffer - # In : vars: bufferTTL bufferTTL_STR scriptHostname label dir_tmp SECONDS + # In : vars: bufferTTL scriptHostname label dir_tmp SECONDS + # : vars: timeBufferStartEpoch timeBufferEndEpoch # : arry: buffer # Out: file:(pathout_tar) # Depends: Bash 5.0.3, date 8.30, yell(), vbm(), dateTimeShort(), ### Note: These arrays should all have the same number of elements: ### pathouts, fileouts, procFileExts, procStrings - local fn timeBufferStartLong timeBufferStart fileoutBasename + local fn timeBufferStartLong timeBufferStart bufferDuration bufferDurationStr fileoutBasename local -a fileouts pathouts local writeCmd1 writeCmd2 writeCmd3 writeCmd4 @@ -1239,12 +1240,17 @@ magicProcessWriteBuffer() { # Determine file paths (time is start of buffer period) ## Calculate start time - timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \ - vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; + timeBufferStartLong="$(date --date="@$timeBufferStartEpoch" --iso-8601=seconds)" && \ + vbm "DEBUG :$fn:timeBufferStartLong:$timeBufferStartLong"; # Note start time in 'date' parsable ISO-8601 timeBufferStart="$(dateTimeShort "$timeBufferStartLong" )" && \ vbm "DEBUG :$fn:timeBufferStart:$timeBufferStart"; # Note start time YYYYmmddTHHMMSS+zzzz (no separators) + ## Calculate buffer duration string (ISO-8601 duration) + bufferDuration="$((timeBufferEndEpoch - timeBufferStartEpoch))" && \ + vbm "DEBUG :$fn:bufferDuration:$bufferDuration"; # length of time (seconds) stdin was read + bufferDurationStr="$(timeDuration "$bufferDuration")" && \ + vbm "DEBUG :$fn:bufferDurationStr:$bufferDurationStr"; # buffer duration (ISO-8601) ## Set common basename - fileoutBasename="$timeBufferStart""--""$bufferTTL_STR""..""$scriptHostname""$label" && \ + fileoutBasename="$timeBufferStart""--""$bufferDurationStr""..""$scriptHostname""$label" && \ vbm "STATUS:$fn:Set fileoutBasename to:$fileoutBasename"; ## Determine output file name array ### in: fileOutBasename cmd_compress_suffix cmd_encrypt_suffix procFileExts @@ -1400,8 +1406,13 @@ main() { # Perform secondary setup operations ## Set script lifespan (scriptTTL from scriptTTL_TE) magicSetScriptTTL "$scriptTTL_TE"; - ## File name substring (ISO-8601 duration from bufferTTL) - bufferTTL_STR="$(timeDuration "$bufferTTL")" && vbm "DEBUG :$fn:bufferTTL_STR:$bufferTTL_STR"; + ## Adjust SECONDS so buffer rounds align with time elements + ### Advance SECONDS the remainder seconds for dividend timeUntilNextDay, divisor bufferTTL + if [[ "$(timeUntilNextDay)" -gt "$bufferTTL" ]]; then + vbm "DEBUG :$fn:SECONDS currently :$SECONDS"; + SECONDS="$(( $(timeUntilNextDay) % bufferTTL ))" && \ + vbm "DEBUG :$fn:SECONDS advanced to:$SECONDS"; + fi; ## Init temp working dir try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp"; ## Initialize output tar (set pathout_tar) @@ -1418,12 +1429,16 @@ main() { bufferRound=0; while [[ $SECONDS -lt "scriptTTL" ]]; do vbm "STATUS:$fn:Starting buffer round:$bufferRound"; - bufferTOD="$((SECONDS + bufferTTL))"; # Set buffer round time-of-death + bufferTOD="$(( (1+bufferRound)*bufferTTL ))"; # Set buffer round time-of-death + # Note start time of data collection + timeBufferStartEpoch="$(date +%s)"; # Consume stdin to fill buffer until buffer time-of-death (TOD) arrives while read -r -t "$bufferTTL" line && [[ $SECONDS -lt "$bufferTOD" ]]; do # Append line to buffer array buffer+=("$line"); done; + # Note end time of data collection + timeBufferEndEpoch="$(date +%s)"; # Create dir_tmp if missing if ! [[ -d "$dir_tmp" ]]; then yell "ERROR:$fn:dir_tmp existence failure:$dir_tmp"; -- 2.30.2