From 52f05d074daddd15d135e8ba63dcca004a059fc6 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Tue, 7 Jul 2020 08:31:59 +0000 Subject: [PATCH 1/1] feat(bkgpslog):Add PID correction of buffer round timing Add function to apply PID (proportional, integral, derivative) adjustments to the sleep period between each buffer round in order to compensate for lag caused by synchronous calculations between each buffer round (including the PID adjustment calculation itself). --- exec/bkgpslog | 135 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 2b19e31..db8af89 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.4.3"; # Define version of script. +SCRIPT_VERSION="0.4.4-alpha"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -24,9 +24,15 @@ declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status declare -a argRecPubKeys # for processArguments function +# declare -a errorHistory # for correcting buffer lag ## Initialize variables OPTION_VERBOSE=""; OPTION_ENCRYPT=""; OPTION_COMPRESS=""; OPTION_TMPDIR=""; +errResetx10e3=0; BUFFER_TTL_ADJ_FLOAT=""; +### PID Control factors +K_P=10; # Gain for compensating buffer round lag +T_I=2; # Consider this number of past buffer rounds to eliminate error +T_D=1; # Predict value this number of buffer rounds into the future #===BEGIN Declare local script functions=== checkapp() { @@ -891,6 +897,104 @@ validateInput() { # Return error if no condition matched. return 1; } # Validates strings +timeEpochNS() { + # Desc: Get epoch nanoseconds + # Usage: timeEpochNS + # Version 0.2.2 + # Input: arg1: 'date'-parsable timestamp string (optional) + # Output: Nanoseconds since 1970-01-01 + # Depends: date 8, yell() + # Ref/Attrib: Force base 10 Bash arith with '10#'. https://stackoverflow.com/a/24777667 + local TIME_CURRENT TIME_INPUT TIME_EPOCH_FLOAT TIME_EPOCH_NSFRAC + local TIME_EPOCH_NS + + argTime="$1"; + + # Get Current Time + TIME_CURRENT="$(date --iso-8601=ns)"; # Produce `date`-parsable current timestamp with resolution of 1 nanosecond. + + # Decide to parse current or supplied time + ## Check if time argument empty + if [[ -z "$argTime" ]]; then + ## T: Time argument empty, use current time + TIME_INPUT="$TIME_CURRENT"; + else + ## F: Time argument exists, validate time + if date --date="$argTime" 1>/dev/null 2>&1; then + ### T: Time argument is valid; use it + TIME_INPUT="$argTime"; + else + ### F: Time argument not valid; exit + yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; + fi; + fi; + # Construct and deliver nanoseconds since 1970-01-01 + TIME_EPOCH_FLOAT="$(date --date="$TIME_INPUT" +%s.%N)"; # Save ssss.NNNNNNNNN + TIME_EPOCH_INT="$(echo "$TIME_EPOCH_FLOAT" | cut -d. -f1)"; # Get ssss + TIME_EPOCH_NSFRAC="$(echo "$TIME_EPOCH_FLOAT" | cut -d. -f2)"; # Get NNNNNNNNN + TIME_EPOCH_NS="$(( (10#"$TIME_EPOCH_INT" * 10**9) + (10#"$TIME_EPOCH_NSFRAC") ))"; + echo "$TIME_EPOCH_NS"; +} # Nanoseconds since 1970-01-01 +magicBufferSleepPID() { + # Desc: Compensates for lag so buffer rounds start every BUFFER_TTL seconds + # Input: vars: BUFFER_TTL, errResetx10e3, K_P, T_I, T_D + # # Input: array: errorHistory + # Output: vars: BUFFER_TTL_ADJ_FLOAT + # Re/Attrib: https://en.wikipedia.org/wiki/PID_controller#Standard_versus_parallel_(ideal)_form + local BUFFER_TTL_NS + local timeBufferStartNS timeBufferStartNSExp errNS errNSx10e3 + local errResetx10e3 errRatex10e3 ADJ BUFFER_TTL_ADJ_NS BUFFER_TTL_ADJ_INT + local BUFFER_TTL_ADJ_FLOATFRAC + # local errorHistorySize + + # ## Define errorHistorySize + # errorHistorySize=100; + ## Define BUFFER_TTL in nanoseconds + BUFFER_TTL_NS=$((BUFFER_TTL * 10**9)); + + # Calculate Error, errNS, in nanoseconds + ## Get current time + timeBufferStartNS="$(timeEpochNS)"; + ## Calculate expected time (from start time, current buffer round number, nominal BUFFER_TTL) + timeBufferStartNSExp="$(( (timeBufferFirstNS) + (BUFFER_TTL_NS * bufferRound) ))"; + ## Calculate error (diff between timeBufferStartNSExp and timeBufferStartNS; usually negative) + errNS="$(( timeBufferStartNSExp - timeBufferStartNS ))"; + errNSx10e3="$((errNS*10**3))"; + # ## Append error to errorHistory + # errorHistory+=("errNS"); + # ### Trim errorHistory array if over errorHistorySize + # while [[ "${#errorHistory[@]}" -gt "errorHistorySize" ]]; then do + # unset "errorHistory[0]"; # remove oldest entry, creating sparse array + # errorHistory=("${errorHistory[@]}"); # reindex sparse array + # vbm "STATUS:Trimmed errorHistory array. Entry count:${#errorHistory[@]}"; + # done; + + # Calculate errReset in nanoseconds^2 + ## errReset = int(errHistory(t),wrt(delta_BUFFER_TTL)) + ## Integrate errorHistory with respect to time + # for value in "${errorHistory[@]}"; do + # errReset=$(( errReset + ( value*BUFFER_TTL_NS ) )); + # done; + errResetx10e3="$(( ( errResetx10e3 + ( errNS * BUFFER_TTL_NS ) )*10**3 ))"; + + # Calculate errRate in nanoseconds per 1000 nanoseconds + errRatex10e3="$(( ( errNSx10e3 ) / BUFFER_TTL_NS ))"; + + # Calculate PID control signal + ## ADJ = K_P * (errNS + errReset/T_I + errRate*T_D) + ADJ="$((K_P*(errNSx10e3 + (errResetx10e3/T_I) + (errRatex10e3*T_D) )/10**3))"; + + # Calculate BUFFER_TTL_ADJ_FLOAT from ADJ (ns) + ## Calculate BUFFER_TTL_ADJ in nanoseconds (BUFFER_TTL_ADJ_NS = BUFFER_TTL_NS + ADJ) + BUFFER_TTL_ADJ_NS="$((BUFFER_TTL_NS + ADJ))"; + ## Calculate integer seconds + BUFFER_TTL_ADJ_INT="$((BUFFER_TTL_ADJ_NS/(10**9)))"; + ## Calculate nanosecond remainder + BUFFER_TTL_ADJ_FLOATFRAC="$((BUFFER_TTL_NS - (BUFFER_TTL_ADJ_INT*(10**9)) ))"; + ## Form float BUFFER_TTL_ADJ_FLOAT + BUFFER_TTL_ADJ_FLOAT="$BUFFER_TTL_ADJ_INT"."$BUFFER_TTL_ADJ_FLOATFRAC"; + vbm "STATUS:Calculated adjusted BUFFER_TTL (seconds):$BUFFER_TTL_ADJ_FLOAT"; +} # Calc BUFFER_TTL_ADJ_FLOAT so buffer starts every BUFFER_TTL seconds magicWriteVersion() { # Desc: Appends time-stamped VERSION to PATHOUT_TAR # Usage: magicWriteVersion @@ -927,11 +1031,16 @@ magicWriteVersion() { } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar() magicGatherWriteBuffer() { # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR - # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP, - # Inputs: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX - # Depends: yell, try, vbm, appendArgTar, tar - local FN="${FUNCNAME[0]}"; - wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) + # Inputs: vars: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP, + # Inputs: vars: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX + # Output: file: (PATHOUT_TAR) + # Depends: yell(), try(), vbm(), appendArgTar(), tar 1, sleep 8, checkMakeTar() + # Depends: magicWriteVersion(), appendFileTar() + local FN + + # Debug:Get function name + FN="${FUNCNAME[0]}"; + # Create buffer file with unique name PATHOUT_BUFFER="$DIR_TMP/buffer$SECONDS"; # Fill buffer @@ -965,6 +1074,7 @@ magicGatherWriteBuffer() { if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi # Write bufferBash to PATHOUT_TAR + wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file @@ -1008,7 +1118,7 @@ magicParseRecipientDir() { done #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected if ! [[ "$updateRecipients" = "false" ]]; then - recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[@]}\""; + recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\""; fi; else yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1; @@ -1045,8 +1155,8 @@ magicParseRecipientArgs() { fi; done vbm "DEBUG:Finished processing argRecPubKeys array"; - vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[@]}"; - recPubKeysValidStatic="${recPubKeysValid[@]}"; # Save static image of pubkeys validated by this function + vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[*]}"; + recPubKeysValidStatic=("${recPubKeysValid[@]}"); # Save static image of pubkeys validated by this function ## Form age command string CMD_ENCRYPT="age ""$recipients " && vbm "CMD_ENCRYPT:$CMD_ENCRYPT"; @@ -1199,10 +1309,13 @@ main() { CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML # MAIN LOOP:Record gps data until script lifespan ends + timeBufferFirstNS="$(timeEpochNS)"; bufferRound=0 while [[ "$SECONDS" -lt "$SCRIPT_TTL" ]]; do - magicParseRecipientDir + magicParseRecipientDir; magicGatherWriteBuffer & - sleep "$BUFFER_TTL"; + magicBufferSleepPID; # Calculates BUFFER_TTL_ADJ from BUFFER_TTL given buffer expected start time vs. actual + sleep "$BUFFER_TTL_ADJ_FLOAT"; + ((bufferRound++)); done # Cleanup -- 2.30.2