feat(bkgpslog):Add PID correction of buffer round timing

Add function to apply PID (proportional, integral, derivative)
adjustments to the sleep period between each buffer round in order to
compensate for lag caused by synchronous calculations between each
buffer round (including the PID adjustment calculation itself).

diff --git a/exec/bkgpslog b/exec/bkgpslog
index 2b19e31e087024c59416c97824fc1962e4b19ca7..db8af896939b24dd57473ba416c925cf9fcb43bd 100755 (executable)
--- a/exec/bkgpslog
+++ b/exec/bkgpslog
@@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory
 SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N);
 PATH="$HOME/.local/bin:$PATH";   # Add "$(systemd-path user-binaries)" path in case apps saved there
 SCRIPT_HOSTNAME=$(hostname);     # Save hostname of system running this script.
 SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N);
 PATH="$HOME/.local/bin:$PATH";   # Add "$(systemd-path user-binaries)" path in case apps saved there
 SCRIPT_HOSTNAME=$(hostname);     # Save hostname of system running this script.

-SCRIPT_VERSION="0.4.3";          # Define version of script.
+SCRIPT_VERSION="0.4.4-alpha";          # Define version of script.

 SCRIPT_NAME="bkgpslog";          # Define basename of script file.
 SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
 AGE_VERSION="1.0.0-beta2";       # Define version of age (encryption program)
 SCRIPT_NAME="bkgpslog";          # Define basename of script file.
 SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
 AGE_VERSION="1.0.0-beta2";       # Define version of age (encryption program)


@@ -24,9 +24,15 @@ declare -Ag appRollCall # Associative array for storing app status
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status
 declare -a argRecPubKeys # for processArguments function
 declare -Ag fileRollCall # Associative array for storing file status
 declare -Ag dirRollCall # Associative array for storing dir status
 declare -a argRecPubKeys # for processArguments function

+# declare -a errorHistory # for correcting buffer lag

 
 ## Initialize variables
 OPTION_VERBOSE=""; OPTION_ENCRYPT=""; OPTION_COMPRESS=""; OPTION_TMPDIR="";
 
 ## Initialize variables
 OPTION_VERBOSE=""; OPTION_ENCRYPT=""; OPTION_COMPRESS=""; OPTION_TMPDIR="";

+errResetx10e3=0; BUFFER_TTL_ADJ_FLOAT="";
+### PID Control factors
+K_P=10; # Gain for compensating buffer round lag
+T_I=2; # Consider this number of past buffer rounds to eliminate error
+T_D=1; # Predict value this number of buffer rounds into the future

 
 #===BEGIN Declare local script functions===
 checkapp() {
 
 #===BEGIN Declare local script functions===
 checkapp() {


@@ -891,6 +897,104 @@ validateInput() {
     # Return error if no condition matched.
     return 1;
 } # Validates strings
     # Return error if no condition matched.
     return 1;
 } # Validates strings

+timeEpochNS() {
+    # Desc: Get epoch nanoseconds
+    # Usage: timeEpochNS
+    # Version 0.2.2
+    # Input: arg1: 'date'-parsable timestamp string (optional)
+    # Output: Nanoseconds since 1970-01-01
+    # Depends: date 8, yell()
+    # Ref/Attrib: Force base 10 Bash arith with '10#'. https://stackoverflow.com/a/24777667
+    local TIME_CURRENT TIME_INPUT TIME_EPOCH_FLOAT TIME_EPOCH_NSFRAC
+    local TIME_EPOCH_NS
+
+    argTime="$1";
+
+    # Get Current Time
+    TIME_CURRENT="$(date --iso-8601=ns)"; # Produce `date`-parsable current timestamp with resolution of 1 nanosecond.
+
+    # Decide to parse current or supplied time
+    ## Check if time argument empty
+    if [[ -z "$argTime" ]]; then
+       ## T: Time argument empty, use current time
+       TIME_INPUT="$TIME_CURRENT";
+    else
+       ## F: Time argument exists, validate time
+       if date --date="$argTime" 1>/dev/null 2>&1; then
+           ### T: Time argument is valid; use it
+           TIME_INPUT="$argTime";
+       else
+           ### F: Time argument not valid; exit
+           yell "ERROR:Invalid time argument supplied. Exiting."; exit 1;
+       fi;
+    fi;
+    # Construct and deliver nanoseconds since 1970-01-01    
+    TIME_EPOCH_FLOAT="$(date --date="$TIME_INPUT" +%s.%N)"; # Save ssss.NNNNNNNNN
+    TIME_EPOCH_INT="$(echo "$TIME_EPOCH_FLOAT" | cut -d. -f1)"; # Get ssss
+    TIME_EPOCH_NSFRAC="$(echo "$TIME_EPOCH_FLOAT" | cut -d. -f2)"; # Get NNNNNNNNN
+    TIME_EPOCH_NS="$(( (10#"$TIME_EPOCH_INT" * 10**9) + (10#"$TIME_EPOCH_NSFRAC") ))";
+    echo "$TIME_EPOCH_NS";
+} # Nanoseconds since 1970-01-01
+magicBufferSleepPID() {
+    # Desc: Compensates for lag so buffer rounds start every BUFFER_TTL seconds
+    # Input: vars: BUFFER_TTL, errResetx10e3, K_P, T_I, T_D
+    # # Input: array: errorHistory
+    # Output: vars: BUFFER_TTL_ADJ_FLOAT
+    # Re/Attrib: https://en.wikipedia.org/wiki/PID_controller#Standard_versus_parallel_(ideal)_form
+    local BUFFER_TTL_NS
+    local timeBufferStartNS timeBufferStartNSExp errNS errNSx10e3
+    local errResetx10e3 errRatex10e3 ADJ BUFFER_TTL_ADJ_NS BUFFER_TTL_ADJ_INT
+    local BUFFER_TTL_ADJ_FLOATFRAC
+    # local errorHistorySize
+    
+    # ## Define errorHistorySize
+    # errorHistorySize=100;
+    ## Define BUFFER_TTL in nanoseconds
+    BUFFER_TTL_NS=$((BUFFER_TTL * 10**9));
+    
+    # Calculate Error, errNS, in nanoseconds
+    ## Get current time
+    timeBufferStartNS="$(timeEpochNS)";
+    ## Calculate expected time (from start time, current buffer round number, nominal BUFFER_TTL)
+    timeBufferStartNSExp="$(( (timeBufferFirstNS) + (BUFFER_TTL_NS * bufferRound) ))";
+    ## Calculate error (diff between timeBufferStartNSExp and timeBufferStartNS; usually negative)
+    errNS="$(( timeBufferStartNSExp - timeBufferStartNS ))";
+    errNSx10e3="$((errNS*10**3))";
+    # ## Append error to errorHistory
+    # errorHistory+=("errNS");
+    # ### Trim errorHistory array if over errorHistorySize
+    # while [[ "${#errorHistory[@]}" -gt "errorHistorySize" ]]; then do
+    #  unset "errorHistory[0]"; # remove oldest entry, creating sparse array
+    #  errorHistory=("${errorHistory[@]}"); # reindex sparse array
+    #  vbm "STATUS:Trimmed errorHistory array. Entry count:${#errorHistory[@]}";
+    # done;
+
+    # Calculate errReset in nanoseconds^2
+    ## errReset = int(errHistory(t),wrt(delta_BUFFER_TTL))
+    ## Integrate errorHistory with respect to time
+    # for value in "${errorHistory[@]}"; do
+    #  errReset=$(( errReset + ( value*BUFFER_TTL_NS ) ));
+    # done;
+    errResetx10e3="$(( ( errResetx10e3 + ( errNS * BUFFER_TTL_NS ) )*10**3 ))";
+
+    # Calculate errRate in nanoseconds per 1000 nanoseconds
+    errRatex10e3="$(( ( errNSx10e3 ) / BUFFER_TTL_NS ))";
+    
+    # Calculate PID control signal
+    ## ADJ = K_P * (errNS + errReset/T_I + errRate*T_D)
+    ADJ="$((K_P*(errNSx10e3 + (errResetx10e3/T_I) + (errRatex10e3*T_D) )/10**3))";
+
+    # Calculate BUFFER_TTL_ADJ_FLOAT from ADJ (ns)
+    ## Calculate BUFFER_TTL_ADJ in nanoseconds (BUFFER_TTL_ADJ_NS = BUFFER_TTL_NS + ADJ)
+    BUFFER_TTL_ADJ_NS="$((BUFFER_TTL_NS + ADJ))";
+    ## Calculate integer seconds
+    BUFFER_TTL_ADJ_INT="$((BUFFER_TTL_ADJ_NS/(10**9)))";
+    ## Calculate nanosecond remainder
+    BUFFER_TTL_ADJ_FLOATFRAC="$((BUFFER_TTL_NS - (BUFFER_TTL_ADJ_INT*(10**9)) ))";
+    ## Form float BUFFER_TTL_ADJ_FLOAT
+    BUFFER_TTL_ADJ_FLOAT="$BUFFER_TTL_ADJ_INT"."$BUFFER_TTL_ADJ_FLOATFRAC";
+    vbm "STATUS:Calculated adjusted BUFFER_TTL (seconds):$BUFFER_TTL_ADJ_FLOAT";
+} # Calc BUFFER_TTL_ADJ_FLOAT so buffer starts every BUFFER_TTL seconds

 magicWriteVersion() {
     # Desc: Appends time-stamped VERSION to PATHOUT_TAR
     # Usage: magicWriteVersion
 magicWriteVersion() {
     # Desc: Appends time-stamped VERSION to PATHOUT_TAR
     # Usage: magicWriteVersion


@@ -927,11 +1031,16 @@ magicWriteVersion() {
 } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar()
 magicGatherWriteBuffer() {
     # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR
 } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar()
 magicGatherWriteBuffer() {
     # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR

-    # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP,
-    # Inputs: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX
-    # Depends: yell, try, vbm, appendArgTar, tar
-    local FN="${FUNCNAME[0]}";
-    wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
+    # Inputs: vars: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP,
+    # Inputs: vars: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX
+    # Output: file: (PATHOUT_TAR)
+    # Depends: yell(), try(), vbm(), appendArgTar(), tar 1, sleep 8, checkMakeTar()
+    # Depends: magicWriteVersion(), appendFileTar()
+    local FN
+
+    # Debug:Get function name
+    FN="${FUNCNAME[0]}";
+    

     # Create buffer file with unique name
     PATHOUT_BUFFER="$DIR_TMP/buffer$SECONDS";
     # Fill buffer
     # Create buffer file with unique name
     PATHOUT_BUFFER="$DIR_TMP/buffer$SECONDS";
     # Fill buffer


@@ -965,6 +1074,7 @@ magicGatherWriteBuffer() {
     if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi
 
     # Write bufferBash to PATHOUT_TAR
     if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi
 
     # Write bufferBash to PATHOUT_TAR

+    wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )

     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data
     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file
     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file
     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data
     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file
     appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file


@@ -1008,7 +1118,7 @@ magicParseRecipientDir() {
            done
            #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected
            if ! [[ "$updateRecipients" = "false" ]]; then
            done
            #### Write updated recPubKeysValid array to recPubKeysValid if no failure detected
            if ! [[ "$updateRecipients" = "false" ]]; then

-               recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[@]}\"";
+               recPubKeysValid=("${candRecPubKeysValid[@]}") && vbm "STATUS:Wrote candRecPubkeysValid to recPubKeysValid:\"${recPubKeysValid[*]}\"";

            fi;
        else
            yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1;
            fi;
        else
            yell "ERROR:$0:Recipient directory $argRecDir does not exist. Exiting."; exit 1;


@@ -1045,8 +1155,8 @@ magicParseRecipientArgs() {
                fi;
            done
            vbm "DEBUG:Finished processing argRecPubKeys array";
                fi;
            done
            vbm "DEBUG:Finished processing argRecPubKeys array";

-           vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[@]}";
-           recPubKeysValidStatic="${recPubKeysValid[@]}"; # Save static image of pubkeys validated by this function
+           vbm "STATUS:Array of validated pubkeys:${recPubKeysValid[*]}";
+           recPubKeysValidStatic=("${recPubKeysValid[@]}"); # Save static image of pubkeys validated by this function

 
            ##  Form age command string
            CMD_ENCRYPT="age ""$recipients " && vbm "CMD_ENCRYPT:$CMD_ENCRYPT";
 
            ##  Form age command string
            CMD_ENCRYPT="age ""$recipients " && vbm "CMD_ENCRYPT:$CMD_ENCRYPT";


@@ -1199,10 +1309,13 @@ main() {
     CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML
 
     # MAIN LOOP:Record gps data until script lifespan ends
     CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML
 
     # MAIN LOOP:Record gps data until script lifespan ends

+    timeBufferFirstNS="$(timeEpochNS)"; bufferRound=0

     while [[ "$SECONDS" -lt "$SCRIPT_TTL" ]]; do
     while [[ "$SECONDS" -lt "$SCRIPT_TTL" ]]; do

-       magicParseRecipientDir
+       magicParseRecipientDir;

        magicGatherWriteBuffer &
        magicGatherWriteBuffer &

-       sleep "$BUFFER_TTL";
+       magicBufferSleepPID; # Calculates BUFFER_TTL_ADJ from BUFFER_TTL given buffer expected start time vs. actual
+       sleep "$BUFFER_TTL_ADJ_FLOAT";
+       ((bufferRound++));

     done    
 
     # Cleanup
     done    
 
     # Cleanup