fix(bkgpslog):Reduce logging downtime between buffer rounds

Reduce logging downtime between buffer rounds by moving commands into
the asynchronous `magicWriteBuffer()` function that is spun off
immediately after `gpspipe` has collected data.

Add functionality to `dateTimeShort()` and `dateShort` so the start
time of each saved chunk appended to the `tar` archive can be
calculated indirectly from when `magicWriteBuffer()` starts rather
than performing slow `date` and associated bash variable changes in
the main loop. This start time is needed to name each inserted file as
well as to identify which `tar` file to insert into.

Save gpspipe buffer data `/dev/shm` instead of a Bash variable.

Add `appendFileTar()` function to facilitate reading and processing of
buffer data from `/dev/shm`.

diff --git a/exec/bkgpslog b/exec/bkgpslog
index 720f59e45fa7cb1e61ac1df4126d1066aca1aec8..c40175f3c91ae22eab838fa9e100346b01d82fbc 100755 (executable)
--- a/exec/bkgpslog
+++ b/exec/bkgpslog
@@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory
 SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N);
 PATH="$HOME/.local/bin:$PATH";   # Add "$(systemd-path user-binaries)" path in case apps saved there
 SCRIPT_HOSTNAME=$(hostname);     # Save hostname of system running this script.
-SCRIPT_VERSION="0.3.2";          # Define version of script.
+SCRIPT_VERSION="0.3.3";          # Define version of script.
 SCRIPT_NAME="bkgpslog";          # Define basename of script file.
 SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
 AGE_VERSION="1.0.0-beta2";       # Define version of age (encryption program)
@@ -334,20 +334,64 @@ timeUntilNextHour(){
 } # Report seconds until next hour
 dateTimeShort(){
     # Desc: Timestamp without separators (YYYYmmddTHHMMSS+zzzz)
-    # Usage: dateTimeShort
+    # Usage: dateTimeShort ([str date])
+    # Version 1.1.0
+    # Input: arg1: 'date'-parsable timestamp string (optional)
     # Output: stdout: timestamp (ISO-8601, no separators)
+    # Depends: yell
     local TIME_CURRENT TIME_CURRENT_SHORT
+
+    argTime="$1";
+    # Get Current Time
     TIME_CURRENT="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second.
-    TIME_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%dT%H%M%S%z)"; # Produce separator-less current timestamp with resolution 1 second.
+    # Decide to parse current or supplied date
+    ## Check if time argument empty
+    if [[ -z "$argTime" ]]; then
+       ## T: Time argument empty, use current time
+       TIME_INPUT="$TIME_CURRENT";
+    else
+       ## F: Time argument exists, validate time
+       if date --date="$argTime" 1>/dev/null 2>&1; then
+           ### T: Time argument is valid; use it
+           TIME_INPUT="$argTime";
+       else
+           ### F: Time argument not valid; exit
+           yell "ERROR:Invalid time argument supplied. Exiting."; exit 1;
+       fi
+    fi
+    # Construct and deliver separator-les date string
+    TIME_CURRENT_SHORT="$(date -d "$TIME_INPUT" +%Y%m%dT%H%M%S%z)";
     echo "$TIME_CURRENT_SHORT";
 } # Get YYYYmmddTHHMMSS±zzzz
 dateShort(){
     # Desc: Date without separators (YYYYmmdd)
-    # Usage: dateShort
+    # Usage: dateShort ([str date])
+    # Version: 1.1.0
+    # Input: arg1: 'date'-parsable timestamp string (optional)
     # Output: stdout: date (ISO-8601, no separators)
+    # Depends: yell
     local TIME_CURRENT DATE_CURRENT_SHORT
+
+    argTime="$1";
+    # Get Current Time
     TIME_CURRENT="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second.
-    DATE_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day.
+    # Decide to parse current or supplied date
+    ## Check if time argument empty
+    if [[ -z "$argTime" ]]; then
+       ## T: Time argument empty, use current time
+       TIME_INPUT="$TIME_CURRENT";
+    else
+       ## F: Time argument exists, validate time
+       if date --date="$argTime" 1>/dev/null 2>&1; then
+           ### T: Time argument is valid; use it
+           TIME_INPUT="$argTime";
+       else
+           ### F: Time argument not valid; exit
+           yell "ERROR:Invalid time argument supplied. Exiting."; exit 1;
+       fi
+    fi
+    # Construct and deliver separator-les date string    
+    DATE_CURRENT_SHORT="$(date -d "$TIME_INPUT" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day.
     echo "$DATE_CURRENT_SHORT";
 } # Get YYYYmmdd
 timeDuration(){
@@ -669,8 +713,8 @@ checkMakeTar() {
 } # checks if arg1 is tar; creates one otherwise
 appendArgTar(){
     # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar
-    # Usage: writeArg "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...)
-    # Version: 1.0.0
+    # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...)
+    # Version: 1.0.2
     # Input: arg1: data to be written
     #        arg2: file name of file to be inserted into tar
     #        arg3: tar archive path (must exist first)
@@ -718,6 +762,53 @@ appendArgTar(){
     try tar --append --directory="$TMP_DIR" --file="$TAR_PATH" "$FILENAME";
     #yell "DEBUG:STATUS:$FN:Finished appendArgTar()."
 } # Append Bash var to file appended to Tar archive
+appendFileTar(){
+    # Desc: Processes first file and then appends to tar
+    # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...)
+    # Version: 1.0.1
+    # Input: arg1: path of file to be (processed and) written
+    #        arg2: name to use for file inserted into tar
+    #        arg3: tar archive path (must exist first)
+    #        arg4: temporary working dir
+    #        arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ")
+    # Output: file written to disk
+    # Example: decrypt multiple large files in parallel
+    #          appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" &
+    #          appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" &
+    #          appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" &
+    # Depends: bash 5
+
+    # Save function name
+    local FN="${FUNCNAME[0]}";
+    #yell "DEBUG:STATUS:$FN:Finished appendFileTar()."
+    
+    # Set file name
+    if ! [ -z "$2" ]; then FILENAME="$2"; else yell "ERROR:$FN:Not enough arguments."; exit 1; fi
+    # Check tar path is a file
+    if [ -f "$3" ]; then TAR_PATH="$3"; else yell "ERROR:$FN:Tar archive arg not a file."; exit 1; fi
+    # Check temp dir arg
+    if ! [ -z "$4" ]; then TMP_DIR="$4"; else yell "ERROR:$FN:No temporary working dir set."; exit 1; fi
+    # Set command strings
+    if ! [ -z "$5" ]; then CMD1="$5"; else CMD1="tee /dev/null "; fi # command string 1
+    if ! [ -z "$6" ]; then CMD2="$6"; else CMD2="tee /dev/null "; fi # command string 2
+    if ! [ -z "$7" ]; then CMD3="$7"; else CMD3="tee /dev/null "; fi # command string 3
+    if ! [ -z "$8" ]; then CMD4="$8"; else CMD4="tee /dev/null "; fi # command string 4
+    # # Debug
+    # yell "DEBUG:STATUS:$FN:CMD1:$CMD1"
+    # yell "DEBUG:STATUS:$FN:CMD2:$CMD2"
+    # yell "DEBUG:STATUS:$FN:CMD3:$CMD3"
+    # yell "DEBUG:STATUS:$FN:CMD4:$CMD4"
+    # yell "DEBUG:STATUS:$FN:FILENAME:$FILENAME"
+    # yell "DEBUG:STATUS:$FN:TAR_PATH:$TAR_PATH"
+    # yell "DEBUG:STATUS:$FN:TMP_DIR:$TMP_DIR"
+    
+    # Write to temporary working dir
+    cat "$1" | $CMD1 | $CMD2 | $CMD3 | $CMD4 > "$TMP_DIR"/"$FILENAME";
+
+    # Append to tar
+    try tar --append --directory="$TMP_DIR" --file="$TAR_PATH" "$FILENAME";
+    #yell "DEBUG:STATUS:$FN:Finished appendFileTar()."
+} # Append file to Tar archive
 magicWriteVersion() {
     # Desc: Appends time-stamped VERSION to PATHOUT_TAR
     # Usage: magicWriteVersion
@@ -754,25 +845,49 @@ magicWriteVersion() {
 } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar()
 magicWriteBuffer() {
     # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR
-    # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP
+    # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP,
+    # Inputs: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX
     # Depends: yell, try, vbm, appendArgTar, tar
     local FN="${FUNCNAME[0]}";
     wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
-    vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer().";
 
+    timeBufferStart="$(dateTimeShort  "$(date --date="$BUFFER_TTL seconds ago")")"; # Note start time#TODO subtract BUFFER_TTL from current time
+    vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer().";
+    # Determine file paths (time is start of buffer period)
+    FILEOUT_BASENAME="$timeBufferStart""--""$bufferTTL_STR""..""$SCRIPT_HOSTNAME""_location" && vbm "STATUS:Set FILEOUT_BASENAME to:$FILEOUT_BASENAME";
+    ## Files saved to DIR_TMP
+    FILEOUT_NMEA="$FILEOUT_BASENAME".nmea"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_NMEA to:$FILEOUT_NMEA";
+    FILEOUT_GPX="$FILEOUT_BASENAME".gpx"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_GPX to:$FILEOUT_GPX";
+    FILEOUT_KML="$FILEOUT_BASENAME".kml"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_KML to:$FILEOUT_KML";
+    PATHOUT_NMEA="$DIR_TMP"/"$FILEOUT_NMEA" && vbm "STATUS:Set PATHOUT_NMEA to:$PATHOUT_NMEA";
+    PATHOUT_GPX="$DIR_TMP"/"$FILEOUT_GPX" && vbm "STATUS:Set PATHOUT_GPX to:$PATHOUT_GPX";
+    PATHOUT_KML="$DIR_TMP"/"$FILEOUT_KML" && vbm "STATUS:Set PATHOUT_KML to:$PATHOUT_KML";
+    ## Files saved to disk (DIR_OUT)
+    ### one file per day (Ex: "20200731..hostname_location.[.gpx.gz].tar")
+    PATHOUT_TAR="$DIR_OUT"/"$(dateShort "$(date --date="$BUFFER_TTL seconds ago")")".."$SCRIPT_HOSTNAME""_location""$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX".tar && \
+       vbm "STATUS:Set PATHOUT_TAR to:$PATHOUT_TAR";
+    # DEBUG: check vars
+    vbm "STATUS:DIR_TMP     :$DIR_TMP";
+    vbm "STATUS:PATHOUT_TAR :$PATHOUT_TAR";
+    vbm "STATUS:PATHOUT_NMEA:$PATHOUT_NMEA";
+    vbm "STATUS:PATHOUT_GPX:$PATHOUT_GPX";
+    vbm "STATUS:PATHOUT_KML:$PATHOUT_KML";
+    
+    
     # Validate PATHOUT_TAR as tar.
     checkMakeTar "$PATHOUT_TAR";
     ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2)
     if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi
 
     # Write bufferBash to PATHOUT_TAR
-    appendArgTar "$bufferBash" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data
-    appendArgTar "$bufferBash" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file
-    appendArgTar "$bufferBash" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file
+    appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data
+    appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file
+    appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file
+
     # Remove secured chunks from DIR_TMP
-    try rm "$PATHOUT_NMEA" "$PATHOUT_GPX" "$PATHOUT_KML";
-    # yell "DEBUG:STATUS:$FN:Finished magicWriteBuffer().";
-} # bkgpslog write function
+    rm "$PATHOUT_BUFFER" "$PATHOUT_NMEA" "$PATHOUT_GPX" "$PATHOUT_KML";
+    vbm "DEBUG:STATUS:$FN:Finished magicWriteBuffer().";
+} # write buffer to disk
 
 main() {
     processArguments "$@" # Process arguments.
@@ -866,47 +981,26 @@ main() {
     checkMakeTar "$PATHOUT_TAR" && vbm "DEBUG:Confirmed or Created to be a tar:$PATHOUT_TAR";
     ## Append VERSION file to PATHOUT_TAR
     magicWriteVersion;
+
+    # Define GPS conversion commands
+    CMD_CONV_NMEA="tee /dev/null " && vbm "STATUS:Set CMD_CONV_NMEA to:$CMD_CONV_NMEA"; # tee as passthrough
+    CMD_CONV_GPX="gpsbabel -i nmea -f - -o gpx -F - " && vbm "STATUS:Set CMD_CONV_GPX to:$CMD_CONV_GPX"; # convert NMEA to GPX
+    CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML
     
-    # Record gps data until script lifespan ends
+    # MAIN LOOP:Record gps data until script lifespan ends
     declare debugCounter; debugCounter="0"; # set debug counter
     while [[ "$SECONDS" -lt "$scriptTTL" ]]; do
-       timeBufferStart="$(dateTimeShort)"; # Note start time
-       # Determine file paths (time is start of buffer period)
-       FILEOUT_BASENAME="$timeBufferStart""--""$bufferTTL_STR""..""$SCRIPT_HOSTNAME""_location" && vbm "STATUS:Set FILEOUT_BASENAME to:$FILEOUT_BASENAME";
-       ## Files saved to DIR_TMP
-       FILEOUT_NMEA="$FILEOUT_BASENAME".nmea"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_NMEA to:$FILEOUT_NMEA";
-       FILEOUT_GPX="$FILEOUT_BASENAME".gpx"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_GPX to:$FILEOUT_GPX";
-       FILEOUT_KML="$FILEOUT_BASENAME".kml"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_KML to:$FILEOUT_KML";
-       PATHOUT_NMEA="$DIR_TMP"/"$FILEOUT_NMEA" && vbm "STATUS:Set PATHOUT_NMEA to:$PATHOUT_NMEA";
-       PATHOUT_GPX="$DIR_TMP"/"$FILEOUT_GPX" && vbm "STATUS:Set PATHOUT_GPX to:$PATHOUT_GPX";
-       PATHOUT_KML="$DIR_TMP"/"$FILEOUT_KML" && vbm "STATUS:Set PATHOUT_KML to:$PATHOUT_KML";
-       ## Files saved to disk (DIR_OUT)
-       ### one file per day (Ex: "20200731..hostname_location.[.gpx.gz].tar")
-       PATHOUT_TAR="$DIR_OUT"/"$(dateShort)".."$SCRIPT_HOSTNAME""_location""$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX".tar && \
-           vbm "STATUS:Set PATHOUT_TAR to:$PATHOUT_TAR";
-       # Define GPS conversion commands
-       CMD_CONV_NMEA="tee /dev/null " && vbm "STATUS:Set CMD_CONV_NMEA to:$CMD_CONV_NMEA"; # tee as passthrough
-       CMD_CONV_GPX="gpsbabel -i nmea -f - -o gpx -F - " && vbm "STATUS:Set CMD_CONV_GPX to:$CMD_CONV_GPX"; # convert NMEA to GPX
-       CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML
+       # Create buffer file with unique name
+       PATHOUT_BUFFER="$DIR_TMP/buffer$debugCounter++";
        # Fill Bash variable buffer
-       bufferBash="$(timeout "$BUFFER_TTL""s" gpspipe -r)" && vbm "STATUS:Successfully filled bufferBash variable with gpspipe data."; # Record gpspipe nmea data to buffer for bufferTTL seconds
+       timeout "$BUFFER_TTL"s gpspipe -r "$PATHOUT_BUFFER" ;
        # Process bufferBash, save secured chunk set to DIR_TMP
-       vbm "STATUS:Beginning to save data to $DIR_TMP";
        magicWriteBuffer &
-       # Append each secured chunk in memory dir (DIR_TMP) to file on disk (PATHOUT_TAR in DIR_OUT)
-       vbm "STATUS:DIR_TMP     :$DIR_TMP";
-       vbm "STATUS:PATHOUT_TAR :$PATHOUT_TAR";
-       vbm "STATUS:PATHOUT_NMEA:$PATHOUT_NMEA";
-       vbm "STATUS:PATHOUT_GPX:$PATHOUT_GPX";
-       vbm "STATUS:PATHOUT_KML:$PATHOUT_KML";
-
-       # Reset buffer and filenames
-       unset bufferBash FILEOUT_BASENAME PATHOUT_NMEA PATHOUT_GPX PATHOUT_KML PATHOUT_TAR timeBufferStart;
-       vbm "DEBUG:Completed buffer session $debugCounter ." 1>&2;
-       ((debugCounter++))
+       ((debugCounter++));
     done
 
-    # Remove DIR_TMP
+    # Cleanup
+    ## Remove DIR_TMP
     try rm -r "$DIR_TMP";
 
     vbm "STATUS:Main function finished.";