From a7b4a273f64d0aafc939d2e31fe67fce03123845 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 21:54:26 +0000 Subject: [PATCH 01/16] style(bkgpslog):Update timeDuration --- exec/bkgpslog | 53 +++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 97231f8..723960f 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -351,59 +351,61 @@ dateShort(){ echo "$DATE_CURRENT_SHORT"; } # Get YYYYmmdd timeDuration(){ - # Desc: Output approximate time duration string before given time (default:current date) + # Desc: Given seconds, output ISO-8601 duration string # Ref/Attrib: ISO-8601:2004(E), §4.4.4.2 Representations of time intervals by duration and context information # Note: "1 month" ("P1M") is assumed to be "30 days" (see ISO-8601:2004(E), §2.2.1.2) - # Usage: timeDuration [arg1] ([arg2]) - # Version: 1.0.1 + # Usage: timeDuration [1:seconds] ([2:precision]) + # Version: 1.0.3 # Input: arg1: seconds as base 10 integer >= 0 (ex: 3601) # arg2: precision level (optional; default=2) # Output: stdout: ISO-8601 duration string (ex: "P1H1S", "P2Y10M15DT10H30M20S") + # exit code 0: success + # exit code 1: error_input + # exit code 2: error_unknown # Example: 'timeDuration 111111 3' yields 'P1DT6H51M' # Depends: date 8 (gnucoreutils), yell, - local returnState ARG1 ARG2 remainder precision witherPrecision + local returnState argSeconds argPrecision remainder precision witherPrecision local fullYears fullMonths fullDays fullHours fullMinutes fullSeconds local displayYears displayMonths displayDays displayHours displayMinutes displaySeconds local hasYears hasMonths hasDays hasHours hasMinutes hasSeconds - ARG1="$1"; - ARG2="$2"; + argSeconds="$1"; # read arg1 (seconds) + argPrecision="$2"; # read arg2 (precision) precision=2; # set default precision - returnState="true"; # set default return state # Check that between one and two arguments is supplied if ! { [[ $# -ge 1 ]] && [[ $# -le 2 ]]; }; then yell "ERROR:Invalid number of arguments:$# . Exiting."; - returnState="ERROR_INPUT"; fi + returnState="error_input"; fi - # Check that arg1 provided + # Check that argSeconds provided if [[ $# -ge 1 ]]; then - ## Check that arg1 is a positive integer - if [[ "$ARG1" =~ ^[[:digit:]]+$ ]]; then + ## Check that argSeconds is a positive integer + if [[ "$argSeconds" =~ ^[[:digit:]]+$ ]]; then : else - yell "ERROR:ARG1 not a digit."; - returnState="ERROR_INPUT"; + yell "ERROR:argSeconds not a digit."; + returnState="error_input"; fi else yell "ERROR:No argument provided. Exiting."; exit 1; fi - # Consider whether arg2 was provided + # Consider whether argPrecision was provided if [[ $# -eq 2 ]]; then - # Check that the second arg is a positive integer - if [[ "$ARG2" =~ ^[[:digit:]]+$ ]] && [[ "$ARG2" -gt 0 ]]; then - precision="$ARG2"; + # Check that argPrecision is a positive integer + if [[ "$argPrecision" =~ ^[[:digit:]]+$ ]] && [[ "$argPrecision" -gt 0 ]]; then + precision="$argPrecision"; else - yell "ERROR:ARG2 not a positive integer. (is $ARG2 ). Leaving early."; - returnState="ERROR_INPUT"; + yell "ERROR:argPrecision not a positive integer. (is $argPrecision ). Leaving early."; + returnState="error_input"; fi; else : fi; - remainder="$ARG1" ; # seconds + remainder="$argSeconds" ; # seconds ## Calculate full years Y, update remainder fullYears=$(( remainder / (365*24*60*60) )); remainder=$(( remainder - (fullYears*365*24*60*60) )); @@ -503,8 +505,6 @@ timeDuration(){ fi; if $witherPrecision; then ((precision--)); fi; - - ## Determine whether or not the "T" separator is needed to separate date and time elements if ( $displayHours || $displayMinutes || $displaySeconds); then displayDateTime="true"; else displayDateTime="false"; fi @@ -527,14 +527,17 @@ timeDuration(){ OUTPUT=$OUTPUT$fullSeconds"S"; fi ## Output duration string to stdout - if [[ "$returnState" = "true" ]]; then echo "$OUTPUT"; fi + echo "$OUTPUT" && returnState="true"; #===Determine function return code=== if [ "$returnState" = "true" ]; then return 0; - else - echo "$returnState" 1>&2; + elif [ "$returnState" = "error_input" ]; then + yell "ERROR:input"; return 1; + else + yell "ERROR:Unknown"; + return 2; fi } # Get duration (ex: PT10M4S ) -- 2.30.2 From aa2d792160da84c9acae4c9a3b48211826f93b82 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 21:55:50 +0000 Subject: [PATCH 02/16] fix(bkgpslog):Clarify echoed newlines per Shellcheck Changed echo "blah""\n""blah" into echo "blah""\\n""blah" --- exec/bkgpslog | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 723960f..ac375b8 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.2.2"; # Define version of script. +SCRIPT_VERSION="0.2.3"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -731,17 +731,17 @@ magicWriteVersion() { # Gather VERSION data in CONTENT_VERSION CONTENT_VERSION="SCRIPT_VERSION=$SCRIPT_VERSION"; - #CONTENT_VERSION="$CONTENT_VERSION""\n"; - CONTENT_VERSION="$CONTENT_VERSION""\n""SCRIPT_NAME=$SCRIPT_NAME"; - CONTENT_VERSION="$CONTENT_VERSION""\n""SCRIPT_URL=$SCRIPT_URL"; - CONTENT_VERSION="$CONTENT_VERSION""\n""AGE_VERSION=$AGE_VERSION"; - CONTENT_VERSION="$CONTENT_VERSION""\n""AGE_URL=$AGE_URL"; - CONTENT_VERSION="$CONTENT_VERSION""\n""DATE=$(date --iso-8601=seconds)"; - CONTENT_VERSION="$CONTENT_VERSION""\n""HOSTNAME=$SCRIPT_HOSTNAME"; + #CONTENT_VERSION="$CONTENT_VERSION""\\n"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""SCRIPT_NAME=$SCRIPT_NAME"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""SCRIPT_URL=$SCRIPT_URL"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""AGE_VERSION=$AGE_VERSION"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""AGE_URL=$AGE_URL"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""DATE=$(date --iso-8601=seconds)"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""HOSTNAME=$SCRIPT_HOSTNAME"; ## Add list of recipient pubkeys for pubkey in "${recPubKeysValid[@]}"; do ((pubKeyIndex++)) - CONTENT_VERSION="$CONTENT_VERSION""\n""PUBKEY_$pubKeyIndex=$pubkey"; + CONTENT_VERSION="$CONTENT_VERSION""\\n""PUBKEY_$pubKeyIndex=$pubkey"; done ## Process newline escapes CONTENT_VERSION="$(echo -e "$CONTENT_VERSION")" -- 2.30.2 From a96495d38f01cbb94cbe7f296bf7f59fe8297005 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 22:00:42 +0000 Subject: [PATCH 03/16] test(bkgpslog):Bump version number to 0.3.0 Incompatible changes from 0.2: - File writes asynchronous from main script forced to be sequential to one another to avoid simultaneous writes to same file. Tar write code consolidated. - VERSION file format changed to include more fields. VERSION file generation code cosolidated. --- exec/bkgpslog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index ac375b8..639a7a5 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.2.3"; # Define version of script. +SCRIPT_VERSION="0.3.0"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) -- 2.30.2 From d7138c7fbf2bd7aadb9ccd22f85c227b9a59300d Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 22:12:58 +0000 Subject: [PATCH 04/16] style(bkgpslog):checkMakeTar():Add/remove debug code --- exec/bkgpslog | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 639a7a5..8b3b67e 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -670,6 +670,7 @@ checkMakeTar() { appendArgTar(){ # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar # Usage: writeArg "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) + # Version: 1.0.0 # Input: arg1: data to be written # arg2: file name of file to be inserted into tar # arg3: tar archive path (must exist first) @@ -683,8 +684,8 @@ appendArgTar(){ # Depends: bash 5 # Save function name - local FN="${FUNCNAME[0]}" - yell "DEBUG:STATUS:$FN:Finished appendArgTar()." + local FN="${FUNCNAME[0]}"; + #yell "DEBUG:STATUS:$FN:Finished appendArgTar()." # Set file name if ! [ -z "$2" ]; then FILENAME="$2"; else yell "ERROR:$FN:Not enough arguments."; exit 1; fi @@ -701,20 +702,21 @@ appendArgTar(){ if ! [ -z "$7" ]; then CMD3="$7"; else CMD3="tee /dev/null "; fi # command string 3 if ! [ -z "$8" ]; then CMD4="$8"; else CMD4="tee /dev/null "; fi # command string 4 - # Debug - yell "STATUS:$FN:CMD1:$CMD1" - yell "STATUS:$FN:CMD2:$CMD2" - yell "STATUS:$FN:CMD3:$CMD3" - yell "STATUS:$FN:CMD4:$CMD4" - yell "STATUS:$FN:FILENAME:$FILENAME" - yell "STATUS:$FN:TAR_PATH:$TAR_PATH" - yell "STATUS:$FN:TMP_DIR:$TMP_DIR" + # # Debug + # yell "DEBUG:STATUS:$FN:CMD1:$CMD1" + # yell "DEBUG:STATUS:$FN:CMD2:$CMD2" + # yell "DEBUG:STATUS:$FN:CMD3:$CMD3" + # yell "DEBUG:STATUS:$FN:CMD4:$CMD4" + # yell "DEBUG:STATUS:$FN:FILENAME:$FILENAME" + # yell "DEBUG:STATUS:$FN:TAR_PATH:$TAR_PATH" + # yell "DEBUG:STATUS:$FN:TMP_DIR:$TMP_DIR" + # Write to temporary working dir echo "$1" | $CMD1 | $CMD2 | $CMD3 | $CMD4 > "$TMP_DIR"/"$FILENAME"; # Append to tar try tar --append --directory="$TMP_DIR" --file="$TAR_PATH" "$FILENAME"; - yell "DEBUG:STATUS:$FN:Finished appendArgTar()." + #yell "DEBUG:STATUS:$FN:Finished appendArgTar()." } # Append Bash var to file appended to Tar archive magicWriteVersion() { # Desc: Appends time-stamped VERSION to PATHOUT_TAR -- 2.30.2 From ea4821d62b4a15c579da5941090f8c5df30c2a17 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 22:15:45 +0000 Subject: [PATCH 05/16] fix(bkgpslog):Remove debug yells --- exec/bkgpslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 8b3b67e..6a4e0d1 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.0"; # Define version of script. +SCRIPT_VERSION="0.3.1"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -771,7 +771,7 @@ magicWriteBuffer() { appendArgTar "$bufferBash" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file # Remove secured chunks from DIR_TMP try rm "$PATHOUT_NMEA" "$PATHOUT_GPX" "$PATHOUT_KML"; - yell "DEBUG:STATUS:$FN:Finished magicWriteBuffer()."; + # yell "DEBUG:STATUS:$FN:Finished magicWriteBuffer()."; } # bkgpslog write function main() { -- 2.30.2 From 3fd6a69e54c4b0b428b0073d28a77f690ea6921f Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 2 Jul 2020 23:46:10 +0000 Subject: [PATCH 06/16] fix(bkgpslog):Increase buffer period 60s to 5m to reduce write freq --- exec/bkgpslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 6a4e0d1..720f59e 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -6,7 +6,7 @@ #==BEGIN Define script parameters== ## Logging Behavior parameters -BUFFER_TTL="60"; # time between file writes +BUFFER_TTL="300"; # time between file writes SCRIPT_TTL="day"; # (day|hour) #### TZ="UTC"; export TZ; # Default time zone; overridden by '--time-zone=[str]' option DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.1"; # Define version of script. +SCRIPT_VERSION="0.3.2"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) -- 2.30.2 From f75428fef49de12c858fe5c8dd0ac4fca1d09a82 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 17:16:28 +0000 Subject: [PATCH 07/16] doc(bkgpslog):Add note to simplify main loop --- exec/bkgpslog-plan.org | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/exec/bkgpslog-plan.org b/exec/bkgpslog-plan.org index da23704..b7cc736 100644 --- a/exec/bkgpslog-plan.org +++ b/exec/bkgpslog-plan.org @@ -27,11 +27,27 @@ when a new session is started. is checked and its status as a valid tar file is validated. This was done using a new function ~checkMakeTar~. ** DONE Add VERSION if output tar deleted between writes + CLOSED: [2020-07-02 Thu 20:22] 2020-07-02T20:21Z; bktei> Added bkgpslog-specified function magicWriteVersion() to be called whenever a new time-stamped ~VERSION~ file needs to be generated and appended to the output tar file ~PATHOUT_TAR~. +** TODO Rewrite buffer loop to reduce lag between gpspipe runs +2020-07-03T17:10Z; bktei> As is, there is still a 5-6 second lag +between when ~gpspipe~ times out at the end of a buffer round and when +~gpspipe~ is called by the subsequent buffer round. I believe this can +be reduced by moving variable manipulations inside the +asynchronously-executed magicWriteBuffer() function. Ideally, the +while loop should look like: + +#+BEGIN_EXAMPLE +while( $SECONDS < $SCRIPT_TTL); do + gpspipe-r > "$DIR_TMP"/buffer.nmea + writeBuffer & +done +#+END_EXAMPLE + * bkgpslog narrative ** Initialize environment *** Init variables -- 2.30.2 From e47e80485f3c3192111ecf88bb09ae5d192cc208 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 19:49:50 +0000 Subject: [PATCH 08/16] fix(bkgpslog):Reduce logging downtime between buffer rounds Reduce logging downtime between buffer rounds by moving commands into the asynchronous `magicWriteBuffer()` function that is spun off immediately after `gpspipe` has collected data. Add functionality to `dateTimeShort()` and `dateShort` so the start time of each saved chunk appended to the `tar` archive can be calculated indirectly from when `magicWriteBuffer()` starts rather than performing slow `date` and associated bash variable changes in the main loop. This start time is needed to name each inserted file as well as to identify which `tar` file to insert into. Save gpspipe buffer data `/dev/shm` instead of a Bash variable. Add `appendFileTar()` function to facilitate reading and processing of buffer data from `/dev/shm`. --- exec/bkgpslog | 190 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 142 insertions(+), 48 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 720f59e..c40175f 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.2"; # Define version of script. +SCRIPT_VERSION="0.3.3"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -334,20 +334,64 @@ timeUntilNextHour(){ } # Report seconds until next hour dateTimeShort(){ # Desc: Timestamp without separators (YYYYmmddTHHMMSS+zzzz) - # Usage: dateTimeShort + # Usage: dateTimeShort ([str date]) + # Version 1.1.0 + # Input: arg1: 'date'-parsable timestamp string (optional) # Output: stdout: timestamp (ISO-8601, no separators) + # Depends: yell local TIME_CURRENT TIME_CURRENT_SHORT + + argTime="$1"; + # Get Current Time TIME_CURRENT="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. - TIME_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%dT%H%M%S%z)"; # Produce separator-less current timestamp with resolution 1 second. + # Decide to parse current or supplied date + ## Check if time argument empty + if [[ -z "$argTime" ]]; then + ## T: Time argument empty, use current time + TIME_INPUT="$TIME_CURRENT"; + else + ## F: Time argument exists, validate time + if date --date="$argTime" 1>/dev/null 2>&1; then + ### T: Time argument is valid; use it + TIME_INPUT="$argTime"; + else + ### F: Time argument not valid; exit + yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; + fi + fi + # Construct and deliver separator-les date string + TIME_CURRENT_SHORT="$(date -d "$TIME_INPUT" +%Y%m%dT%H%M%S%z)"; echo "$TIME_CURRENT_SHORT"; } # Get YYYYmmddTHHMMSS±zzzz dateShort(){ # Desc: Date without separators (YYYYmmdd) - # Usage: dateShort + # Usage: dateShort ([str date]) + # Version: 1.1.0 + # Input: arg1: 'date'-parsable timestamp string (optional) # Output: stdout: date (ISO-8601, no separators) + # Depends: yell local TIME_CURRENT DATE_CURRENT_SHORT + + argTime="$1"; + # Get Current Time TIME_CURRENT="$(date --iso-8601=seconds)" ; # Produce `date`-parsable current timestamp with resolution of 1 second. - DATE_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day. + # Decide to parse current or supplied date + ## Check if time argument empty + if [[ -z "$argTime" ]]; then + ## T: Time argument empty, use current time + TIME_INPUT="$TIME_CURRENT"; + else + ## F: Time argument exists, validate time + if date --date="$argTime" 1>/dev/null 2>&1; then + ### T: Time argument is valid; use it + TIME_INPUT="$argTime"; + else + ### F: Time argument not valid; exit + yell "ERROR:Invalid time argument supplied. Exiting."; exit 1; + fi + fi + # Construct and deliver separator-les date string + DATE_CURRENT_SHORT="$(date -d "$TIME_INPUT" +%Y%m%d)"; # Produce separator-less current date with resolution 1 day. echo "$DATE_CURRENT_SHORT"; } # Get YYYYmmdd timeDuration(){ @@ -669,8 +713,8 @@ checkMakeTar() { } # checks if arg1 is tar; creates one otherwise appendArgTar(){ # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar - # Usage: writeArg "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) - # Version: 1.0.0 + # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) + # Version: 1.0.2 # Input: arg1: data to be written # arg2: file name of file to be inserted into tar # arg3: tar archive path (must exist first) @@ -718,6 +762,53 @@ appendArgTar(){ try tar --append --directory="$TMP_DIR" --file="$TAR_PATH" "$FILENAME"; #yell "DEBUG:STATUS:$FN:Finished appendArgTar()." } # Append Bash var to file appended to Tar archive +appendFileTar(){ + # Desc: Processes first file and then appends to tar + # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...) + # Version: 1.0.1 + # Input: arg1: path of file to be (processed and) written + # arg2: name to use for file inserted into tar + # arg3: tar archive path (must exist first) + # arg4: temporary working dir + # arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ") + # Output: file written to disk + # Example: decrypt multiple large files in parallel + # appendFileTar /tmp/largefile1.gpg "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendFileTar /tmp/largefile2.gpg "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" & + # appendFileTar /tmp/largefile3.gpg "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" & + # Depends: bash 5 + + # Save function name + local FN="${FUNCNAME[0]}"; + #yell "DEBUG:STATUS:$FN:Finished appendFileTar()." + + # Set file name + if ! [ -z "$2" ]; then FILENAME="$2"; else yell "ERROR:$FN:Not enough arguments."; exit 1; fi + # Check tar path is a file + if [ -f "$3" ]; then TAR_PATH="$3"; else yell "ERROR:$FN:Tar archive arg not a file."; exit 1; fi + # Check temp dir arg + if ! [ -z "$4" ]; then TMP_DIR="$4"; else yell "ERROR:$FN:No temporary working dir set."; exit 1; fi + # Set command strings + if ! [ -z "$5" ]; then CMD1="$5"; else CMD1="tee /dev/null "; fi # command string 1 + if ! [ -z "$6" ]; then CMD2="$6"; else CMD2="tee /dev/null "; fi # command string 2 + if ! [ -z "$7" ]; then CMD3="$7"; else CMD3="tee /dev/null "; fi # command string 3 + if ! [ -z "$8" ]; then CMD4="$8"; else CMD4="tee /dev/null "; fi # command string 4 + # # Debug + # yell "DEBUG:STATUS:$FN:CMD1:$CMD1" + # yell "DEBUG:STATUS:$FN:CMD2:$CMD2" + # yell "DEBUG:STATUS:$FN:CMD3:$CMD3" + # yell "DEBUG:STATUS:$FN:CMD4:$CMD4" + # yell "DEBUG:STATUS:$FN:FILENAME:$FILENAME" + # yell "DEBUG:STATUS:$FN:TAR_PATH:$TAR_PATH" + # yell "DEBUG:STATUS:$FN:TMP_DIR:$TMP_DIR" + + # Write to temporary working dir + cat "$1" | $CMD1 | $CMD2 | $CMD3 | $CMD4 > "$TMP_DIR"/"$FILENAME"; + + # Append to tar + try tar --append --directory="$TMP_DIR" --file="$TAR_PATH" "$FILENAME"; + #yell "DEBUG:STATUS:$FN:Finished appendFileTar()." +} # Append file to Tar archive magicWriteVersion() { # Desc: Appends time-stamped VERSION to PATHOUT_TAR # Usage: magicWriteVersion @@ -754,25 +845,49 @@ magicWriteVersion() { } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar() magicWriteBuffer() { # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR - # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP + # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP, + # Inputs: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX # Depends: yell, try, vbm, appendArgTar, tar local FN="${FUNCNAME[0]}"; wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) - vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer()."; + timeBufferStart="$(dateTimeShort "$(date --date="$BUFFER_TTL seconds ago")")"; # Note start time#TODO subtract BUFFER_TTL from current time + vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer()."; + # Determine file paths (time is start of buffer period) + FILEOUT_BASENAME="$timeBufferStart""--""$bufferTTL_STR""..""$SCRIPT_HOSTNAME""_location" && vbm "STATUS:Set FILEOUT_BASENAME to:$FILEOUT_BASENAME"; + ## Files saved to DIR_TMP + FILEOUT_NMEA="$FILEOUT_BASENAME".nmea"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_NMEA to:$FILEOUT_NMEA"; + FILEOUT_GPX="$FILEOUT_BASENAME".gpx"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_GPX to:$FILEOUT_GPX"; + FILEOUT_KML="$FILEOUT_BASENAME".kml"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_KML to:$FILEOUT_KML"; + PATHOUT_NMEA="$DIR_TMP"/"$FILEOUT_NMEA" && vbm "STATUS:Set PATHOUT_NMEA to:$PATHOUT_NMEA"; + PATHOUT_GPX="$DIR_TMP"/"$FILEOUT_GPX" && vbm "STATUS:Set PATHOUT_GPX to:$PATHOUT_GPX"; + PATHOUT_KML="$DIR_TMP"/"$FILEOUT_KML" && vbm "STATUS:Set PATHOUT_KML to:$PATHOUT_KML"; + ## Files saved to disk (DIR_OUT) + ### one file per day (Ex: "20200731..hostname_location.[.gpx.gz].tar") + PATHOUT_TAR="$DIR_OUT"/"$(dateShort "$(date --date="$BUFFER_TTL seconds ago")")".."$SCRIPT_HOSTNAME""_location""$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX".tar && \ + vbm "STATUS:Set PATHOUT_TAR to:$PATHOUT_TAR"; + # DEBUG: check vars + vbm "STATUS:DIR_TMP :$DIR_TMP"; + vbm "STATUS:PATHOUT_TAR :$PATHOUT_TAR"; + vbm "STATUS:PATHOUT_NMEA:$PATHOUT_NMEA"; + vbm "STATUS:PATHOUT_GPX:$PATHOUT_GPX"; + vbm "STATUS:PATHOUT_KML:$PATHOUT_KML"; + + # Validate PATHOUT_TAR as tar. checkMakeTar "$PATHOUT_TAR"; ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2) if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi # Write bufferBash to PATHOUT_TAR - appendArgTar "$bufferBash" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data - appendArgTar "$bufferBash" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file - appendArgTar "$bufferBash" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file + appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_NMEA" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_NMEA" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write NMEA data + appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_GPX" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_GPX" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write GPX file + appendFileTar "$PATHOUT_BUFFER" "$FILEOUT_KML" "$PATHOUT_TAR" "$DIR_TMP" "$CMD_CONV_KML" "$CMD_COMPRESS" "$CMD_ENCRYPT"; # Write KML file + # Remove secured chunks from DIR_TMP - try rm "$PATHOUT_NMEA" "$PATHOUT_GPX" "$PATHOUT_KML"; - # yell "DEBUG:STATUS:$FN:Finished magicWriteBuffer()."; -} # bkgpslog write function + rm "$PATHOUT_BUFFER" "$PATHOUT_NMEA" "$PATHOUT_GPX" "$PATHOUT_KML"; + vbm "DEBUG:STATUS:$FN:Finished magicWriteBuffer()."; +} # write buffer to disk main() { processArguments "$@" # Process arguments. @@ -866,47 +981,26 @@ main() { checkMakeTar "$PATHOUT_TAR" && vbm "DEBUG:Confirmed or Created to be a tar:$PATHOUT_TAR"; ## Append VERSION file to PATHOUT_TAR magicWriteVersion; + + # Define GPS conversion commands + CMD_CONV_NMEA="tee /dev/null " && vbm "STATUS:Set CMD_CONV_NMEA to:$CMD_CONV_NMEA"; # tee as passthrough + CMD_CONV_GPX="gpsbabel -i nmea -f - -o gpx -F - " && vbm "STATUS:Set CMD_CONV_GPX to:$CMD_CONV_GPX"; # convert NMEA to GPX + CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML - # Record gps data until script lifespan ends + # MAIN LOOP:Record gps data until script lifespan ends declare debugCounter; debugCounter="0"; # set debug counter while [[ "$SECONDS" -lt "$scriptTTL" ]]; do - timeBufferStart="$(dateTimeShort)"; # Note start time - # Determine file paths (time is start of buffer period) - FILEOUT_BASENAME="$timeBufferStart""--""$bufferTTL_STR""..""$SCRIPT_HOSTNAME""_location" && vbm "STATUS:Set FILEOUT_BASENAME to:$FILEOUT_BASENAME"; - ## Files saved to DIR_TMP - FILEOUT_NMEA="$FILEOUT_BASENAME".nmea"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_NMEA to:$FILEOUT_NMEA"; - FILEOUT_GPX="$FILEOUT_BASENAME".gpx"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_GPX to:$FILEOUT_GPX"; - FILEOUT_KML="$FILEOUT_BASENAME".kml"$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX" && vbm "STATUS:Set FILEOUT_KML to:$FILEOUT_KML"; - PATHOUT_NMEA="$DIR_TMP"/"$FILEOUT_NMEA" && vbm "STATUS:Set PATHOUT_NMEA to:$PATHOUT_NMEA"; - PATHOUT_GPX="$DIR_TMP"/"$FILEOUT_GPX" && vbm "STATUS:Set PATHOUT_GPX to:$PATHOUT_GPX"; - PATHOUT_KML="$DIR_TMP"/"$FILEOUT_KML" && vbm "STATUS:Set PATHOUT_KML to:$PATHOUT_KML"; - ## Files saved to disk (DIR_OUT) - ### one file per day (Ex: "20200731..hostname_location.[.gpx.gz].tar") - PATHOUT_TAR="$DIR_OUT"/"$(dateShort)".."$SCRIPT_HOSTNAME""_location""$CMD_COMPRESS_SUFFIX""$CMD_ENCRYPT_SUFFIX".tar && \ - vbm "STATUS:Set PATHOUT_TAR to:$PATHOUT_TAR"; - # Define GPS conversion commands - CMD_CONV_NMEA="tee /dev/null " && vbm "STATUS:Set CMD_CONV_NMEA to:$CMD_CONV_NMEA"; # tee as passthrough - CMD_CONV_GPX="gpsbabel -i nmea -f - -o gpx -F - " && vbm "STATUS:Set CMD_CONV_GPX to:$CMD_CONV_GPX"; # convert NMEA to GPX - CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML + # Create buffer file with unique name + PATHOUT_BUFFER="$DIR_TMP/buffer$debugCounter++"; # Fill Bash variable buffer - bufferBash="$(timeout "$BUFFER_TTL""s" gpspipe -r)" && vbm "STATUS:Successfully filled bufferBash variable with gpspipe data."; # Record gpspipe nmea data to buffer for bufferTTL seconds + timeout "$BUFFER_TTL"s gpspipe -r "$PATHOUT_BUFFER" ; # Process bufferBash, save secured chunk set to DIR_TMP - vbm "STATUS:Beginning to save data to $DIR_TMP"; magicWriteBuffer & - # Append each secured chunk in memory dir (DIR_TMP) to file on disk (PATHOUT_TAR in DIR_OUT) - vbm "STATUS:DIR_TMP :$DIR_TMP"; - vbm "STATUS:PATHOUT_TAR :$PATHOUT_TAR"; - vbm "STATUS:PATHOUT_NMEA:$PATHOUT_NMEA"; - vbm "STATUS:PATHOUT_GPX:$PATHOUT_GPX"; - vbm "STATUS:PATHOUT_KML:$PATHOUT_KML"; - - # Reset buffer and filenames - unset bufferBash FILEOUT_BASENAME PATHOUT_NMEA PATHOUT_GPX PATHOUT_KML PATHOUT_TAR timeBufferStart; - vbm "DEBUG:Completed buffer session $debugCounter ." 1>&2; - ((debugCounter++)) + ((debugCounter++)); done - # Remove DIR_TMP + # Cleanup + ## Remove DIR_TMP try rm -r "$DIR_TMP"; vbm "STATUS:Main function finished."; -- 2.30.2 From 2b74d16b103a287517be4c270bcf91ddccbbc372 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 20:12:59 +0000 Subject: [PATCH 09/16] fix(bkgpslog):Add missing gpspipe option Need `-o` to write to file. --- exec/bkgpslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index c40175f..67b8f90 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.3"; # Define version of script. +SCRIPT_VERSION="0.3.4"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -993,7 +993,7 @@ main() { # Create buffer file with unique name PATHOUT_BUFFER="$DIR_TMP/buffer$debugCounter++"; # Fill Bash variable buffer - timeout "$BUFFER_TTL"s gpspipe -r "$PATHOUT_BUFFER" ; + timeout "$BUFFER_TTL"s gpspipe -r -o "$PATHOUT_BUFFER" ; # Process bufferBash, save secured chunk set to DIR_TMP magicWriteBuffer & ((debugCounter++)); -- 2.30.2 From 811f50f240ecce2696d0cc609fa6ca700a9b9e41 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 20:31:53 +0000 Subject: [PATCH 10/16] fix(bkgpslog):Simplify main loop further Move `gpspipe` command itself into write function. --- exec/bkgpslog | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 67b8f90..3029dfc 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.4"; # Define version of script. +SCRIPT_VERSION="0.3.5"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -843,14 +843,17 @@ magicWriteVersion() { appendArgTar "$CONTENT_VERSION" "$FILEOUT_VERSION" "$PATHOUT_TAR" "$DIR_TMP"; } # bkgpslog: write version data to PATHOUT_TAR via appendArgTar() -magicWriteBuffer() { +magicGatherWriteBuffer() { # Desc: bkgpslog-specific meta function for writing data to DIR_TMP then appending each file to PATHOUT_TAR # Inputs: PATHOUT_TAR FILEOUT_{NMEA,GPX,KML} CMD_CONV_{NMEA,GPX,KML} CMD_{COMPRESS,ENCRYPT} DIR_TMP, # Inputs: BUFFER_TTL bufferTTL_STR SCRIPT_HOSTNAME CMD_COMPRESS_SUFFIX CMD_ENCRYPT_SUFFIX # Depends: yell, try, vbm, appendArgTar, tar local FN="${FUNCNAME[0]}"; wait; # Wait to avoid collision with older magicWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html ) - + # Create buffer file with unique name + PATHOUT_BUFFER="$DIR_TMP/buffer$SECONDS"; + # Fill buffer + timeout "$BUFFER_TTL"s gpspipe -r -o "$PATHOUT_BUFFER" ; timeBufferStart="$(dateTimeShort "$(date --date="$BUFFER_TTL seconds ago")")"; # Note start time#TODO subtract BUFFER_TTL from current time vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer()."; # Determine file paths (time is start of buffer period) @@ -986,18 +989,12 @@ main() { CMD_CONV_NMEA="tee /dev/null " && vbm "STATUS:Set CMD_CONV_NMEA to:$CMD_CONV_NMEA"; # tee as passthrough CMD_CONV_GPX="gpsbabel -i nmea -f - -o gpx -F - " && vbm "STATUS:Set CMD_CONV_GPX to:$CMD_CONV_GPX"; # convert NMEA to GPX CMD_CONV_KML="gpsbabel -i nmea -f - -o kml -F - " && vbm "STATUS:Set CMD_CONV_KML to:$CMD_CONV_KML"; # convert NMEA to KML - + # MAIN LOOP:Record gps data until script lifespan ends - declare debugCounter; debugCounter="0"; # set debug counter while [[ "$SECONDS" -lt "$scriptTTL" ]]; do - # Create buffer file with unique name - PATHOUT_BUFFER="$DIR_TMP/buffer$debugCounter++"; - # Fill Bash variable buffer - timeout "$BUFFER_TTL"s gpspipe -r -o "$PATHOUT_BUFFER" ; - # Process bufferBash, save secured chunk set to DIR_TMP - magicWriteBuffer & - ((debugCounter++)); - done + magicGatherWriteBuffer & + sleep "$BUFFER_TTL" + done # Cleanup ## Remove DIR_TMP -- 2.30.2 From 11f61f1a2c94ec3a62c5fa902f99c2dbf5f429f2 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 20:38:01 +0000 Subject: [PATCH 11/16] chore(bkgpslog):Minor comment cleanup --- exec/bkgpslog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 3029dfc..40faba2 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -854,7 +854,7 @@ magicGatherWriteBuffer() { PATHOUT_BUFFER="$DIR_TMP/buffer$SECONDS"; # Fill buffer timeout "$BUFFER_TTL"s gpspipe -r -o "$PATHOUT_BUFFER" ; - timeBufferStart="$(dateTimeShort "$(date --date="$BUFFER_TTL seconds ago")")"; # Note start time#TODO subtract BUFFER_TTL from current time + timeBufferStart="$(dateTimeShort "$(date --date="$BUFFER_TTL seconds ago")")"; # Note start time vbm "DEBUG:STATUS:$FN:Started magicWriteBuffer()."; # Determine file paths (time is start of buffer period) FILEOUT_BASENAME="$timeBufferStart""--""$bufferTTL_STR""..""$SCRIPT_HOSTNAME""_location" && vbm "STATUS:Set FILEOUT_BASENAME to:$FILEOUT_BASENAME"; -- 2.30.2 From 238775e6ff3d7a699a601590f881e9c08a9397e6 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 20:46:46 +0000 Subject: [PATCH 12/16] style(bkgpslog):Minor semicolons --- exec/bkgpslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 40faba2..6429b84 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -924,7 +924,7 @@ main() { if [[ "$OPTION_ENCRYPT" = "true" ]]; then # Check if encryption option active. if checkapp age; then # Check that age is available. for pubkey in "${recPubKeys[@]}"; do # Validate recipient pubkey strings by forming test message - vbm "DEBUG:Testing pubkey string:$pubkey" + vbm "DEBUG:Testing pubkey string:$pubkey"; if echo "butts" | age -a -r "$pubkey" 1>/dev/null; then #### Form age recipient string recipients="$recipients""-r $pubkey "; @@ -993,7 +993,7 @@ main() { # MAIN LOOP:Record gps data until script lifespan ends while [[ "$SECONDS" -lt "$scriptTTL" ]]; do magicGatherWriteBuffer & - sleep "$BUFFER_TTL" + sleep "$BUFFER_TTL"; done # Cleanup -- 2.30.2 From 3592a7e928008b0e89286c91aa72b4db89a50d5f Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 20:58:01 +0000 Subject: [PATCH 13/16] doc(bkgpslog-plan):Closed TODO item --- exec/bkgpslog-plan.org | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog-plan.org b/exec/bkgpslog-plan.org index b7cc736..47154eb 100644 --- a/exec/bkgpslog-plan.org +++ b/exec/bkgpslog-plan.org @@ -33,7 +33,8 @@ done using a new function ~checkMakeTar~. magicWriteVersion() to be called whenever a new time-stamped ~VERSION~ file needs to be generated and appended to the output tar file ~PATHOUT_TAR~. -** TODO Rewrite buffer loop to reduce lag between gpspipe runs +** DONE Rewrite buffer loop to reduce lag between gpspipe runs + CLOSED: [2020-07-03 Fri 20:57] 2020-07-03T17:10Z; bktei> As is, there is still a 5-6 second lag between when ~gpspipe~ times out at the end of a buffer round and when ~gpspipe~ is called by the subsequent buffer round. I believe this can @@ -47,7 +48,16 @@ while( $SECONDS < $SCRIPT_TTL); do writeBuffer & done #+END_EXAMPLE - +2020-07-03T20:56Z; bktei> I simplified it futher to something like +this: +#+BEGIN_EXAMPLE +while( $SECONDS < $SCRIPT_TTL); do + writeBuffer & + sleep $SCRIPT_TTL +done +#+END_EXAMPLE +Raspberry Pi Zero W shows approximately 71ms of drift per buffer round +with 10s buffer. * bkgpslog narrative ** Initialize environment *** Init variables -- 2.30.2 From 9ae3346724bb2d3a82ce2303e30f27f7465eaa0b Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 22:02:17 +0000 Subject: [PATCH 14/16] doc(bkgpslog):Add TODO for future features - recipient watch folder - simplify gps data --- exec/bkgpslog-plan.org | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/exec/bkgpslog-plan.org b/exec/bkgpslog-plan.org index 47154eb..4ba6806 100644 --- a/exec/bkgpslog-plan.org +++ b/exec/bkgpslog-plan.org @@ -34,6 +34,7 @@ magicWriteVersion() to be called whenever a new time-stamped ~VERSION~ file needs to be generated and appended to the output tar file ~PATHOUT_TAR~. ** DONE Rewrite buffer loop to reduce lag between gpspipe runs + CLOSED: [2020-07-03 Fri 20:57] 2020-07-03T17:10Z; bktei> As is, there is still a 5-6 second lag between when ~gpspipe~ times out at the end of a buffer round and when @@ -56,8 +57,75 @@ while( $SECONDS < $SCRIPT_TTL); do sleep $SCRIPT_TTL done #+END_EXAMPLE + Raspberry Pi Zero W shows approximately 71ms of drift per buffer round with 10s buffer. +** TODO Feature: Recipient watch folder +2020-07-03T21:28Z; bktei> This feature would be to scan the contents +of a specified directory at the start of every buffer round in order +to determine encryption (age) recipients. This would allow a device to +dynamically encrypt location data in response to automated changes +made by other tools. For example, if such a directory were +synchronized via Syncthing and changes to such a directory were +managed by a trusted remote server, then that server could respond to +human requests to secure location data. + +Two specific privacy subfeatures come to mind: + +1. Parallel encryption: Given a set of ~n~ public keys, encrypt data + with a single ~age~ command with options causing all ~n~ pubkeys to + be recipients. In order to decrypt the data, any individual private + key could be used. No coordination between key owners would be + required to decrypt. + +2. Sequential encryption: Given a set of ~n~ public keys, encrypt data + with ~n~ sequential ~age~ commands all piped in series with each + ~age~ command utilizing only one of the ~n~ public keys. In order + to decrypt the data, all ~n~ private keys would be required to + decrypt the data. Since coordination is required, it is less + convenient than parallel encryption. + +In either case, a directory would be useful for holding configuration +files specifying how to execute which or combination of which features +at the start of every buffer round. + +I don't yet know how to program the rules, although I think it'd be +easier to simply add an option providing ~bkgpslog~ with a directory +to watch. When examining the directory, check for a file with the +appropriate file extension (ex: .pubkey) and then read the first line +into the script's pubKey array. + +** TODO Feature: Simplify option to reduce output size + +~gpsbabel~ [[https://www.gpsbabel.org/htmldoc-development/filter_simplify.html][features]] a ~simplify~ option to trim data points from GPS +data. There are several methods for prioritizing which points to keep +and which to trim, although the following seems useful given some +sample data I've recorded in a test run of ninfacyzga-01: + +#+BEGIN_EXAMPLE +gpsbabel -i nmea -f all.nmea -x simplify,error=10,relative -o gpx \ +-F all-simp-rel-10.gpx +#+END_EXAMPLE + +An error level of "10" with the "relative" option seems to retain all +desireable features for GPS data while reducing the number of points +along straightaways. File size is reduced by a factor of +about 11. Noise from local stay-in-place drift isn't removed; a +relative error of about 1000 is required to remove stay-in-place drift +noise but this also trims all but 100m-size features of the recorded +path. A relative error of 1000 reduces file size by a factor of +about 450. + +#+BEGIN_EXAMPLE + 67M relerror-0.001.kml + 66M relerror-0.01.kml + 58M relerror-0.1.kml + 21M relerror-1.kml +5.8M relerror-10.kml +797K relerror-100.kml +152K relerror-1000.kml +#+END_EXAMPLE + * bkgpslog narrative ** Initialize environment *** Init variables -- 2.30.2 From c582ac3ba65af9ec5e360a9fa04a33edf2e84fa8 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 22:43:05 +0000 Subject: [PATCH 15/16] fix(bkgpslog):Fix pubkey strings with spaces ssh pubkeys have a space in them and caused age encryption operation to fail. --- exec/bkgpslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 6429b84..50b83dc 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.5"; # Define version of script. +SCRIPT_VERSION="0.3.6"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -927,7 +927,7 @@ main() { vbm "DEBUG:Testing pubkey string:$pubkey"; if echo "butts" | age -a -r "$pubkey" 1>/dev/null; then #### Form age recipient string - recipients="$recipients""-r $pubkey "; + recipients="$recipients""-r \"$pubkey\" "; vbm "STATUS:Added pubkey for forming age recipient string:""$pubkey"; vbm "DEBUG:recipients:""$recipients"; #### Add validated pubkey to recPubKeysValid array -- 2.30.2 From be9ef23e2a53f4f8ec7553388d9e6f3d169af0e4 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Fri, 3 Jul 2020 22:54:50 +0000 Subject: [PATCH 16/16] test(bkgpslog):Test recipient string quoting recipient strings with spaces in them don't fare well with `\"` in them when such strings are used to create new strings. --- exec/bkgpslog | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/exec/bkgpslog b/exec/bkgpslog index 50b83dc..397fe11 100755 --- a/exec/bkgpslog +++ b/exec/bkgpslog @@ -14,7 +14,7 @@ DIR_TMP_DEFAULT="/dev/shm"; # Default parent of working directory SCRIPT_TIME_START=$(date +%Y%m%dT%H%M%S.%N); PATH="$HOME/.local/bin:$PATH"; # Add "$(systemd-path user-binaries)" path in case apps saved there SCRIPT_HOSTNAME=$(hostname); # Save hostname of system running this script. -SCRIPT_VERSION="0.3.6"; # Define version of script. +SCRIPT_VERSION="0.3.7"; # Define version of script. SCRIPT_NAME="bkgpslog"; # Define basename of script file. SCRIPT_URL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script. AGE_VERSION="1.0.0-beta2"; # Define version of age (encryption program) @@ -927,7 +927,7 @@ main() { vbm "DEBUG:Testing pubkey string:$pubkey"; if echo "butts" | age -a -r "$pubkey" 1>/dev/null; then #### Form age recipient string - recipients="$recipients""-r \"$pubkey\" "; + recipients="$recipients""-r '$pubkey' "; vbm "STATUS:Added pubkey for forming age recipient string:""$pubkey"; vbm "DEBUG:recipients:""$recipients"; #### Add validated pubkey to recPubKeysValid array @@ -939,26 +939,26 @@ main() { vbm "DEBUG:Finished processing recPubKeys array"; ## Form age command string - CMD_ENCRYPT="age ""$recipients "; - CMD_ENCRYPT_SUFFIX=".age"; + CMD_ENCRYPT="age ""$recipients " && vbm "CMD_ENCRYPT:$CMD_ENCRYPT"; + CMD_ENCRYPT_SUFFIX=".age" && vbm "CMD_ENCRYPT_SUFFIX:$CMD_ENCRYPT_SUFFIX"; else yell "ERROR:Encryption enabled but \"age\" not found. Exiting."; exit 1; fi else - CMD_ENCRYPT="tee /dev/null "; - CMD_ENCRYPT_SUFFIX=""; + CMD_ENCRYPT="tee /dev/null " && vbm "CMD_ENCRYPT:$CMD_ENCRYPT"; + CMD_ENCRYPT_SUFFIX="" && vbm "CMD_ENCRYPT_SUFFIX:$CMD_ENCRYPT_SUFFIX"; vbm "DEBUG:Encryption not enabled." fi if [[ "$OPTION_COMPRESS" = "true" ]]; then # Check if compression option active if checkapp gzip; then # Check if gzip available - CMD_COMPRESS="gzip "; - CMD_COMPRESS_SUFFIX=".gz"; + CMD_COMPRESS="gzip " && vbm "CMD_COMPRESS:$CMD_COMPRESS"; + CMD_COMPRESS_SUFFIX=".gz" && vbm "CMD_COMPRESS_SUFFIX:$CMD_COMPRESS_SUFFIX"; else yell "ERROR:Compression enabled but \"gzip\" not found. Exiting."; exit 1; fi else - CMD_COMPRESS="tee /dev/null "; - CMD_COMPRESS_SUFFIX=""; + CMD_COMPRESS="tee /dev/null " && vbm "CMD_COMPRESS:$CMD_COMPRESS"; + CMD_COMPRESS_SUFFIX="" && vbm "CMD_COMPRESS_SUFFIX:$CMD_COMPRESS_SUFFIX"; vbm "DEBUG:Compression not enabled."; fi -- 2.30.2