debug(bklog):Test tar --append

[EVA-2020-02.git] / exec / bklog

diff --git a/exec/bklog b/exec/bklog
index 38f4f3f76cf05da00b1192875734fbdd4721fc2c..ecc4e85c9f92d6c4fa007571adfc27a9292bb3e3 100644 (file)
--- a/exec/bklog
+++ b/exec/bklog
@@ -11,7 +11,7 @@ dirTmpDefault="/dev/shm"; # Default parent of working directory
 
 # Script Metadata
 scriptName="bklog";             # Define basename of script file.
 
 # Script Metadata
 scriptName="bklog";             # Define basename of script file.

-scriptVersion="0.1.16";          # Define version of script.
+scriptVersion="0.1.21-debug";          # Define version of script.

 scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
 scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN
 scriptHostname=$(hostname);     # Save hostname of system running this script.
 scriptURL="https://gitlab.com/baltakatei/ninfacyzga-01"; # Define wesite hosting this script.
 scriptTimeStart="$(date +%Y%m%dT%H%M%S.%N)"; # YYYYmmddTHHMMSS.NNNNNNNNN
 scriptHostname=$(hostname);     # Save hostname of system running this script.


@@ -225,6 +225,64 @@ displayMissing() {
     #==END Display errors==
 } # Display missing apps, files, dirs
 
     #==END Display errors==
 } # Display missing apps, files, dirs
 

+appendArgTar(){
+    # Desc: Writes first argument to temporary file with arguments as options, then appends file to tar
+    # Usage: appendArgTar "$(echo "Data to be written.")" [name of file to be inserted] [tar path] [temp dir] ([cmd1] [cmd2] [cmd3] [cmd4]...)
+    # Version: 1.0.6
+    # Input: arg1: data to be written
+    #        arg2: file name of file to be inserted into tar
+    #        arg3: tar archive path (must exist first)
+    #        arg4: temporary working dir
+    #        arg5+: command strings (ex: "gpsbabel -i nmea -f - -o kml -F - ")
+    # Output: file written to disk
+    # Example: decrypt multiple large files in parallel
+    #          appendArgTar "$(cat /tmp/largefile1.gpg)" "largefile1" $HOME/archive.tar /tmp "gpg --decrypt" &
+    #          appendArgTar "$(cat /tmp/largefile2.gpg)" "largefile2" $HOME/archive.tar /tmp "gpg --decrypt" &
+    #          appendArgTar "$(cat /tmp/largefile3.gpg)" "largefile3" $HOME/archive.tar /tmp "gpg --decrypt" &
+    # Depends: bash 5, tar 1, yell()
+    # Ref/Attrib: Using 'eval' to construct command strings https://askubuntu.com/a/476533
+
+    local fn fileName tarPath tmpDir cmd0 cmd1 cmd2 cmd3 cmd4
+
+    # Save function name
+    fn="${FUNCNAME[0]}";
+    #yell "DEBUG:STATUS:$fn:Finished appendArgTar()."
+    
+    # Set file name
+    if ! [ -z "$2" ]; then fileName="$2"; else yell "ERROR:$fn:Not enough arguments."; exit 1; fi
+    
+    # Check tar path is a file
+    if [ -f "$3" ]; then tarPath="$3"; else yell "ERROR:$fn:Tar archive arg not a file."; exit 1; fi
+       
+    # Check temp dir arg
+    if ! [ -z "$4" ]; then tmpDir="$4"; else yell "ERROR:$fn:No temporary working dir set."; exit 1; fi
+    
+    # Set command strings
+    if ! [ -z "$5" ]; then cmd1="$5"; else cmd1="cat "; fi # command string 1
+    if ! [ -z "$6" ]; then cmd2="$6"; else cmd2="cat "; fi # command string 2
+    if ! [ -z "$7" ]; then cmd3="$7"; else cmd3="cat "; fi # command string 3
+    if ! [ -z "$8" ]; then cmd4="$8"; else cmd4="cat "; fi # command string 4
+
+    # Input command
+    cmd0="echo \"\$1\""
+
+    # # Debug
+    # yell "DEBUG:STATUS:$fn:cmd0:$cmd0"
+    # yell "DEBUG:STATUS:$fn:cmd1:$cmd1"
+    # yell "DEBUG:STATUS:$fn:cmd2:$cmd2"
+    # yell "DEBUG:STATUS:$fn:cmd3:$cmd3"
+    # yell "DEBUG:STATUS:$fn:cmd4:$cmd4"
+    # yell "DEBUG:STATUS:$fn:fileName:$fileName"
+    # yell "DEBUG:STATUS:$fn:tarPath:$tarPath"
+    # yell "DEBUG:STATUS:$fn:tmpDir:$tmpDir"
+    
+    # Write to temporary working dir
+    eval "$cmd0 | $cmd1 | $cmd2 | $cmd3 | $cmd4" > "$tmpDir"/"$fileName";
+
+    # Append to tar
+    try tar --append --directory="$tmpDir" --file="$tarPath" "$fileName";
+    #yell "DEBUG:STATUS:$fn:Finished appendArgTar()."
+} # Append Bash var to file appended to Tar archive

 appendFileTar(){
     # Desc: Appends [processed] file to tar
     # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd])
 appendFileTar(){
     # Desc: Appends [processed] file to tar
     # Usage: appendFileTar [file path] [name of file to be inserted] [tar path] [temp dir] ([process cmd])


@@ -787,7 +845,7 @@ magicInitCheckTar() {
     # input: vars: scriptHostname
     # output: vars: pathout_tar
     # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion()
     # input: vars: scriptHostname
     # output: vars: pathout_tar
     # depends: Bash 5.0.3, vbm(), dateShort(), checkMakeTar(), magicWriteVersion()

-    local fn
+    local fn checkMakeTarES

 
     # Save function name
     fn="${FUNCNAME[0]}";    
 
     # Save function name
     fn="${FUNCNAME[0]}";    


@@ -797,10 +855,10 @@ magicInitCheckTar() {
     pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \
        vbm "STATUS:$fn:Set pathout_tar to:$pathout_tar";
     # Validate pathout_tar as tar.
     pathout_tar="$dirOut"/"$(dateShort "$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)")".."$scriptHostname""$label""$cmd_compress_suffix""$cmd_encrypt_suffix".tar && \
        vbm "STATUS:$fn:Set pathout_tar to:$pathout_tar";
     # Validate pathout_tar as tar.

-    checkMakeTar "$pathout_tar";
+    checkMakeTar "$pathout_tar"; checkMakeTarES="$?";

     ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2)
     ## Add VERSION file if checkMakeTar had to create a tar (exited 1) or replace one (exited 2)

-    vbm "STATUS:$fn:exit status before magicWriteVersion:$?"
-    if [[ $? -eq 1 ]] || [[ $? -eq 2 ]]; then magicWriteVersion; fi
+    vbm "STATUS:$fn:exit status before magicWriteVersion:$checkMakeTarES"
+    if [[ "$checkMakeTarES" -eq 1 ]] || [[ "$checkMakeTarES" -eq 2 ]]; then magicWriteVersion; fi

     vbm "STATUS:$fn:Finished magicInitCheckTar() function.";
 } # Initialize tar, set pathout_tar
 magicParseCompressionArg() {
     vbm "STATUS:$fn:Finished magicInitCheckTar() function.";
 } # Initialize tar, set pathout_tar
 magicParseCompressionArg() {


@@ -970,6 +1028,8 @@ magicParseProcessStrings() {
            yell "ERROR:$fn:Illegal character '-' at start of process string element:\"$element\"";
            exit 1; fi; done;
     vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents.";
            yell "ERROR:$fn:Illegal character '-' at start of process string element:\"$element\"";
            exit 1; fi; done;
     vbm "STATUS:$fn:Quick check shows argProcStrings and argProcFileExts appear to have valid contents.";

+    vbm "STATUS:$fn:argProcStrings:${argProcStrings[*]}"
+    vbm "STATUS:$fn:argProcStrings:${argProcFileExts[*]}"

     procStrings+=("${argProcStrings[@]}"); # Export process command strings
     procFileExts+=("${argProcFileExts[@]}"); # Export process command strings
     vbm "STATUS:$fn:Finished magicParseProcessStrings() function.";
     procStrings+=("${argProcStrings[@]}"); # Export process command strings
     procFileExts+=("${argProcFileExts[@]}"); # Export process command strings
     vbm "STATUS:$fn:Finished magicParseProcessStrings() function.";


@@ -1164,6 +1224,10 @@ magicProcessWriteBuffer() {
     fn="${FUNCNAME[0]}";
     
     vbm "STATUS:$fn:Started magicProcessWriteBuffer().";
     fn="${FUNCNAME[0]}";
     
     vbm "STATUS:$fn:Started magicProcessWriteBuffer().";

+    vbm "DEBUG :$fn:buffer array element count:${#buffer[@]}";
+    vbm "DEBUG :$fn:buffer array first element:${buffer[0]}";
+    vbm "DEBUG :$fn:buffer array last element :${buffer[-1]}";
+    

     # Determine file paths (time is start of buffer period)
     ## Calculate start time
     timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \
     # Determine file paths (time is start of buffer period)
     ## Calculate start time
     timeBufferStartLong="$(date --date="$bufferTTL seconds ago" --iso-8601=seconds)" && \


@@ -1194,22 +1258,30 @@ magicProcessWriteBuffer() {
     writeCmd4="$cmd_encrypt";
     
     ## Process buffer and write to dir_tmp
     writeCmd4="$cmd_encrypt";
     
     ## Process buffer and write to dir_tmp

+    vbm "DEBUG :$fn:fileouts    element count:${#fileouts[@]}";
+    vbm "DEBUG :$fn:pathouts    element count:${#pathouts[@]}";
+    vbm "DEBUG :$fn:procStrings element count:${#pathouts[@]}";
+    vbm "DEBUG :$fn:fileouts    contents:${fileouts[*]}";
+    vbm "DEBUG :$fn:pathouts    contents:${pathouts[*]}";
+    vbm "DEBUG :$fn:procStrings contents:${pathouts[*]}";

     for index in "${!pathouts[@]}"; do
        writeCmd2="${procStrings[$index]}";
        writeCmdAll="$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" && vbm "STATUS:$fn:Assembled command:\"$writeCmdAll\"";
     for index in "${!pathouts[@]}"; do
        writeCmd2="${procStrings[$index]}";
        writeCmdAll="$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" && vbm "STATUS:$fn:Assembled command:\"$writeCmdAll\"";

-       eval "$writeCmdAll" >> "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}";
+       eval "$writeCmd1 | $writeCmd2 | $writeCmd3 | $writeCmd4" > "${pathouts[$index]}" && vbm "STATUS:$fn:Wrote command output to ${pathouts[$index]}";

     done;
     
     # Append dir_tmp files to pathout_tar
     wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
     for index in "${!pathouts[@]}"; do
     done;
     
     # Append dir_tmp files to pathout_tar
     wait; # Wait to avoid collision with older magicProcessWriteBuffer() instances (see https://www.tldp.org/LDP/abs/html/x9644.html )
     for index in "${!pathouts[@]}"; do

-       appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \
+       tar --append --directory="$dir_tmp" --file="$pathout_tar" "${fileouts[$index]}" && \

            vbm "STATUS:$fn:Appended ${pathouts[$index]} to $pathout_tar";
            vbm "STATUS:$fn:Appended ${pathouts[$index]} to $pathout_tar";

+       #appendFileTar "${pathouts[$index]}" "${fileouts[$index]}" "$pathout_tar" "$dir_tmp" && \

     done;
     
     # Remove secured chunks from dir_tmp
     for path in "${pathouts[@]}"; do
     done;
     
     # Remove secured chunks from dir_tmp
     for path in "${pathouts[@]}"; do

-       rm "$path" && vbm "STATUS:$fn:Removed:$path";
+       # rm "$path" && vbm "STATUS:$fn:Removed:$path";
+       :

     done;
     
     vbm "STATUS:$fn:Finished magicProcessWriteBuffer().";
     done;
     
     vbm "STATUS:$fn:Finished magicProcessWriteBuffer().";


@@ -1325,7 +1397,9 @@ main() {
     ## Init temp working dir
     try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp";
     ## Initialize output tar (set pathout_tar)
     ## Init temp working dir
     try mkdir "$dir_tmp" && vbm "DEBUG :$fn:Working dir created at dir_tmp:$dir_tmp";
     ## Initialize output tar (set pathout_tar)

-    magicInitCheckTar; 
+    magicInitCheckTar;
+    ## Append VERSION file to tar
+    magicWriteVersion;

     
     # Check vital apps, files, dirs
     if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then
     
     # Check vital apps, files, dirs
     if ! checkapp tar && ! checkdir "$dirOut" "dir_tmp"; then