From: Steven Baltakatei Sandoval Date: Fri, 27 May 2022 08:55:02 +0000 (+0000) Subject: feat(user/bkots):Optimize with GNU Parallel for executing commands X-Git-Tag: 0.5.1~17 X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/commitdiff_plain/90152692eced87d53ab0c492c35b89928d849870 feat(user/bkots):Optimize with GNU Parallel for executing commands - Assemble each command and save as string in array; then execute array. - Use bash function as wrapper to echo and group each command being performed along with its stdout/stderr messages. - Use regex search of '/.' to find and exclude dotfile paths instead of resolving each basename of each parent directory in each path. - Use double quotes to handle file names with special characters. - Escape out double quotes when they appear in file names. - Note: Incrementing major version number due to added dependency of GNU Parallel 20210822 --- diff --git a/user/bkots b/user/bkots index 79c6692..d37b9c2 100755 --- a/user/bkots +++ b/user/bkots @@ -5,12 +5,12 @@ declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status declare -ag arrayPosArgs # Associative array for processArgs() function -declare -g ots_delay; ots_delay=1 # minimum time in seconds between ots operations declare -ag calendars; calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); +declare -a commands # array for storing assembled commands age_threshold="60"; # min age to add file; seconds; # Declare functions @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 0.0.9 +bkots 1.0.0 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -354,6 +354,14 @@ get_parent_dirnames() { echo "$name_base"; done; }; # Output parent dirnames to stdout +cmdwrap() { + # print command to stderr + echo "$@" 1>&2; + + # execute command + "$@"; +}; # print and execute string together +export -f cmdwrap; # export cmdwrap for use in other functions main() { # Desc: Creates `.ots` file: # - for each file specified in arrayPosArgs array @@ -365,10 +373,11 @@ main() { # age_threshold var: mininum age in seconds to timestamp file # Output: file(s) creates `.ots` file alongside specified files - # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort) + # Depends: find (GNU findutils) 4.8.0, GNU Coreutils 8.32 (sort), GNU Parallel 20210822 # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194 # [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028 # [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377 + # [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001 local -a file_list file_list_pruned; local -a files_to_verify files_to_upgrade files_to_stamp local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned @@ -377,7 +386,7 @@ main() { processArgs "$@"; # Check dependencies - if ! checkapp ots find; then + if ! checkapp ots find parallel; then displayMissing; die "FATAL:Missing dependencies."; fi; @@ -439,49 +448,55 @@ main() { for item in "${file_list[@]}"; do ## Ignore files that end in '.ots.bak'. if [[ $item =~ '.ots.bak'$ ]]; then - yell "INFO :Skipping file ending in '.ots.bak':item:$item"; + vbm "DEBUG:Skipping file ending in '.ots.bak':item:$item"; continue; # skip to next item fi; ## Ignore dotfiles if ! [[ $option_include_dotfiles == "true" ]]; then - ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config') - unset flag_contains_dotfile_parent; - while read -r line; do - #### Check line from output of get_parent_dirnames - pattern="^\."; - if [[ $line =~ $pattern ]]; then - ##### line starts with '.' - vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item"; - vbm "DEBUG:Dotfile in path:item:$item"; - vbm "DEBUG:Dotfile parent:line:$line"; - flag_contains_dotfile_parent="true"; - break - fi; - done < <(get_parent_dirnames "$item"); - if [[ $flag_contains_dotfile_parent == "true" ]]; then - unset flag_contains_dotfile_parent; - continue; # skip to next item (i.e. don't add to file_list_pruned) - fi; - - ### Ignore dotfiles themselves - item_basename="$(basename "$item")"; - pattern="^\."; - if [[ $item_basename =~ $pattern ]]; then - vbm "INFO :Skipping dotfile:item:$item"; - continue; # skip to next item + ### Ignore files if '/.' contained within canonical path + pattern="/\."; # a dot after a forward slash + if [[ $item =~ $pattern ]]; then + continue; fi; + + # ### Ignore files located beneath a dotfile directory (e.g. '/home/my_repo/.git/config') + # unset flag_contains_dotfile_parent; + # while read -r line; do + # #### Check line from output of get_parent_dirnames + # pattern="^\."; + # if [[ $line =~ $pattern ]]; then + # ##### line starts with '.' + # vbm "DEBUG:Dotfile parent detected. Not including in file_list_pruned:$item"; + # vbm "DEBUG:Dotfile in path:item:$item"; + # vbm "DEBUG:Dotfile parent:line:$line"; + # flag_contains_dotfile_parent="true"; + # break + # fi; + # done < <(get_parent_dirnames "$item"); + # if [[ $flag_contains_dotfile_parent == "true" ]]; then + # unset flag_contains_dotfile_parent; + # continue; # skip to next item (i.e. don't add to file_list_pruned) + # fi; + + # ### Ignore dotfiles themselves + # item_basename="$(basename "$item")"; + # pattern="^\."; + # if [[ $item_basename =~ $pattern ]]; then + # vbm "INFO :Skipping dotfile:item:$item"; + # continue; # skip to next item + # fi; fi; ## Ignore files with newlines present in filename. See [2]. if [[ $item =~ $'\n' ]]; then - yell "INFO :Skipping file name with newline:$item"; + vbm "DEBUG:Skipping file name with newline:$item"; continue; # skip to next item fi; ## Ignore files that end in '~'. if [[ $item =~ ~$ ]]; then - yell "INFO :Skipping file ending in tilde:$item"; + vbm "DEBUG:Skipping file ending in tilde:$item"; continue; # skip to next item fi; @@ -625,9 +640,10 @@ main() { fi; # Act on files - ## Upgrade files + ## Assemble upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_prf" ]]; then yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; continue; @@ -637,18 +653,24 @@ main() { ### Try upgrade with known calendars in random order while read -r url; do vbm "DEBUG:Upgrading with calendar:url:$url"; - ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; + if [[ "$opVerbose" = "true" ]]; then + commands+=("cmdwrap ots -v -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; + else + commands+=("cmdwrap ots -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; + fi; + #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; done < <(printf "%s\n" "${calendars[@]}" | shuf); else yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; fi; - #sleep "$ots_delay"; done; - ## Verify files + ## Assemble verify file commands for item in "${files_to_verify_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. + path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then yell "ERROR:blank verify item encountered. Skipping:item:$item"; continue; @@ -659,30 +681,43 @@ main() { ### Try verify with known calendars in random order while read -r url; do vbm "DEBUG:Verifying with calendar:url:$url"; - ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + if [[ "$opVerbose" = "true" ]]; then + commands+=("cmdwrap ots -v -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; + else + commands+=("cmdwrap ots -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; + fi; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; done < <(printf "%s\n" "${calendars[@]}" | shuf); else yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; fi; - #sleep "$ots_delay"; done; - ## Stamp files + ## Assemble stamp file commands for item in "${files_to_stamp_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_src" ]]; then yell "ERROR:blank stamp item encountered. Skipping:item:$item"; continue; fi; vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; if [[ ! $option_dry_run == "true" ]]; then - ots stamp "$path_src"; + if [[ "$opVerbose" = "true" ]]; then + commands+=("cmdwrap ots -v stamp \"$path_src_sesc\""); + else + commands+=("cmdwrap ots stamp \"$path_src_sesc\""); + fi; + #ots stamp "$path_src"; else yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; fi; - sleep "$ots_delay"; done; + ## Run commands + yell "DEBUG:commands:$(printf "%s\n" "${commands[@]}")"; + printf "%s\n" "${commands[@]}" | parallel --group; + }; # main program # Run program