From 55d2c6d0cb3211c3aa57669c74546225fd985ac6 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Tue, 16 Jul 2024 00:03:42 +0000 Subject: [PATCH 01/16] feat(unitproc/bkt-remove-leading_zeroes):Remove read_stdin() --- unitproc/bkt-remove_leading_zeroes | 38 ++---------------------------- 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/unitproc/bkt-remove_leading_zeroes b/unitproc/bkt-remove_leading_zeroes index d934e93..503dd40 100755 --- a/unitproc/bkt-remove_leading_zeroes +++ b/unitproc/bkt-remove_leading_zeroes @@ -1,48 +1,14 @@ #!/bin/bash -read_stdin() { - # Desc: Consumes stdin; outputs as stdout lines - # Input: stdin (consumes) - # Output: stdout (newline delimited) - # return 0 stdin read - # return 1 stdin not present - # Example: printf "foo\nbar\n" | read_stdin - # Depends: GNU bash (version 5.1.16), GNU Coreutils 8.32 (cat) - # Version: 0.1.1 - # Attrib: Steven Baltakatei Sandoval (2024-01-29). reboil.com - local input_stdin output; - - # Store stdin - if [[ -p /dev/stdin ]]; then - input_stdin="$(cat -)" || { - echo "FATAL:Error reading stdin." 1>&2; return 1; }; - else - return 1; - fi; - - # Store as output array elements - ## Read in stdin - if [[ -n $input_stdin ]]; then - while read -r line; do - output+=("$line"); - done < <(printf "%s\n" "$input_stdin") || { - echo "FATAL:Error parsing stdin."; return 1; }; - fi; - - # Print to stdout - printf "%s\n" "${output[@]}"; - - return 0; -}; # read stdin to stdout lines remove_leading_zeroes() { # Desc: Removes leading zeroes from lines # Input: stdin # Output: stdout # Depends: BK-2020-03 read_stdin() - # Version: 0.0.1 + # Version: 0.0.2 while read -r line; do printf "%s\n" "$line" | sed -E -e 's/(^0*)([0-9].*)/\2/'; - done < <(read_stdin); + done; }; printf "00000.jpg\n0001.jpg\n2.jpg\n000003.jpg\n0010.jpg\n"; -- 2.30.2 From 0fc97e3eeb52abebc4d972e128ad206ab10386da Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Wed, 17 Jul 2024 00:03:24 +0000 Subject: [PATCH 02/16] feat(unitproc/bkt-get_path_hierarchy_level):Add bash function --- unitproc/bkt-get_path_hierarchy_level | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 unitproc/bkt-get_path_hierarchy_level diff --git a/unitproc/bkt-get_path_hierarchy_level b/unitproc/bkt-get_path_hierarchy_level new file mode 100755 index 0000000..a76cb1f --- /dev/null +++ b/unitproc/bkt-get_path_hierarchy_level @@ -0,0 +1,77 @@ +#!/bin/bash +get_path_hierarchy_level() { + # Desc: Outputs hierarchy level of input paths + # Example: $ cat lines.txt | get_path_hierarchy_level + # Input: stdin str lines with /-delimited paths + # Output: stdout int hierarchy level of each path + # Version: 0.0.1 + + local line level; + local flag_root; + local -a output; + + n=0; + while read -r line; do + # Check for mixed absolute/relative paths. + if [[ $n -le 0 ]] && [[ "$line" =~ ^/ ]]; then + flag_root=true; + else + flag_root=false; + fi; + if { [[ "$flag_root" == "true" ]] && [[ ! "$line" =~ ^/ ]]; } || \ + { [[ "$flag_root" == "false" ]] && [[ "$line" =~ ^/ ]]; } then + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; return 1; + fi; + + # Squeeze multiple slashes and remove trailing slashes + line="$(echo "$line" | tr -s '/' | sed 's:/*$::' )"; + + # Count the number of slashes to determine hierarchy level + level="$(echo "$line" | awk -F'/' '{print NF-1}' )"; + if [[ "$flag_root" == "true" ]]; then ((level--)); fi; + + # Append to output + output+=("$level"); + #declare -p flag_root level; + ((n++)); + done; + # Print output + printf "%s\n" "${output[@]}"; +}; + +# Test the function with the provided lines +printf "\n\n========Test 1========\n" +input_lines=( + "foo" + "foo/" + "foo/bar" + "foo/bar/" + "foo/bar///" + "foo///bar" + "/foo/bar/baz" + "/foo/bar/baz/" + "//foo/bar/baz////" +); + +printf "%s\n" "${input_lines[@]}"; +printf "========Test 1 results:========\n"; +# Convert array to newline-separated string and pass to function +printf "%s\n" "${input_lines[@]}" | get_path_hierarchy_level + +printf "\n\n========Test 2========\n" +input_lines=( + "foo" + "foo/" + "foo/bar" + "foo/bar/" + "foo/bar///" + "foo///bar" + "foo/bar/baz" + "foo/bar/baz/" + "foo/bar/baz////" +); + +printf "%s\n" "${input_lines[@]}"; +printf "========Test 2 results:========\n"; +# Convert array to newline-separated string and pass to function +printf "%s\n" "${input_lines[@]}" | get_path_hierarchy_level -- 2.30.2 From ede080f66be6d19f701d1bffea26229fc9343e35 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Wed, 17 Jul 2024 02:25:04 +0000 Subject: [PATCH 03/16] feat(unitproc/bkt-prune_path_rootside):Add Bash function --- unitproc/bkt-prune_path_rootside | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 unitproc/bkt-prune_path_rootside diff --git a/unitproc/bkt-prune_path_rootside b/unitproc/bkt-prune_path_rootside new file mode 100755 index 0000000..9736299 --- /dev/null +++ b/unitproc/bkt-prune_path_rootside @@ -0,0 +1,54 @@ +#!/bin/bash + +prune_path_rootside() { + # Desc: Prunes a path from the root-side to a specified prune level. + # Input: arg1 str path + # arg2 int prune level (0-indexed) + # Depends: GNU sed 4.8 + # Version: 0.0.1 + local path="$1"; + local prune_level="$2"; + + # Check for absolute or relative path + if [[ "$path" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path="$(echo "$path" | sed -e 's:^/::' )"; + else + flag_root=false; + fi; + + # Save path as array with `/` as element delimiter + local IFS='/'; + read -ra parts <<< "$path"; + + # Assemble pruned path from prune_level + local pruned_path=""; + for (( i=prune_level; i<${#parts[@]}; i++ )); do + pruned_path+="${parts[i]}/"; + done; + + # Trim trailing `/` delimiter + pruned_path=$(echo "$pruned_path" | sed 's:/*$::'); + + # Restore initial / if appropriate + if [[ "$flag_root" == "true" ]] && [[ "$prune_level" -eq 0 ]]; then + pruned_path=/"$pruned_path"; + fi; + + # Output pruned path + echo "$pruned_path"; + #declare -p path prune_level parts pruned_path && printf "========\n"; # debug + return 0; +}; # prune path rootside to int specified level + +printf "========Test 1========\n"; +prune_path_rootside "foo/bar/baz" 0; +prune_path_rootside "foo/bar/baz" 1; +prune_path_rootside "foo/bar/baz" 2; +prune_path_rootside "foo/bar/baz" 3; +printf "========Test 2========\n"; +prune_path_rootside "/foo/bar/baz" 0; +prune_path_rootside "/foo/bar/baz" 1; +prune_path_rootside "/foo/bar/baz" 2; +prune_path_rootside "/foo/bar/baz" 3; -- 2.30.2 From 670c5ae4201367f8ad4b6257e3a79cdef313a760 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Wed, 17 Jul 2024 03:02:20 +0000 Subject: [PATCH 04/16] feat(unitproc/bkt-get_path_fork_level):Add Bash function --- unitproc/bkt-get_path_fork_level | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 unitproc/bkt-get_path_fork_level diff --git a/unitproc/bkt-get_path_fork_level b/unitproc/bkt-get_path_fork_level new file mode 100755 index 0000000..521b42d --- /dev/null +++ b/unitproc/bkt-get_path_fork_level @@ -0,0 +1,74 @@ +#!/bin/bash + +get_path_fork_level() { + # Desc: Get fork level from two paths + # Input: arg1 str path + # arg2 str path + # Output: stdout int fork level + # Version: 0.0.1 + local path1="$1"; + local path2="$2"; + + # Squeeze multiple slashes and remove trailing slashes + path1="$(echo "$path1" | tr -s '/' | sed 's:/*$::' )"; + path2="$(echo "$path2" | tr -s '/' | sed 's:/*$::' )"; + + # Check for mixed absolute/relative paths + if [[ "$path1" =~ ^/ ]] && [[ "$path2" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path1="$(echo "$path1" | sed -e 's:^/::' )"; + path2="$(echo "$path2" | sed -e 's:^/::' )"; + elif [[ ! "$path1" =~ ^/ ]] && [[ ! "$path2" =~ ^/ ]]; then + flag_root=false; + else + declare -p path1 path2 flag_root; + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; + return 1; + fi; + + # Save path as arrays with `/` as element delimiter + local IFS='/'; + read -ra parts1 <<< "$path1"; + read -ra parts2 <<< "$path2"; + + # Get fork level by counting identical path elements from rootside + local fork_level=0; + for (( i=0; i<${#parts1[@]} && i<${#parts2[@]}; i++ )); do + if [[ "${parts1[i]}" != "${parts2[i]}" ]]; then break; fi; + ((fork_level++)); + done; + + echo "$fork_level"; + #declare -p path1 path2 flag_root parts1 parts2 fork_level; # debug + return 0; +}; # Get fork level int from two paths + +printf "========Test 1========\n"; # fork at 0 +p1="foo"; p2="bee"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 2========\n"; # fork at 1 +p1="foo"; p2="foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 3========\n"; # fork at 1 +p1="foo"; p2="foo/bar/"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 4========\n"; # fork at 2 +p1="foo/bar/baz"; p2="foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 5========\n"; # fork at 2 +p1="/foo/bar/baz"; p2="/foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 6========\n"; # ERROR: No mixed +p1="foo/bar/baz"; p2="/bee/boo/tax"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 7========\n"; # fork at 0 +p1="/foo/bar/baz"; p2="/bee/boo/tax"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 8========\n"; # # ERROR: No mixed +p1="/foo/bar/baz"; p2="foo"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 9========\n"; # fork at 3 +p1="foo///////////bar////////baz//////"; p2="foo/////////bar/////////baz///bee"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; + -- 2.30.2 From 4f51959a8c300e9436864e8bd1bc50b53a942c81 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Wed, 17 Jul 2024 03:51:20 +0000 Subject: [PATCH 05/16] feat(user/mw_create_subpage_navlinks.sh):Handle deep subpage paths --- unitproc/bkt-get_path_hierarchy_level | 4 +- user/mw_create_subpage_navlinks.sh | 196 ++++++++++++++++++++++---- 2 files changed, 172 insertions(+), 28 deletions(-) diff --git a/unitproc/bkt-get_path_hierarchy_level b/unitproc/bkt-get_path_hierarchy_level index a76cb1f..7e03981 100755 --- a/unitproc/bkt-get_path_hierarchy_level +++ b/unitproc/bkt-get_path_hierarchy_level @@ -32,12 +32,12 @@ get_path_hierarchy_level() { # Append to output output+=("$level"); - #declare -p flag_root level; + #declare -p flag_root level; # debug ((n++)); done; # Print output printf "%s\n" "${output[@]}"; -}; +}; # return hierarchy level of lines as integers # Test the function with the provided lines printf "\n\n========Test 1========\n" diff --git a/user/mw_create_subpage_navlinks.sh b/user/mw_create_subpage_navlinks.sh index de37365..5cdf74a 100755 --- a/user/mw_create_subpage_navlinks.sh +++ b/user/mw_create_subpage_navlinks.sh @@ -3,8 +3,8 @@ # Input: file text file with list of chapters # stdin text with list of chapters # Output: [[../Chapter 4|Next]], [[../Chapter 2|Previous]], [[../|Up]] -# Version: 0.0.1 -# Attrib: Steven Baltakatei Sandoval. (2024-01-29). reboil.com +# Version: 0.1.0 +# Attrib: Steven Baltakatei Sandoval. (2024-07-17). reboil.com # License: GPLv3+ yell() { echo "$0: $*" >&2; } # print script path and all args to stderr @@ -63,6 +63,129 @@ read_stdin() { return 0; }; # read stdin to stdout lines +get_path_fork_level() { + # Desc: Get fork level from two paths + # Input: arg1 str path + # arg2 str path + # Output: stdout int fork level + # Version: 0.0.1 + local path1="$1"; + local path2="$2"; + + # Squeeze multiple slashes and remove trailing slashes + path1="$(echo "$path1" | tr -s '/' | sed 's:/*$::' )"; + path2="$(echo "$path2" | tr -s '/' | sed 's:/*$::' )"; + + # Check for mixed absolute/relative paths + if [[ "$path1" =~ ^/ ]] && [[ "$path2" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path1="$(echo "$path1" | sed -e 's:^/::' )"; + path2="$(echo "$path2" | sed -e 's:^/::' )"; + elif [[ ! "$path1" =~ ^/ ]] && [[ ! "$path2" =~ ^/ ]]; then + flag_root=false; + else + declare -p path1 path2 flag_root; + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; + return 1; + fi; + + # Save path as arrays with `/` as element delimiter + local IFS='/'; + read -ra parts1 <<< "$path1"; + read -ra parts2 <<< "$path2"; + + # Get fork level by counting identical path elements from rootside + local fork_level=0; + for (( i=0; i<${#parts1[@]} && i<${#parts2[@]}; i++ )); do + if [[ "${parts1[i]}" != "${parts2[i]}" ]]; then break; fi; + ((fork_level++)); + done; + + echo "$fork_level"; + #declare -p path1 path2 flag_root parts1 parts2 fork_level; # debug + return 0; +}; # Get fork level int from two paths +prune_path_rootside() { + # Desc: Prunes a path from the root-side to a specified prune level. + # Input: arg1 str path + # arg2 int prune level (0-indexed) + # Depends: GNU sed 4.8 + # Version: 0.0.1 + local path="$1"; + local prune_level="$2"; + + # Check for absolute or relative path + if [[ "$path" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path="$(echo "$path" | sed -e 's:^/::' )"; + else + flag_root=false; + fi; + + # Save path as array with `/` as element delimiter + local IFS='/'; + read -ra parts <<< "$path"; + + # Assemble pruned path from prune_level + local pruned_path=""; + for (( i=prune_level; i<${#parts[@]}; i++ )); do + pruned_path+="${parts[i]}/"; + done; + + # Trim trailing `/` delimiter + pruned_path=$(echo "$pruned_path" | sed 's:/*$::'); + + # Restore initial / if appropriate + if [[ "$flag_root" == "true" ]] && [[ "$prune_level" -eq 0 ]]; then + pruned_path=/"$pruned_path"; + fi; + + # Output pruned path + echo "$pruned_path"; + #declare -p path prune_level parts pruned_path && printf "========\n"; # debug + return 0; +}; # prune path rootside to int specified level +get_path_hierarchy_level() { + # Desc: Outputs hierarchy level of input paths + # Example: $ cat lines.txt | get_path_hierarchy_level + # Input: stdin str lines with /-delimited paths + # Output: stdout int hierarchy level of each path + # Version: 0.0.1 + + local line level; + local flag_root; + local -a output; + + n=0; + while read -r line; do + # Check for mixed absolute/relative paths. + if [[ $n -le 0 ]] && [[ "$line" =~ ^/ ]]; then + flag_root=true; + else + flag_root=false; + fi; + if { [[ "$flag_root" == "true" ]] && [[ ! "$line" =~ ^/ ]]; } || \ + { [[ "$flag_root" == "false" ]] && [[ "$line" =~ ^/ ]]; } then + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; return 1; + fi; + + # Squeeze multiple slashes and remove trailing slashes + line="$(echo "$line" | tr -s '/' | sed 's:/*$::' )"; + + # Count the number of slashes to determine hierarchy level + level="$(echo "$line" | awk -F'/' '{print NF-1}' )"; + if [[ "$flag_root" == "true" ]]; then ((level--)); fi; + + # Append to output + output+=("$level"); + #declare -p flag_root level; # debug + ((n++)); + done; + # Print output + printf "%s\n" "${output[@]}"; +}; # return hierarchy level of lines as integers validate_subpage_list() { # Desc: Check for illegal characters in subpage titles # Input: stdin unvalidated subpage list @@ -97,18 +220,27 @@ validate_subpage_list() { echo "FATAL:Error reading stdin." 1>&2; return 1; }; }; generate_wikicode() { + # Desc: Generates navigational link wikicode for subpages # Input: stdin validated subpage list # Output: stdout wikicode - local lprev lnext; + # Depends: get_path_fork_level() + # prune_path_rootside() + # get_path_hierarchy_level() n=0; while read -r line; do #yell "$n:Processing line:$line"; # debug + # Advance input lines lprev="$lcurr"; lcurr="$lnext"; lnext="$line"; #declare -p lprev lcurr lnext; # debug + # Update hierarchy tracker states + lprev_hier="$lcurr_hier"; + lcurr_hier="$lnext_hier"; + lnext_hier="$(echo "$lnext" | get_path_hierarchy_level)"; + # Skip first iteration if [[ "$n" -eq 0 ]]; then # yell "$n:DEBUG:Skipping first iteration."; # debug @@ -116,30 +248,42 @@ generate_wikicode() { #printf -- "----\n" 1>&2; # debug continue; fi; - # Handle first valid input set - # yell "$n:DEBUG:Handling first valid input set."; # debug - if [[ "$n" -eq 1 ]]; then - printf "[[../%s|Next]], [[../|Up]]\n" \ - "$lnext"; - #printf -- "----\n" 1>&2; # debug - ((n++)); continue; fi; - - # Handle middle lines - # yell "$n:DEBUG:Handling middle lines."; # debug - printf "[[../%s|Next]], [[../%s|Previous]], [[../|Up]]\n" \ - "$lnext" "$lprev"; - ((n++)); - #printf -- "----\n" 1>&2; # debug - done; + # Get path fork levels + fork_level_next="$(get_path_fork_level "$lcurr" "$lnext")"; + fork_level_prev="$(get_path_fork_level "$lcurr" "$lprev")"; - # Handle last line - lprev="$lcurr"; - lcurr="$lnext"; - lnext="$line"; - printf "[[../%s|Previous]], [[../|Up]]\n" \ - "$lprev"; - ((n++)); -}; + # Count relative ups needed (`../`) + relups_next="$((lcurr_hier - fork_level_next + 1))"; + relups_prev="$((lcurr_hier - fork_level_prev + 1))"; + + # Initialize Next and Prev links with relative ups to fork. + link_next=""; + for (( i=0; i&2; + fi; + + #declare -p n lprev lcurr lnext lprev_hier lcurr_hier lnext_hier; # debug + #declare -p fork_level_next fork_level_prev relups_next relups_prev; # debug + #declare -p link_next link_prev; # debug + ((n++)); + done < <(read_stdin); # read stdin plus one more blank line +}; # Generate wikicode from validated subpage lines main() { read_input "$@" | validate_subpage_list | generate_wikicode; }; # main program -- 2.30.2 From c7c5b9096051b6f4f585ba222041ddd48dd9a135 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 14:29:21 +0000 Subject: [PATCH 06/16] chore(prvt):Update --- prvt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prvt b/prvt index 77f7b15..c7a0852 160000 --- a/prvt +++ b/prvt @@ -1 +1 @@ -Subproject commit 77f7b156f3b0b92706967702088055768446807b +Subproject commit c7a08526e21baf55fa3a76fab5a1c05cce1a1196 -- 2.30.2 From 5ed2992871e3395e1c39387116bedaa3dbbe9cf9 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 17:13:49 +0000 Subject: [PATCH 07/16] feat(user/bkots):Execute via array elements, not string concat - Note: v1.0.3 --> v2.0.0 --- user/bkots | 116 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 35 deletions(-) diff --git a/user/bkots b/user/bkots index baffb08..cca1fd8 100755 --- a/user/bkots +++ b/user/bkots @@ -10,13 +10,13 @@ calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); -declare -a commands # array for storing assembled commands age_threshold="60"; # min age to add file; seconds; +max_job_count="2"; # default max job count # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails checkapp() { # Desc: If arg is a command, save result in assoc array 'appRollCall' # Usage: checkapp arg1 arg2 arg3 ... @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 1.0.3 +bkots 2.0.0 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -239,6 +239,8 @@ showUsage() { --include-dotfiles Include files and directories starting with '.' (not included by default). + -j, --jobs + Specify simultaneous job count (default: 2) -r, --recursive Consider files in dirs recursively. --version @@ -268,6 +270,32 @@ showUsage() { bkots foo.txt bar.pdf /home/username/Pictures/ EOF } # Display information on how to use this script. +count_jobs() { + # Desc: Count and return total number of jobs + # Usage: count_jobs + # Input: None. + # Output: stdout integer number of jobs + # Depends: Bash 5.1.16 + # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; + # Version: 0.0.1 + + local job_count; + job_count="$(jobs -r | wc -l | tr -d ' ' )"; + #yell "DEBUG:job_count:$job_count"; + if [[ -z $job_count ]]; then job_count="0"; fi; + echo "$job_count"; +}; # Return number of background jobs +wait_for_jobslot() { + # Desc: Does not return until count_jobs() falls below $max_job_count + # Input: var max_job_count + # Output: return code 0 + # Depends: count_jobs(), yell(); + while [[ $(count_jobs) -ge $max_job_count ]]; do + printf "\r%d:Working..." "$SECONDS"; + sleep 1; + done; + return 0; +}; processArgs() { # Desc: Processes arguments provided to script. # Usage: processArgs "$@" @@ -302,6 +330,15 @@ processArgs() { --include-dotfiles) # Include dotfiles option_include_dotfiles="true"; vbm "DEBUG:Option enabled:include dotfiles";; + -j | --jobs) # Specify max_job_count + if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then + max_job_count="$2"; + vbm "STATUS:Max job count set to:$max_job_count"; + shift; + else + showUsage; + die "FATAL:Invalid job count:$2"; + fi;; -r | --recursive) # Specify recursive option option_recursive="true"; vbm "DEBUG:option enabled:include files in dirs recursively";; @@ -336,7 +373,7 @@ get_parent_dirnames() { # Input: arg1 input path # Output: stdout newline-delimited list of parent dirs # Version: 0.0.1 - # Depends: yell(), die(), try() + # Depends: yell(), die(), must() local path # Check input @@ -354,14 +391,6 @@ get_parent_dirnames() { echo "$name_base"; done; }; # Output parent dirnames to stdout -cmdwrap() { - # print command to stderr - echo "$@" 1>&2; - - # execute command - "$@"; -}; # print and execute string together -export -f cmdwrap; # export cmdwrap for use in other functions main() { # Desc: Creates `.ots` file: # - for each file specified in arrayPosArgs array @@ -640,7 +669,7 @@ main() { fi; # Act on files - ## Assemble upgrade file commands + ## Assemble and execute upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. @@ -653,11 +682,19 @@ main() { ### Try upgrade with known calendars in random order while read -r url; do vbm "DEBUG:Upgrading with calendar:url:$url"; - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; - else - commands+=("cmdwrap ots -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; - fi; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("upgrade" "$path_prf_sesc"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + wait_for_jobslot && must "${cmd_temp[@]}" & + unset cmd_temp; + break; #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; done < <(printf "%s\n" "${calendars[@]}" | shuf); else @@ -665,7 +702,7 @@ main() { fi; done; - ## Assemble verify file commands + ## Assemble and execute verify file commands for item in "${files_to_verify_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; @@ -681,11 +718,19 @@ main() { ### Try verify with known calendars in random order while read -r url; do vbm "DEBUG:Verifying with calendar:url:$url"; - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; - else - commands+=("cmdwrap ots -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; - fi; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src_sesc" "$path_prf_sesc"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + wait_for_jobslot && must "${cmd_temp[@]}" & + unset cmd_temp; + break; #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; done < <(printf "%s\n" "${calendars[@]}" | shuf); else @@ -693,7 +738,7 @@ main() { fi; done; - ## Assemble stamp file commands + ## Assemble and execute stamp file commands for item in "${files_to_stamp_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. @@ -703,21 +748,22 @@ main() { fi; vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; if [[ ! $option_dry_run == "true" ]]; then - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v stamp \"$path_src_sesc\""); - else - commands+=("cmdwrap ots stamp \"$path_src_sesc\""); - fi; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("stamp" "$path_src_sesc"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + wait_for_jobslot && must "${cmd_temp[@]}" & + unset cmd_temp; #ots stamp "$path_src"; else yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; fi; done; - - ## Run commands - #yell "DEBUG:commands:$(printf "%s\n" "${commands[@]}")"; - printf "%s\n" "${commands[@]}" | parallel --group --jobs 25%; - }; # main program # Run program -- 2.30.2 From 26ab84d93abd3ccaa2b2c8871a8dfc9e937c06e7 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 17:42:14 +0000 Subject: [PATCH 08/16] update(user/bkots):Remove unnecessary must() --- user/bkots | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/user/bkots b/user/bkots index cca1fd8..b38f0ac 100755 --- a/user/bkots +++ b/user/bkots @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.0.0 +bkots 2.0.1 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -692,7 +692,7 @@ main() { if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command - wait_for_jobslot && must "${cmd_temp[@]}" & + wait_for_jobslot && "${cmd_temp[@]}" & unset cmd_temp; break; #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; @@ -728,7 +728,7 @@ main() { if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command - wait_for_jobslot && must "${cmd_temp[@]}" & + wait_for_jobslot && "${cmd_temp[@]}" & unset cmd_temp; break; #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; @@ -757,7 +757,7 @@ main() { if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command - wait_for_jobslot && must "${cmd_temp[@]}" & + wait_for_jobslot && "${cmd_temp[@]}" & unset cmd_temp; #ots stamp "$path_src"; else -- 2.30.2 From 8378a004f4c28569f463bf39da4ffe32f7d41167 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 18:13:49 +0000 Subject: [PATCH 09/16] fix(user/bkots):Remove unnecessary backslash escaping --- user/bkots | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/user/bkots b/user/bkots index b38f0ac..a7f61ee 100755 --- a/user/bkots +++ b/user/bkots @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.0.1 +bkots 2.0.2 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -406,7 +406,6 @@ main() { # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194 # [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028 # [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377 - # [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001 local -a file_list file_list_pruned; local -a files_to_verify files_to_upgrade files_to_stamp local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned @@ -672,7 +671,6 @@ main() { ## Assemble and execute upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_prf" ]]; then yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; continue; @@ -688,7 +686,7 @@ main() { cmd_temp+=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("upgrade" "$path_prf_sesc"); + cmd_temp+=("upgrade" "$path_prf"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command @@ -706,8 +704,6 @@ main() { for item in "${files_to_verify_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. - path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then yell "ERROR:blank verify item encountered. Skipping:item:$item"; continue; @@ -724,7 +720,7 @@ main() { cmd_temp+=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("verify" "-f" "$path_src_sesc" "$path_prf_sesc"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command @@ -741,7 +737,6 @@ main() { ## Assemble and execute stamp file commands for item in "${files_to_stamp_pruned[@]}"; do path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. if [[ -z "$path_src" ]]; then yell "ERROR:blank stamp item encountered. Skipping:item:$item"; continue; @@ -753,7 +748,7 @@ main() { local -a cmd_temp; cmd_temp+=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("stamp" "$path_src_sesc"); + cmd_temp+=("stamp" "$path_src"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command -- 2.30.2 From 21beb7be61fdf90ea0df0a542c5f29f70a4a2434 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 18:32:45 +0000 Subject: [PATCH 10/16] feat(user/bkots):Use subshells to speed up jobs --- user/bkots | 167 ++++++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 79 deletions(-) diff --git a/user/bkots b/user/bkots index a7f61ee..ec371f6 100755 --- a/user/bkots +++ b/user/bkots @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Define variables +declare -g max_job_count="2"; # default max job count +declare -g age_threshold="60"; # min age to add file; seconds; declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status @@ -10,8 +12,6 @@ calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); -age_threshold="60"; # min age to add file; seconds; -max_job_count="2"; # default max job count # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.0.2 +bkots 2.1.0 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -666,99 +666,108 @@ main() { vbm "DEBUG:files_to_stamp_pruned:"; printf "%s\n" "${files_to_stamp_pruned[@]}"; fi; - + # Act on files ## Assemble and execute upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do - path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_prf" ]]; then - yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try upgrade with known calendars in random order - while read -r url; do - vbm "DEBUG:Upgrading with calendar:url:$url"; - - #### assemble command - local -a cmd_temp; - cmd_temp+=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("upgrade" "$path_prf"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & - unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; - fi; + wait_for_jobslot && { + path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_prf" ]]; then + yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try upgrade with known calendars in random order + while read -r url; do + vbm "DEBUG:Upgrading with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("upgrade" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; + fi; + } & done; ## Assemble and execute verify file commands for item in "${files_to_verify_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then - yell "ERROR:blank verify item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; - vbm "DEBUG: against proof file: path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try verify with known calendars in random order - while read -r url; do - vbm "DEBUG:Verifying with calendar:url:$url"; + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then + yell "ERROR:blank verify item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; + vbm "DEBUG: against proof file: path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try verify with known calendars in random order + while read -r url; do + vbm "DEBUG:Verifying with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; + fi; + } & + done; + + ## Assemble and execute stamp file commands + for item in "${files_to_stamp_pruned[@]}"; do + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_src" ]]; then + yell "ERROR:blank stamp item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; + if [[ ! $option_dry_run == "true" ]]; then #### assemble command local -a cmd_temp; cmd_temp+=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + cmd_temp+=("stamp" "$path_src"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & + "${cmd_temp[@]}"; unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; - fi; - done; - - ## Assemble and execute stamp file commands - for item in "${files_to_stamp_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_src" ]]; then - yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; - if [[ ! $option_dry_run == "true" ]]; then - - #### assemble command - local -a cmd_temp; - cmd_temp+=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("stamp" "$path_src"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & - unset cmd_temp; - #ots stamp "$path_src"; - else - yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; - fi; + #ots stamp "$path_src"; + else + yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; + fi; + } & done; + + ## Wait for jobs to finish. + wait; }; # main program # Run program -- 2.30.2 From 3329584b8848c977d0d05bf6757cf4ecb812efe3 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 18:38:59 +0000 Subject: [PATCH 11/16] fix(user/bkots):Fix shellcheck delinter complaints --- user/bkots | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/user/bkots b/user/bkots index ec371f6..3c9705c 100755 --- a/user/bkots +++ b/user/bkots @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.1.0 +bkots 2.1.1 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -674,7 +674,7 @@ main() { path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; if [[ -z "$path_prf" ]]; then yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - continue; + return 1; # would have been `continue` were it not in a subshell fi; vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; if [[ ! $option_dry_run == "true" ]]; then @@ -684,7 +684,7 @@ main() { #### assemble command local -a cmd_temp; - cmd_temp+=("ots"); + cmd_temp=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; cmd_temp+=("-l" "$url" "--no-default-whitelist"); cmd_temp+=("upgrade" "$path_prf"); @@ -709,7 +709,7 @@ main() { path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then yell "ERROR:blank verify item encountered. Skipping:item:$item"; - continue; + return 1; # would have been `continue` were it not in a subshell fi; vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; vbm "DEBUG: against proof file: path_prf:$path_prf"; @@ -720,7 +720,7 @@ main() { #### assemble command local -a cmd_temp; - cmd_temp+=("ots"); + cmd_temp=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; cmd_temp+=("-l" "$url" "--no-default-whitelist"); cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); @@ -744,14 +744,14 @@ main() { path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; if [[ -z "$path_src" ]]; then yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - continue; + return 1; # would have been `continue` were it not in a subshell fi; vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; if [[ ! $option_dry_run == "true" ]]; then #### assemble command local -a cmd_temp; - cmd_temp+=("ots"); + cmd_temp=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; cmd_temp+=("stamp" "$path_src"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; -- 2.30.2 From cb0f307f5af95dd07eb32050bc060342f0097f54 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 21:16:24 +0000 Subject: [PATCH 12/16] feat(user/bkytpldl-generic):Add generic yt-dlp wrapper --- user/bkytpldl-generic | 110 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 user/bkytpldl-generic diff --git a/user/bkytpldl-generic b/user/bkytpldl-generic new file mode 100644 index 0000000..ce019e8 --- /dev/null +++ b/user/bkytpldl-generic @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Desc: Download YouTube videos +# Usage: $ ./bkytpldl-generic +# Version: 4.0.0 + +# Adjust me +dir_out="~/Videos/"; +url_playlist1="https://www.youtube.com/playlist?list=PLxxx"; +url_playlist2="https://www.youtube.com/playlist?list=PLxxx"; +url_playlist3="https://www.youtube.com/playlist?list=PLxxx"; + +declare -a args; # array for yt-dlp arguments +declare -a urls urls_rand; # array for YouTube playlist URLs + +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails + +# check dependencies +if ! command -v yt-dlp 1>/dev/random 2>&1; then die "FATAL:yt-dlp not found."; fi; + +# Donʼt run multiple yt-dlp instances +if pgrep "^yt-dlp$" 1>/dev/random 2>&1; then die "FATAL:yt-dlp already running."; fi; + +# Check directories +if [[ ! -d $dir_out ]]; then mkdir -p "$dir_out"; fi; + +# == Download Video == + +# yt-dlp output options +## Restrict file name character set +#args+=("--restrict-filenames"); # Remove non-ASCII characters +args+=("--trim-filenames=120"); # Use in tandem with `%(title).120B` + +## Request to write accompanying files +args+=("--write-subs"); # Write subtitles file +args+=("--write-auto-subs"); # Write subtitles file +#args+=("--all-subs"); # Download all available subtitles (causes many requests) +#subLangs="en.*,ja.*,id.*,es.*,zh-Hans.*,zh-Hant.*,sv.*,el.*,hi.*,ru.*,bn.*,fr.*,ko.*,ar.*,nv.*"; # custom language list +subLangs="en,en-orig,en.*"; # custom language list +args+=("--sub-langs" "$subLangs"); +args+=("--write-info-json"); # Write accompanying json file +args+=("--no-overwrites"); # Don't overwrite files +args+=("--write-thumbnail"); # Write thumbnail + +## Only download metadata +#args+=("--no-download"); # Don't download video file. + +## Save meta-data +args+=("--write-comments"); # Get comments +### Limit comments +### comment_sort values: +### top : use YouTube top comment algorithm +### new : get newest comments (default) +### max_comments values: +### max-comments : max number of parent comments or replies +### max-parents : max number of comment threads +### max-replies : max number of replies across all threads +### max-replies-per-thread : max number of replies per thread +args+=("--extractor-args" "youtube:comment_sort=top;max_comments=10000,100,10000,100"); + +## Randomize order in which playlist items are downloaded +args+=("--playlist-random"); + +## Delay between downloads +minSleep="30"; +maxSleep="$(( minSleep + (RANDOM + RANDOM + RANDOM) / ( 3 * 400) ))"; # roughly 60 seconds +args+=("--min-sleep-interval" "$minSleep"); +args+=("--max-sleep-interval" "$maxSleep"); +args+=("--sleep-requests" "2"); # delay on metadata requests +args+=("--sleep-subtitles" "10"); # delay for subtitles + +## Remember downloaded videos to avoid redownload attempts +pathDA="$dir_out"/.bkytpldl_history.txt; +args+=("--download-archive" "$pathDA"); + +## Use firefox 'default-release' profile cookies +## Example: Linux: from ~/.mozilla/firefox/deadbeef.default-release/ +#args+=("--cookies-from-browser"); +#args+=("firefox:deadbeef.default-release"); Default Firefox profile name + +## Specify output filename format +## Note: `$(title).120B` shortens title to 120 bytes (useful for +## titles with UTF-8 characters. +args+=("-o"); +args+=("%(playlist)s/%(upload_date)s.%(channel)s.%(channel_id)s.%(title).120B.%(id)s.%(ext)s"); + +## Limit download resolution to 1080p +args+=("-S" "res:1080"); + +## Specify playlist URLs to download +urls+=("$url_playlist1"); +urls+=("$url_playlist2"); +urls+=("$url_playlist3"); +### Shuffle playlist download order +mapfile -t urls_rand < <(printf "%s\n" "${urls[@]}" | shuf); +for url in "${urls_rand[@]}"; do + args+=("$url"); +done; + +# Change working directory to output dir +pushd "$dir_out" || die "FATAL:Failed to change pwd to:dir_out:$dir_out"; + +# Download videos +#yell "DEBUG:args:$(declare -p args)"; # debug command +must yt-dlp "${args[@]}"; # execute command +popd || die "FATAL:Failed to return from dir_out:$dir_out"; + +# Author: Steven Baltakatei Sandoval +# License; GPLv3+ -- 2.30.2 From 2a4d0fe27da3613470cc2f19d3586acb44900499 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 21:29:15 +0000 Subject: [PATCH 13/16] style(user/bkytpldl-generic):Cleanup script settings section --- user/bkytpldl-generic | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/user/bkytpldl-generic b/user/bkytpldl-generic index ce019e8..d94bfac 100644 --- a/user/bkytpldl-generic +++ b/user/bkytpldl-generic @@ -1,17 +1,17 @@ #!/usr/bin/env bash # Desc: Download YouTube videos # Usage: $ ./bkytpldl-generic -# Version: 4.0.0 - -# Adjust me -dir_out="~/Videos/"; -url_playlist1="https://www.youtube.com/playlist?list=PLxxx"; -url_playlist2="https://www.youtube.com/playlist?list=PLxxx"; -url_playlist3="https://www.youtube.com/playlist?list=PLxxx"; +# Version: 4.1.0 declare -a args; # array for yt-dlp arguments declare -a urls urls_rand; # array for YouTube playlist URLs +# Settings +dir_out="~/Videos/"; +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me + yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails @@ -25,7 +25,7 @@ if pgrep "^yt-dlp$" 1>/dev/random 2>&1; then die "FATAL:yt-dlp already running." # Check directories if [[ ! -d $dir_out ]]; then mkdir -p "$dir_out"; fi; -# == Download Video == +# == Assemble options == # yt-dlp output options ## Restrict file name character set @@ -89,9 +89,6 @@ args+=("%(playlist)s/%(upload_date)s.%(channel)s.%(channel_id)s.%(title).120B.%( args+=("-S" "res:1080"); ## Specify playlist URLs to download -urls+=("$url_playlist1"); -urls+=("$url_playlist2"); -urls+=("$url_playlist3"); ### Shuffle playlist download order mapfile -t urls_rand < <(printf "%s\n" "${urls[@]}" | shuf); for url in "${urls_rand[@]}"; do @@ -101,7 +98,8 @@ done; # Change working directory to output dir pushd "$dir_out" || die "FATAL:Failed to change pwd to:dir_out:$dir_out"; -# Download videos +# == Download videos == + #yell "DEBUG:args:$(declare -p args)"; # debug command must yt-dlp "${args[@]}"; # execute command popd || die "FATAL:Failed to return from dir_out:$dir_out"; -- 2.30.2 From 107b9c5341a52d03350d698b5ec42e26d810a93c Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 21:30:59 +0000 Subject: [PATCH 14/16] style(user/bkytpldl-generic):Adjust comment --- user/bkytpldl-generic | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/user/bkytpldl-generic b/user/bkytpldl-generic index d94bfac..ae79832 100644 --- a/user/bkytpldl-generic +++ b/user/bkytpldl-generic @@ -1,16 +1,16 @@ #!/usr/bin/env bash # Desc: Download YouTube videos # Usage: $ ./bkytpldl-generic -# Version: 4.1.0 +# Version: 4.1.1 declare -a args; # array for yt-dlp arguments declare -a urls urls_rand; # array for YouTube playlist URLs # Settings dir_out="~/Videos/"; -urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me -urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me -urls+=("https://www.youtube.com/playlist?list=PLxxx"); # adjust me +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -- 2.30.2 From 203add48226f4fb9e6dbdde301e70daa15ab1a54 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Sun, 28 Jul 2024 20:32:17 +0000 Subject: [PATCH 15/16] style(user/bknpass):Simplify boilerplate --- user/bknpass | 74 ++++++++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/user/bknpass b/user/bknpass index c1c1d1a..678ab7f 100755 --- a/user/bknpass +++ b/user/bknpass @@ -1,59 +1,32 @@ #!/bin/bash - -# Author: Steven Baltakatei Sandoval (baltakatei.com) -# -# License: This bash script, `bknpass`, is licensed under GPLv3 or -# later by Steven Baltakatei Sandoval: -# -# `bknpass`, an alphanumeric password generator -# Copyright (C) 2021 Steven Baltakatei Sandoval (baltakatei.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# A copy of the GNU General Public License may be found at -# . -# -# Description: This bash script generates alphanumeric passphrases -# with a char-count determined by a user-provided number of bits of -# entropy. The passphrase is then outputted to stdout with a trailing -# newline. It works as follows: +# Desc: Generate alphanumeric passphrase with specified bits of entropy. +# Usage: bknpass [int] +# Example: bknpass 128 +# Depends: bash, echo, bc, awk, tr, head, Debian 10 +# Version: 0.1.0 +# Note: This bash script generates alphanumeric passphrases +# with a char-count determined by a user-provided number of bits of +# entropy. The passphrase is then outputted to stdout with a trailing +# newline. It works as follows: # -# - Prompt user for an integer. This integer is the number of bits -# of entropy that the generated password should have. +# - Prompt user for an integer. This integer is the number of bits +# of entropy that the generated password should have. # -# - Check if user-provided string is an integer using `bash` regular -# expression test. +# - Check if user-provided string is an integer using `bash` regular +# expression test. # -# - Calculate the minimum number of bech32 base32 characters -# required to encode the specified number of bits of entropy. +# - Calculate the minimum number of bech32 base32 characters +# required to encode the specified number of bits of entropy. # -# - This step uses `bc` to calculate a logarithm float string -# and `awk` to convert the float into an integer, rounding up. +# - This step uses `bc` to calculate a logarithm float string +# and `awk` to convert the float into an integer, rounding up. # -# - Use `tr`, `/dev/urandom`, and `head` to generate a random -# alphanumeric string with the length calculated in the previous -# step. +# - Use `tr`, `/dev/urandom`, and `head` to generate a random +# alphanumeric string with the length calculated in the previous +# step. # -# - Use `echo` to display the passphrase in stdout with a trailing -# newline. -# -# Usage: bknpass [int] -# -# Example: bknpass 256 -# -# Dependencies: bash, echo, bc, awk, tr, head. See end of file -# -# Tested on: -# -# - GNU/Linux Debian 10 +# - Use `echo` to display the passphrase in stdout with a trailing +# newline. #==Initialization== @@ -209,3 +182,6 @@ echo -e "$PASS1" # There is NO WARRANTY, to the extent permitted by law. # # Written by David MacKenzie and Jim Meyering. + +# Author: Steven Baltakatei Sandoval +# License: GPLv3+ -- 2.30.2 From 738ad68b7df736f438f74dfeffd56e400fb2c1bf Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Sun, 28 Jul 2024 20:42:13 +0000 Subject: [PATCH 16/16] feat(user/bknpass):Move unitproc ver to user - chore(unitproc/bknpass):Delete --- unitproc/bknpass | 189 ------------------------------------------ user/bknpass | 208 ++++++++++++++++++++++++----------------------- 2 files changed, 105 insertions(+), 292 deletions(-) delete mode 100755 unitproc/bknpass diff --git a/unitproc/bknpass b/unitproc/bknpass deleted file mode 100755 index f4930a0..0000000 --- a/unitproc/bknpass +++ /dev/null @@ -1,189 +0,0 @@ -#!/bin/bash -# Desc: Generate passphrase with specified number of bits of entropy -# Usage: bknpass [integer] -# Example: bknpass 256 -# Result: 9tnzcl0m4dsm22a95525zj93jj -# Version: 0.2.1 -# Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 -# License: -# `bknpass`, an alphanumeric password generator -# Copyright (C) 2022 Steven Baltakatei Sandoval (baltakatei.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# A copy of the GNU General Public License may be found at -# . - -yell() { echo "$0: $*" >&2; } # print script path and all args to stderr -die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails -showUsage() { - # Desc: Display script usage information - # Usage: showUsage - # Version 0.0.1 - # Input: none - # Output: stdout - # Depends: GNU-coreutils 8.30 (cat) - cat <<'EOF' - USAGE: - bknpass [int entropy_bit_count] - - EXAMPLE: - bknpass 128 -EOF -}; # Display information on how to use this script. -main() { - # Desc: main program - # Usage: main "$@" - # Input: $1: integer bits of entropy - # Output: stdout: string passphrase - # Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 - # Ref/Attrib: [1] "Check existence of input argument in a Bash shell script". https://stackoverflow.com/a/6482403 - # [2] "How do I test if a variable is a number in Bash?". https://stackoverflow.com/a/806923 - # [3] "Round a Number in a Bash Script". https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html - # [4] "Logarithms in GNU bc". https://web.archive.org/web/20210211050732/http://phodd.net/gnu-bc/bcfaq.html#bclog - # [5] "BIP 173". Note: bech32 charset. https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki - # [6] "Bash Function to generate random password". https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ - charset="qpzry9x8gf2tvdw0s3jn54khce6mua7l"; # bech32. See [5]. - charset="$(tr -s "$charset" < <(echo -n "$charset"))"; # remove repeat chars - alphabet_size="$(echo -n "$charset" | wc -c)"; # number of unique chars - log_base=2; # entropy unit base (i.e. 2 for "bits of entropy") - - # Check inputs - if [[ $# -gt 1 ]]; then showUsage; die "ERROR:Too many arguments:$#"; fi; - - # Check for bc - if ! command -v bc 1>/dev/null 2>&1; then die "ERROR:bc not available"; fi; - - ## Get entropy_bit_count - if [[ -z "$1" ]]; then - ### prompt user - showUsage; - echo -n "Please specify the required strength of the password in bits of entropy (ex: 256):" 1>&2; # prompt via stderr - read -r entropy_bit_count - else - entropy_bit_count="$1"; - fi; # See [1]. - - ## Check entropy_bit_count - pattern="^[0-9]+$"; - if ! [[ $entropy_bit_count =~ $pattern ]] ; then die "ERROR:Not an integer:$entropy_bit_count"; fi; # See [2] - - ## Calculate minimum count of chars needed for passphrase as float using 'bc' - ### Solve ln(a^n)/ln(2)=b for n where: - ### a=$alphabet_size - ### n=$char_count_float - ### b=$entropy_bit_count - char_count_float="$(echo "$entropy_bit_count*l($log_base)/l($alphabet_size)" | bc -l)"; # See [4] - - ## Round $char_count_float to next highest integer - char_count="$(echo "$char_count_float" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')"; # See [3] - - ## Generate and output passphrase - echo "$(LC_ALL=C tr -cd "$charset" < /dev/urandom | head -c "$char_count")"; # See [6] -}; # main program - -( main "$@" ); # run 'main()' in subshell for 'die()' portability of script as sourced bash function. - -#==References== -# -# - How to echo a string as stderr instead of stdout. -# https://stackoverflow.com/a/2990533 -# Author: James Roth -# Date: 2010-06-07T14:52Z -# Date Accessed: 2020-01-20 -# -# - How to check if script argument exists or not. -# https://stackoverflow.com/a/6482403 -# Author: phoxix -# Date: 2011-06-26T05:55Z -# Date Accessed: 2020-01-20 -# -# - How to check that a string is an integer using regular expression test. -# https://stackoverflow.com/a/806923 -# Author: Charles Duffy -# Date: 2009-04-30T13:32Z -# Date Accessed: 2020-01-20 -# -# - How to use `bc` to calculate logarithms in Bash -# http://phodd.net/gnu-bc/bcfaq.html#bashlog -# Author: unknown -# Date Accessed: 2020-01-20 -# -# - How to use `awk` to convert and round up a float to an integer. -# https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html -# Author: Md. Minhazul Haque -# Date: 2015-01-09 -# Date Accessed: 2020-01-20 -# -# - How to use `/dev/urandom`, `tr`, and `head` to generate a random password in Bash. -# https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ -# Author: SASIKALA, Ramesh Natarajan -# Date: 2010-04-21 -# Date Accessed: 2020-01-20 -# -# - Bech32 base32 charset -# https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki -# Author: Pieter Wuille -# Date: 2017-03-20 -# License: BSD-2-Clause -# Date: Accessed: 2021-01-23 -# -# - Dependencies: bash, bc, echo, awk, tr, head -# -# - GNU bash, version 5.1.8(1)-release (x86_64-pc-linux-gnu) -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later -# This is free software; you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# - bc 1.07.1 -# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. -# -# - echo (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by Brian Fox and Chet Ramey. -# -# - GNU Awk 5.1.0, API: 3.0 (GNU MPFR 4.1.0, GNU MP 6.2.1) -# Copyright (C) 1989, 1991-2020 Free Software Foundation. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. -# -# - tr (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by Jim Meyering. -# -# - head (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by David MacKenzie and Jim Meyering. diff --git a/user/bknpass b/user/bknpass index 678ab7f..f4930a0 100755 --- a/user/bknpass +++ b/user/bknpass @@ -1,92 +1,97 @@ #!/bin/bash -# Desc: Generate alphanumeric passphrase with specified bits of entropy. -# Usage: bknpass [int] -# Example: bknpass 128 -# Depends: bash, echo, bc, awk, tr, head, Debian 10 -# Version: 0.1.0 -# Note: This bash script generates alphanumeric passphrases -# with a char-count determined by a user-provided number of bits of -# entropy. The passphrase is then outputted to stdout with a trailing -# newline. It works as follows: -# -# - Prompt user for an integer. This integer is the number of bits -# of entropy that the generated password should have. -# -# - Check if user-provided string is an integer using `bash` regular -# expression test. -# -# - Calculate the minimum number of bech32 base32 characters -# required to encode the specified number of bits of entropy. -# -# - This step uses `bc` to calculate a logarithm float string -# and `awk` to convert the float into an integer, rounding up. -# -# - Use `tr`, `/dev/urandom`, and `head` to generate a random -# alphanumeric string with the length calculated in the previous -# step. -# -# - Use `echo` to display the passphrase in stdout with a trailing -# newline. - - -#==Initialization== - -let ALPHABET_SIZE="32" # number of unique chars in bech32 base32 charset -LOG_BASE=2 # Set logarithm base to 2 +# Desc: Generate passphrase with specified number of bits of entropy +# Usage: bknpass [integer] +# Example: bknpass 256 +# Result: 9tnzcl0m4dsm22a95525zj93jj +# Version: 0.2.1 +# Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 +# License: +# `bknpass`, an alphanumeric password generator +# Copyright (C) 2022 Steven Baltakatei Sandoval (baltakatei.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# A copy of the GNU General Public License may be found at +# . -# Define `echoerr` function which outputs text to stderr - # Note: function copied from https://stackoverflow.com/a/2990533 -function echoerr { - echo "$@" 1>&2; -} +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +showUsage() { + # Desc: Display script usage information + # Usage: showUsage + # Version 0.0.1 + # Input: none + # Output: stdout + # Depends: GNU-coreutils 8.30 (cat) + cat <<'EOF' + USAGE: + bknpass [int entropy_bit_count] -# Define `rpass` function which generates a base32 passphrase of length $1 (ex: `rpass 20` generates a 20-char string) -# Note: function adapted from https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ -# Note: base32 charset uses bech32 charset -function rpass { - cat /dev/urandom | LC_ALL=C tr -cd "qpzry9x8gf2tvdw0s3jn54khce6mua7l" | head -c ${1:-22} -} + EXAMPLE: + bknpass 128 +EOF +}; # Display information on how to use this script. +main() { + # Desc: main program + # Usage: main "$@" + # Input: $1: integer bits of entropy + # Output: stdout: string passphrase + # Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 + # Ref/Attrib: [1] "Check existence of input argument in a Bash shell script". https://stackoverflow.com/a/6482403 + # [2] "How do I test if a variable is a number in Bash?". https://stackoverflow.com/a/806923 + # [3] "Round a Number in a Bash Script". https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html + # [4] "Logarithms in GNU bc". https://web.archive.org/web/20210211050732/http://phodd.net/gnu-bc/bcfaq.html#bclog + # [5] "BIP 173". Note: bech32 charset. https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki + # [6] "Bash Function to generate random password". https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ + charset="qpzry9x8gf2tvdw0s3jn54khce6mua7l"; # bech32. See [5]. + charset="$(tr -s "$charset" < <(echo -n "$charset"))"; # remove repeat chars + alphabet_size="$(echo -n "$charset" | wc -c)"; # number of unique chars + log_base=2; # entropy unit base (i.e. 2 for "bits of entropy") + # Check inputs + if [[ $# -gt 1 ]]; then showUsage; die "ERROR:Too many arguments:$#"; fi; -#==Main Program== + # Check for bc + if ! command -v bc 1>/dev/null 2>&1; then die "ERROR:bc not available"; fi; -# Define $ENTROPY_BIT_COUNT1 as argument $1 or prompt user if $1 is not defined. -# Note: argument test adapted from https://stackoverflow.com/a/6482403 -if [ -z "$1" ] -then - echo "Entropy bit count argument (\$1) not supplied." - # Get from user the number of bits of entropy. - echoerr -n "Please specify the required strength of the password in bits of entropy (ex: 256):" # prompt via stderr - read ENTROPY_BIT_COUNT1 -else - ENTROPY_BIT_COUNT1="$1" -fi + ## Get entropy_bit_count + if [[ -z "$1" ]]; then + ### prompt user + showUsage; + echo -n "Please specify the required strength of the password in bits of entropy (ex: 256):" 1>&2; # prompt via stderr + read -r entropy_bit_count + else + entropy_bit_count="$1"; + fi; # See [1]. -# Check if $ENTROPY_BIT_COUNT1 is an non-negative integer -# Note: Regular expression test is adapted from https://stackoverflow.com/a/806923 -RETEST1='^[0-9]+$' -if ! [[ $ENTROPY_BIT_COUNT1 =~ $RETEST1 ]] ; then - echo "error: Not an integer." >&2; exit 1 -fi + ## Check entropy_bit_count + pattern="^[0-9]+$"; + if ! [[ $entropy_bit_count =~ $pattern ]] ; then die "ERROR:Not an integer:$entropy_bit_count"; fi; # See [2] -# Calculate minimum count of chars needed to encode $ENTROPY_BIT_COUNT1 with alphabet size of $ALPHABET_SIZE as float -# Solve ln(a^n)/ln(2)=b for n using `bc` where -# a=$ALPHABET_SIZE -# n=$CHAR_COUNT1_FLOAT -# b=$ENTROPY_BIT_COUNT1 -# Note: `bc` logarithm usage adapted from http://phodd.net/gnu-bc/bcfaq.html#bashlog -CHAR_COUNT1_FLOAT=$(echo "$ENTROPY_BIT_COUNT1*l($LOG_BASE)/l($ALPHABET_SIZE)" | bc -l) -# Note: Float will be of form "21.49744370650136860806". -# Note: This particular example float should be rounded to "22" later. + ## Calculate minimum count of chars needed for passphrase as float using 'bc' + ### Solve ln(a^n)/ln(2)=b for n where: + ### a=$alphabet_size + ### n=$char_count_float + ### b=$entropy_bit_count + char_count_float="$(echo "$entropy_bit_count*l($log_base)/l($alphabet_size)" | bc -l)"; # See [4] -# Round $CHAR_COUNT1_FLOAT1 up to next highest integer for use as argument in later bash functions. -# Note: awk expression from https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html -CHAR_COUNT1=$(echo "$CHAR_COUNT1_FLOAT" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}') + ## Round $char_count_float to next highest integer + char_count="$(echo "$char_count_float" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')"; # See [3] -# Generate passphrase -PASS1=$(rpass "$CHAR_COUNT1") -echo -e "$PASS1" + ## Generate and output passphrase + echo "$(LC_ALL=C tr -cd "$charset" < /dev/urandom | head -c "$char_count")"; # See [6] +}; # main program +( main "$@" ); # run 'main()' in subshell for 'die()' portability of script as sourced bash function. #==References== # @@ -132,56 +137,53 @@ echo -e "$PASS1" # License: BSD-2-Clause # Date: Accessed: 2021-01-23 # -# - Dependencies: bash, echo, bc, awk, tr, head. +# - Dependencies: bash, bc, echo, awk, tr, head # -# - GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) -# Copyright (C) 2019 Free Software Foundation, Inc. +# - GNU bash, version 5.1.8(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later # This is free software; you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# -# - echo (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# +# - bc 1.07.1 +# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. +# +# - echo (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. # # Written by Brian Fox and Chet Ramey. -# -# - bc 1.07.1 -# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. -# -# - GNU Awk 4.2.1, API: 2.0 (GNU MPFR 4.0.2, GNU MP 6.1.2) -# Copyright (C) 1989, 1991-2018 Free Software Foundation. -# +# +# - GNU Awk 5.1.0, API: 3.0 (GNU MPFR 4.1.0, GNU MP 6.2.1) +# Copyright (C) 1989, 1991-2020 Free Software Foundation. +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. # -# - tr (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# - tr (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# +# # Written by Jim Meyering. -# -# - head (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# +# - head (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# +# # Written by David MacKenzie and Jim Meyering. - -# Author: Steven Baltakatei Sandoval -# License: GPLv3+ -- 2.30.2