From: Steven Baltakatei Sandoval Date: Sun, 28 Jul 2024 20:42:13 +0000 (+0000) Subject: feat(user/bknpass):Move unitproc ver to user X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/commitdiff_plain/HEAD?hp=3dc0bdeffbb2c135ecf15672f52f5676c511866b feat(user/bknpass):Move unitproc ver to user - chore(unitproc/bknpass):Delete --- diff --git a/prvt b/prvt index 1e677a1..c7a0852 160000 --- a/prvt +++ b/prvt @@ -1 +1 @@ -Subproject commit 1e677a1f0ad42cab02345a89a73234367f9e1170 +Subproject commit c7a08526e21baf55fa3a76fab5a1c05cce1a1196 diff --git a/unitproc/bk_export_audio.sh b/unitproc/bk_export_audio.sh index 1100811..bbf3f3d 100755 --- a/unitproc/bk_export_audio.sh +++ b/unitproc/bk_export_audio.sh @@ -1,7 +1,7 @@ #!/bin/bash # Desc: Extracts audio from video files # Usage: bk_export_audio.sh [input_dir] ([output_dir]) -# Version: 0.1.0 +# Version: 0.1.1 # Depends: bash 5.1.16, GNU Coreutils (8.32) # Plumbing @@ -226,14 +226,18 @@ extract_audio_file() { # arg2: desired output file extension # arg3: output dir path # Output: audio file at path [arg3]/[arg1].[arg2] - local file_in file_in_ext dir_out file_in_basename; + local file_in file_in_ext dir_out file_in_basename path_out; file_in="$1"; file_in_ext="$2"; dir_out="$3"; + file_in_basename="$(basename "$file_in")"; + path_out="$dir_out"/"$file_in_basename"."$file_in_ext"; + + # Skip if output file already exists. + if [[ -f "$path_out" ]]; then return 1; fi; # Extract audio file - file_in_basename="$(basename "$file_in")"; - ffmpeg -i "$file_in" -vn -acodec copy "$dir_out"/"$file_in_basename"."$file_in_ext"; + ffmpeg -i "$file_in" -vn -acodec copy "$path_out"; } # Create audio file from video file count_jobs() { # Desc: Count and return total number of jobs diff --git a/unitproc/bknpass b/unitproc/bknpass deleted file mode 100755 index f4930a0..0000000 --- a/unitproc/bknpass +++ /dev/null @@ -1,189 +0,0 @@ -#!/bin/bash -# Desc: Generate passphrase with specified number of bits of entropy -# Usage: bknpass [integer] -# Example: bknpass 256 -# Result: 9tnzcl0m4dsm22a95525zj93jj -# Version: 0.2.1 -# Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 -# License: -# `bknpass`, an alphanumeric password generator -# Copyright (C) 2022 Steven Baltakatei Sandoval (baltakatei.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# A copy of the GNU General Public License may be found at -# . - -yell() { echo "$0: $*" >&2; } # print script path and all args to stderr -die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails -showUsage() { - # Desc: Display script usage information - # Usage: showUsage - # Version 0.0.1 - # Input: none - # Output: stdout - # Depends: GNU-coreutils 8.30 (cat) - cat <<'EOF' - USAGE: - bknpass [int entropy_bit_count] - - EXAMPLE: - bknpass 128 -EOF -}; # Display information on how to use this script. -main() { - # Desc: main program - # Usage: main "$@" - # Input: $1: integer bits of entropy - # Output: stdout: string passphrase - # Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 - # Ref/Attrib: [1] "Check existence of input argument in a Bash shell script". https://stackoverflow.com/a/6482403 - # [2] "How do I test if a variable is a number in Bash?". https://stackoverflow.com/a/806923 - # [3] "Round a Number in a Bash Script". https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html - # [4] "Logarithms in GNU bc". https://web.archive.org/web/20210211050732/http://phodd.net/gnu-bc/bcfaq.html#bclog - # [5] "BIP 173". Note: bech32 charset. https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki - # [6] "Bash Function to generate random password". https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ - charset="qpzry9x8gf2tvdw0s3jn54khce6mua7l"; # bech32. See [5]. - charset="$(tr -s "$charset" < <(echo -n "$charset"))"; # remove repeat chars - alphabet_size="$(echo -n "$charset" | wc -c)"; # number of unique chars - log_base=2; # entropy unit base (i.e. 2 for "bits of entropy") - - # Check inputs - if [[ $# -gt 1 ]]; then showUsage; die "ERROR:Too many arguments:$#"; fi; - - # Check for bc - if ! command -v bc 1>/dev/null 2>&1; then die "ERROR:bc not available"; fi; - - ## Get entropy_bit_count - if [[ -z "$1" ]]; then - ### prompt user - showUsage; - echo -n "Please specify the required strength of the password in bits of entropy (ex: 256):" 1>&2; # prompt via stderr - read -r entropy_bit_count - else - entropy_bit_count="$1"; - fi; # See [1]. - - ## Check entropy_bit_count - pattern="^[0-9]+$"; - if ! [[ $entropy_bit_count =~ $pattern ]] ; then die "ERROR:Not an integer:$entropy_bit_count"; fi; # See [2] - - ## Calculate minimum count of chars needed for passphrase as float using 'bc' - ### Solve ln(a^n)/ln(2)=b for n where: - ### a=$alphabet_size - ### n=$char_count_float - ### b=$entropy_bit_count - char_count_float="$(echo "$entropy_bit_count*l($log_base)/l($alphabet_size)" | bc -l)"; # See [4] - - ## Round $char_count_float to next highest integer - char_count="$(echo "$char_count_float" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')"; # See [3] - - ## Generate and output passphrase - echo "$(LC_ALL=C tr -cd "$charset" < /dev/urandom | head -c "$char_count")"; # See [6] -}; # main program - -( main "$@" ); # run 'main()' in subshell for 'die()' portability of script as sourced bash function. - -#==References== -# -# - How to echo a string as stderr instead of stdout. -# https://stackoverflow.com/a/2990533 -# Author: James Roth -# Date: 2010-06-07T14:52Z -# Date Accessed: 2020-01-20 -# -# - How to check if script argument exists or not. -# https://stackoverflow.com/a/6482403 -# Author: phoxix -# Date: 2011-06-26T05:55Z -# Date Accessed: 2020-01-20 -# -# - How to check that a string is an integer using regular expression test. -# https://stackoverflow.com/a/806923 -# Author: Charles Duffy -# Date: 2009-04-30T13:32Z -# Date Accessed: 2020-01-20 -# -# - How to use `bc` to calculate logarithms in Bash -# http://phodd.net/gnu-bc/bcfaq.html#bashlog -# Author: unknown -# Date Accessed: 2020-01-20 -# -# - How to use `awk` to convert and round up a float to an integer. -# https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html -# Author: Md. Minhazul Haque -# Date: 2015-01-09 -# Date Accessed: 2020-01-20 -# -# - How to use `/dev/urandom`, `tr`, and `head` to generate a random password in Bash. -# https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ -# Author: SASIKALA, Ramesh Natarajan -# Date: 2010-04-21 -# Date Accessed: 2020-01-20 -# -# - Bech32 base32 charset -# https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki -# Author: Pieter Wuille -# Date: 2017-03-20 -# License: BSD-2-Clause -# Date: Accessed: 2021-01-23 -# -# - Dependencies: bash, bc, echo, awk, tr, head -# -# - GNU bash, version 5.1.8(1)-release (x86_64-pc-linux-gnu) -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later -# This is free software; you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# - bc 1.07.1 -# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. -# -# - echo (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by Brian Fox and Chet Ramey. -# -# - GNU Awk 5.1.0, API: 3.0 (GNU MPFR 4.1.0, GNU MP 6.2.1) -# Copyright (C) 1989, 1991-2020 Free Software Foundation. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. -# -# - tr (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by Jim Meyering. -# -# - head (GNU coreutils) 8.32 -# Copyright (C) 2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later . -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by David MacKenzie and Jim Meyering. diff --git a/unitproc/bkt-get_path_fork_level b/unitproc/bkt-get_path_fork_level new file mode 100755 index 0000000..521b42d --- /dev/null +++ b/unitproc/bkt-get_path_fork_level @@ -0,0 +1,74 @@ +#!/bin/bash + +get_path_fork_level() { + # Desc: Get fork level from two paths + # Input: arg1 str path + # arg2 str path + # Output: stdout int fork level + # Version: 0.0.1 + local path1="$1"; + local path2="$2"; + + # Squeeze multiple slashes and remove trailing slashes + path1="$(echo "$path1" | tr -s '/' | sed 's:/*$::' )"; + path2="$(echo "$path2" | tr -s '/' | sed 's:/*$::' )"; + + # Check for mixed absolute/relative paths + if [[ "$path1" =~ ^/ ]] && [[ "$path2" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path1="$(echo "$path1" | sed -e 's:^/::' )"; + path2="$(echo "$path2" | sed -e 's:^/::' )"; + elif [[ ! "$path1" =~ ^/ ]] && [[ ! "$path2" =~ ^/ ]]; then + flag_root=false; + else + declare -p path1 path2 flag_root; + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; + return 1; + fi; + + # Save path as arrays with `/` as element delimiter + local IFS='/'; + read -ra parts1 <<< "$path1"; + read -ra parts2 <<< "$path2"; + + # Get fork level by counting identical path elements from rootside + local fork_level=0; + for (( i=0; i<${#parts1[@]} && i<${#parts2[@]}; i++ )); do + if [[ "${parts1[i]}" != "${parts2[i]}" ]]; then break; fi; + ((fork_level++)); + done; + + echo "$fork_level"; + #declare -p path1 path2 flag_root parts1 parts2 fork_level; # debug + return 0; +}; # Get fork level int from two paths + +printf "========Test 1========\n"; # fork at 0 +p1="foo"; p2="bee"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 2========\n"; # fork at 1 +p1="foo"; p2="foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 3========\n"; # fork at 1 +p1="foo"; p2="foo/bar/"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 4========\n"; # fork at 2 +p1="foo/bar/baz"; p2="foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 5========\n"; # fork at 2 +p1="/foo/bar/baz"; p2="/foo/bar"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 6========\n"; # ERROR: No mixed +p1="foo/bar/baz"; p2="/bee/boo/tax"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 7========\n"; # fork at 0 +p1="/foo/bar/baz"; p2="/bee/boo/tax"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 8========\n"; # # ERROR: No mixed +p1="/foo/bar/baz"; p2="foo"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; +printf "========Test 9========\n"; # fork at 3 +p1="foo///////////bar////////baz//////"; p2="foo/////////bar/////////baz///bee"; declare -p p1 p2; +get_path_fork_level "$p1" "$p2"; + diff --git a/unitproc/bkt-get_path_hierarchy_level b/unitproc/bkt-get_path_hierarchy_level new file mode 100755 index 0000000..7e03981 --- /dev/null +++ b/unitproc/bkt-get_path_hierarchy_level @@ -0,0 +1,77 @@ +#!/bin/bash +get_path_hierarchy_level() { + # Desc: Outputs hierarchy level of input paths + # Example: $ cat lines.txt | get_path_hierarchy_level + # Input: stdin str lines with /-delimited paths + # Output: stdout int hierarchy level of each path + # Version: 0.0.1 + + local line level; + local flag_root; + local -a output; + + n=0; + while read -r line; do + # Check for mixed absolute/relative paths. + if [[ $n -le 0 ]] && [[ "$line" =~ ^/ ]]; then + flag_root=true; + else + flag_root=false; + fi; + if { [[ "$flag_root" == "true" ]] && [[ ! "$line" =~ ^/ ]]; } || \ + { [[ "$flag_root" == "false" ]] && [[ "$line" =~ ^/ ]]; } then + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; return 1; + fi; + + # Squeeze multiple slashes and remove trailing slashes + line="$(echo "$line" | tr -s '/' | sed 's:/*$::' )"; + + # Count the number of slashes to determine hierarchy level + level="$(echo "$line" | awk -F'/' '{print NF-1}' )"; + if [[ "$flag_root" == "true" ]]; then ((level--)); fi; + + # Append to output + output+=("$level"); + #declare -p flag_root level; # debug + ((n++)); + done; + # Print output + printf "%s\n" "${output[@]}"; +}; # return hierarchy level of lines as integers + +# Test the function with the provided lines +printf "\n\n========Test 1========\n" +input_lines=( + "foo" + "foo/" + "foo/bar" + "foo/bar/" + "foo/bar///" + "foo///bar" + "/foo/bar/baz" + "/foo/bar/baz/" + "//foo/bar/baz////" +); + +printf "%s\n" "${input_lines[@]}"; +printf "========Test 1 results:========\n"; +# Convert array to newline-separated string and pass to function +printf "%s\n" "${input_lines[@]}" | get_path_hierarchy_level + +printf "\n\n========Test 2========\n" +input_lines=( + "foo" + "foo/" + "foo/bar" + "foo/bar/" + "foo/bar///" + "foo///bar" + "foo/bar/baz" + "foo/bar/baz/" + "foo/bar/baz////" +); + +printf "%s\n" "${input_lines[@]}"; +printf "========Test 2 results:========\n"; +# Convert array to newline-separated string and pass to function +printf "%s\n" "${input_lines[@]}" | get_path_hierarchy_level diff --git a/unitproc/bkt-prune_path_rootside b/unitproc/bkt-prune_path_rootside new file mode 100755 index 0000000..9736299 --- /dev/null +++ b/unitproc/bkt-prune_path_rootside @@ -0,0 +1,54 @@ +#!/bin/bash + +prune_path_rootside() { + # Desc: Prunes a path from the root-side to a specified prune level. + # Input: arg1 str path + # arg2 int prune level (0-indexed) + # Depends: GNU sed 4.8 + # Version: 0.0.1 + local path="$1"; + local prune_level="$2"; + + # Check for absolute or relative path + if [[ "$path" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path="$(echo "$path" | sed -e 's:^/::' )"; + else + flag_root=false; + fi; + + # Save path as array with `/` as element delimiter + local IFS='/'; + read -ra parts <<< "$path"; + + # Assemble pruned path from prune_level + local pruned_path=""; + for (( i=prune_level; i<${#parts[@]}; i++ )); do + pruned_path+="${parts[i]}/"; + done; + + # Trim trailing `/` delimiter + pruned_path=$(echo "$pruned_path" | sed 's:/*$::'); + + # Restore initial / if appropriate + if [[ "$flag_root" == "true" ]] && [[ "$prune_level" -eq 0 ]]; then + pruned_path=/"$pruned_path"; + fi; + + # Output pruned path + echo "$pruned_path"; + #declare -p path prune_level parts pruned_path && printf "========\n"; # debug + return 0; +}; # prune path rootside to int specified level + +printf "========Test 1========\n"; +prune_path_rootside "foo/bar/baz" 0; +prune_path_rootside "foo/bar/baz" 1; +prune_path_rootside "foo/bar/baz" 2; +prune_path_rootside "foo/bar/baz" 3; +printf "========Test 2========\n"; +prune_path_rootside "/foo/bar/baz" 0; +prune_path_rootside "/foo/bar/baz" 1; +prune_path_rootside "/foo/bar/baz" 2; +prune_path_rootside "/foo/bar/baz" 3; diff --git a/unitproc/bkt-remove_leading_zeroes b/unitproc/bkt-remove_leading_zeroes new file mode 100755 index 0000000..503dd40 --- /dev/null +++ b/unitproc/bkt-remove_leading_zeroes @@ -0,0 +1,17 @@ +#!/bin/bash + +remove_leading_zeroes() { + # Desc: Removes leading zeroes from lines + # Input: stdin + # Output: stdout + # Depends: BK-2020-03 read_stdin() + # Version: 0.0.2 + while read -r line; do + printf "%s\n" "$line" | sed -E -e 's/(^0*)([0-9].*)/\2/'; + done; +}; + +printf "00000.jpg\n0001.jpg\n2.jpg\n000003.jpg\n0010.jpg\n"; +printf "========================\n"; +printf "00000.jpg\n0001.jpg\n2.jpg\n000003.jpg\n0010.jpg\n" | remove_leading_zeroes; + diff --git a/user/bk-copy-rand-music b/user/bk-copy-rand-music index 7a7d299..597a663 100755 --- a/user/bk-copy-rand-music +++ b/user/bk-copy-rand-music @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Desc: Copies random audio files # Usage: bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION] ([int BYTES]) -# Version: 0.4.0 +# Version: 0.4.1 # Depends: BK-2020-03: bkshuf v0.1.0 declare -Ag appRollCall # Associative array for storing app status @@ -463,7 +463,7 @@ main() { path_candfile="$line"; # path of candidate file ### Check size - siz_cand="$(du -b "$path_candfile" | awk '{ print $1 }')"; # size in bytes + siz_cand="$(du -Lb "$path_candfile" | awk '{ print $1 }')"; # size in bytes if ! checkInt "$siz_cand"; then continue; fi; # reject if [[ "$((siz + siz_cand))" -gt "$siz_dest" ]]; then continue; fi; # reject if [[ "$siz_cand" -lt "$min_file_size" ]]; then continue; fi; # reject diff --git a/user/bkmml b/user/bkmml index c27ffc4..53981eb 100755 --- a/user/bkmml +++ b/user/bkmml @@ -4,7 +4,7 @@ # Input: arg1 find iname expression # arg2 grep -Ei expression # arg3 grep -Eiv expression -# Version: 0.0.1 +# Version: 0.0.3 # Depends: GNU Grep 3.7; GNU find utils 4.8.0; GNU Parallel 20210822 # Load env vars @@ -43,6 +43,8 @@ done < <(find -L "$(readlink -f "$BKMML_TARGET_DIR")" \ -o -iname "*.aac" \ -o -iname "*.opus" \ -o -iname "*.wav" \ + -o -iname "*.aif" \ + -o -iname "*.ape" \ -o -iname "*.ogg" \) \ -a -iname "*$FIND_SEARCH*" \ 2>/dev/random | \ @@ -54,6 +56,3 @@ done < <(find -L "$(readlink -f "$BKMML_TARGET_DIR")" \ # Author: Steven Baltakatei Sandoval # License: GPLv3+ - -# -# parallel b2sum '{}' | sort | uniq -w 128 | awk '{print $2}' diff --git a/user/bkmpv2 b/user/bkmpv2 index 19f605e..e4128f0 100755 --- a/user/bkmpv2 +++ b/user/bkmpv2 @@ -1,15 +1,17 @@ #!/usr/bin/env bash -# Desc: Wrapper for mpv that accepts directory paths via posargs or stdin lines +# Desc: Wrapper for mpv that accepts directory or file paths via posargs or stdin lines # Usage: bkmpv2 [DIR] -# Version: 0.0.1 +# Version: 0.2.0 # Depends: GNU Parallel, GNU Bash v5.1.16, mpv v0.34.1, bc v1.07.1 # Ref/Attrib: [1] Tange, Ole. GNU Parallel with Bash Array. 2019-03-24. https://unix.stackexchange.com/a/508365/411854 # Example: find $HOME/Music -type d | bkmpv2 # Example: bkmpv2 $HOME/Music/ +# Example: find $HOME -type f -name "*.mp3" | bkmpv2 # Note: Does not follow symlinks # Find settings -firegex=".+\(aac\|aif\|aiff\|flac\|m4a\|mp3\|mp4\|ogg\|opus\|wav\)$"; # update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u` +firegex=".+\(aac\|aif\|aiff\|flac\|m4a\|mp3\|mp4\|ogg\|opus\|wav\)$"; # POSIX regex for find. Update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u` +file_regex=".+(aac|aif|aiff|flac|m4a|mp3|mp4|ogg|opus|wav)$"; # extended regex for Bash. fsize="10k"; # default: minimum "10k" fdepth_posarg="10"; # find depth for positional arguments fdepth_stdin="1"; # find depth for stdin @@ -29,23 +31,23 @@ checkapp() { # Input: global assoc. array 'appRollCall' # Output: adds/updates key(value) to global assoc array 'appRollCall' # Depends: bash 5.0.3 - local returnState + local returnState #===Process Args=== for arg in "$@"; do - if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command - appRollCall[$arg]="true"; - if ! [ "$returnState" = "false" ]; then returnState="true"; fi; - else - appRollCall[$arg]="false"; returnState="false"; - fi; + if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command + appRollCall[$arg]="true"; + if ! [ "$returnState" = "false" ]; then returnState="true"; fi; + else + appRollCall[$arg]="false"; returnState="false"; + fi; done; #===Determine function return code=== if [ "$returnState" = "true" ]; then - return 0; + return 0; else - return 1; + return 1; fi; } # Check that app exists checkfile() { @@ -60,19 +62,19 @@ checkfile() { #===Process Args=== for arg in "$@"; do - if [ -f "$arg" ]; then - fileRollCall["$arg"]="true"; - if ! [ "$returnState" = "false" ]; then returnState="true"; fi; - else - fileRollCall["$arg"]="false"; returnState="false"; - fi; + if [ -f "$arg" ]; then + fileRollCall["$arg"]="true"; + if ! [ "$returnState" = "false" ]; then returnState="true"; fi; + else + fileRollCall["$arg"]="false"; returnState="false"; + fi; done; - + #===Determine function return code=== if [ "$returnState" = "true" ]; then - return 0; + return 0; else - return 1; + return 1; fi; } # Check that file exists checkdir() { @@ -87,19 +89,19 @@ checkdir() { #===Process Args=== for arg in "$@"; do - if [ -d "$arg" ]; then - dirRollCall["$arg"]="true"; - if ! [ "$returnState" = "false" ]; then returnState="true"; fi - else - dirRollCall["$arg"]="false"; returnState="false"; - fi + if [ -d "$arg" ]; then + dirRollCall["$arg"]="true"; + if ! [ "$returnState" = "false" ]; then returnState="true"; fi + else + dirRollCall["$arg"]="false"; returnState="false"; + fi done - + #===Determine function return code=== if [ "$returnState" = "true" ]; then - return 0; + return 0; else - return 1; + return 1; fi } # Check that dir exists displayMissing() { @@ -111,21 +113,21 @@ displayMissing() { # Depends: bash 5, checkAppFileDir() local missingApps value appMissing missingFiles fileMissing local missingDirs dirMissing - + #==BEGIN Display errors== #===BEGIN Display Missing Apps=== missingApps="Missing apps :"; #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done for key in "${!appRollCall[@]}"; do - value="${appRollCall[$key]}"; - if [ "$value" = "false" ]; then - #echo "DEBUG:Missing apps: $key => $value"; - missingApps="$missingApps""$key "; - appMissing="true"; - fi; + value="${appRollCall[$key]}"; + if [ "$value" = "false" ]; then + #echo "DEBUG:Missing apps: $key => $value"; + missingApps="$missingApps""$key "; + appMissing="true"; + fi; done; if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing. - echo "$missingApps" 1>&2; + echo "$missingApps" 1>&2; fi; unset value; #===END Display Missing Apps=== @@ -134,15 +136,15 @@ displayMissing() { missingFiles="Missing files:"; #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done for key in "${!fileRollCall[@]}"; do - value="${fileRollCall[$key]}"; - if [ "$value" = "false" ]; then - #echo "DEBUG:Missing files: $key => $value"; - missingFiles="$missingFiles""$key "; - fileMissing="true"; - fi; + value="${fileRollCall[$key]}"; + if [ "$value" = "false" ]; then + #echo "DEBUG:Missing files: $key => $value"; + missingFiles="$missingFiles""$key "; + fileMissing="true"; + fi; done; if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing. - echo "$missingFiles" 1>&2; + echo "$missingFiles" 1>&2; fi; unset value; #===END Display Missing Files=== @@ -151,15 +153,15 @@ displayMissing() { missingDirs="Missing dirs:"; #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done for key in "${!dirRollCall[@]}"; do - value="${dirRollCall[$key]}"; - if [ "$value" = "false" ]; then - #echo "DEBUG:Missing dirs: $key => $value"; - missingDirs="$missingDirs""$key "; - dirMissing="true"; - fi; + value="${dirRollCall[$key]}"; + if [ "$value" = "false" ]; then + #echo "DEBUG:Missing dirs: $key => $value"; + missingDirs="$missingDirs""$key "; + dirMissing="true"; + fi; done; if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing. - echo "$missingDirs" 1>&2; + echo "$missingDirs" 1>&2; fi; unset value; #===END Display Missing Directories=== @@ -185,21 +187,21 @@ checkInt() { #===Process Arg=== if [[ $# -ne 1 ]]; then - die "ERROR:Invalid number of arguments:$#"; + die "ERROR:Invalid number of arguments:$#"; fi; - + RETEST1='^[0-9]+$'; # Regular Expression to test if [[ ! $1 =~ $RETEST1 ]] ; then - returnState="false"; + returnState="false"; else - returnState="true"; + returnState="true"; fi; #===Determine function return code=== if [ "$returnState" = "true" ]; then - return 0; + return 0; else - return 1; + return 1; fi; } # Checks if arg is integer read_stdin() { @@ -214,8 +216,8 @@ read_stdin() { # Store stdin if [[ -p /dev/stdin ]]; then input_stdin="$(cat -)"; - fi; - + fi; + # Store as output array elements ## Read in stdin if [[ -n $input_stdin ]]; then @@ -235,12 +237,12 @@ read_psarg() { # Depends: GNU bash (version 5.1.16) # Version: 0.0.1 local input_psarg output; - + # Store arguments if [[ $# -gt 0 ]]; then input_psarg="$*"; fi; - + # Store as output array elements ## Read in positional arguments if [[ -n $input_psarg ]]; then @@ -261,6 +263,21 @@ find_flist() { if [[ ! -d "$1" ]]; then return 1; fi; must find "$1" -maxdepth "$fdepth" -type f -iregex "$firegex" -size +"$fsize"; }; # print file list to stdout from dir with script parameters +check_files() { + # Desc: Applies $file_regex to files specified by path + # Input: var: file_regex + # array: files_stdin + # Output: array: files_stdin + local file; + declare -a filtered_files_stdin; + + for file in "${files_stdin[@]}"; do + if [[ "$file" =~ $file_regex ]]; then + filtered_files_stdin+=("$file"); + fi; + done; + files_stdin=("${filtered_files_stdin[@]}"); +}; # apply $firegex to files_stdin array main() { # Input: var: firegex find iregex file name pattern # var: fsize find minimum file siz @@ -268,7 +285,7 @@ main() { # var: fc_redbase logarithm base to reduce output file count local re_dotfile; - declare -a dirs_stdin dirs_psarg; + declare -a files_stdin dirs_stdin dirs_psarg; declare -a paths_files; declare list_paths_files; declare -a cmd_args; @@ -280,45 +297,40 @@ main() { re_dotfile="^\."; # first char is a dot while read -r line; do line="$(readlink -e "$line")"; - # Check if dir - if [[ ! -d "$line" ]]; then - echo "ERROR:Not a dir:$line" 1>&2; + line_bn="$(basename "$line")"; + # Check if dir and not dotfile + if [[ -d "$line" ]] && [[ ! "$line_bn" =~ $re_dotfile ]]; then + dirs_stdin+=("$line"); continue; fi; - dir_name="$(basename "$line")"; - # Exclude dotdirs - if [[ "$dir_name" =~ $re_dotfile ]]; then - echo "ERROR:Is a dotdir:$line" 1>&2; - continue - fi; - dirs_stdin+=("$line"); - done < <(read_stdin); - yell "STATUS:$SECONDS:Read stdin."; - ## Read positional arguments as lines - re_dotfile="^\."; # first char is a dot - while read -r line; do - line="$(readlink -e "$line")"; - # Check if dir - if [[ ! -d "$line" ]]; then - echo "ERROR:Not a dir:$line" 1>&2; + + # Check if file + if [[ -f "$line" ]]; then + files_stdin+=("$line"); continue; fi; - dir_name="$(basename "$line")"; - # Exclude dotdirs - if [[ "$dir_name" =~ $re_dotfile ]]; then - echo "ERROR:Is a dotdir:$line" 1>&2; - continue - fi; - dirs_psarg+=("$line"); - done < <(read_psarg "$@"); - yell "STATUS:$SECONDS:Read posargs."; - + + # Throw warning + yell "WARNING:Not a valid dir or file:$line"; + done < <( read_stdin; read_psarg "$@"; ); + yell "STATUS:$SECONDS:Read stdin and psargs."; + + # Apply the $file_regex to $files_stdin array + check_files; + # Catch all arrays empty - if [[ "${#dirs_stdin[@]}" -le 0 ]] && [[ "${#dirs_psarg[@]}" -le 0 ]]; then - die "FATAL:No valid directories provided."; + if [[ "${#dirs_stdin[@]}" -le 0 ]] && \ + [[ "${#dirs_psarg[@]}" -le 0 ]] && \ + [[ "${#files_stdin[@]}" -le 0 ]]; then + die "FATAL:No valid directories or files provided."; fi; # Generate file list + ## Add stdin argument input + if [[ "${#files_stdin[@]}" -gt 0 ]]; then + paths_files+=("${files_stdin[@]}"); + fi; + ## Call find_filelist() in parallel for positional argument input if [[ "${#dirs_psarg[@]}" -gt 0 ]]; then fdepth="$fdepth_posarg"; export fdepth; # for dirs from positional arguments @@ -354,7 +366,7 @@ main() { bc_exp="$fc_falloff * (1 + l( $fc / $fc_falloff )/l($fc_redbase))"; fc_out="$( echo "$bc_exp" | bc -l )"; else - fc_out="$fc"; + fc_out="$fc"; fi; ## Reduce output file count by fixed fraction (bkshuf optimization) fc_out="$(echo "$fc_out * 0.75" | bc -l)"; @@ -365,7 +377,7 @@ main() { ### Round file count down fc_out="$(printf "%.0f" "$fc_out")"; # round float down to int ### Get neighbor-preserving shuffled subset (size $fc_out) - yell "STATUS:$SECONDS:Selecting $fc_out files via bkshuf..."; + yell "STATUS:$SECONDS:Selecting $fc_out of $fc files via bkshuf..."; must \ echo -n "$list_paths_files" | \ bkshuf "$fc_out" > "$list_paths_files_tmp"; diff --git a/user/bknpass b/user/bknpass index c1c1d1a..f4930a0 100755 --- a/user/bknpass +++ b/user/bknpass @@ -1,12 +1,13 @@ #!/bin/bash - -# Author: Steven Baltakatei Sandoval (baltakatei.com) -# -# License: This bash script, `bknpass`, is licensed under GPLv3 or -# later by Steven Baltakatei Sandoval: -# +# Desc: Generate passphrase with specified number of bits of entropy +# Usage: bknpass [integer] +# Example: bknpass 256 +# Result: 9tnzcl0m4dsm22a95525zj93jj +# Version: 0.2.1 +# Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 +# License: # `bknpass`, an alphanumeric password generator -# Copyright (C) 2021 Steven Baltakatei Sandoval (baltakatei.com) +# Copyright (C) 2022 Steven Baltakatei Sandoval (baltakatei.com) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,100 +21,77 @@ # # A copy of the GNU General Public License may be found at # . -# -# Description: This bash script generates alphanumeric passphrases -# with a char-count determined by a user-provided number of bits of -# entropy. The passphrase is then outputted to stdout with a trailing -# newline. It works as follows: -# -# - Prompt user for an integer. This integer is the number of bits -# of entropy that the generated password should have. -# -# - Check if user-provided string is an integer using `bash` regular -# expression test. -# -# - Calculate the minimum number of bech32 base32 characters -# required to encode the specified number of bits of entropy. -# -# - This step uses `bc` to calculate a logarithm float string -# and `awk` to convert the float into an integer, rounding up. -# -# - Use `tr`, `/dev/urandom`, and `head` to generate a random -# alphanumeric string with the length calculated in the previous -# step. -# -# - Use `echo` to display the passphrase in stdout with a trailing -# newline. -# -# Usage: bknpass [int] -# -# Example: bknpass 256 -# -# Dependencies: bash, echo, bc, awk, tr, head. See end of file -# -# Tested on: -# -# - GNU/Linux Debian 10 - - -#==Initialization== -let ALPHABET_SIZE="32" # number of unique chars in bech32 base32 charset -LOG_BASE=2 # Set logarithm base to 2 +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +showUsage() { + # Desc: Display script usage information + # Usage: showUsage + # Version 0.0.1 + # Input: none + # Output: stdout + # Depends: GNU-coreutils 8.30 (cat) + cat <<'EOF' + USAGE: + bknpass [int entropy_bit_count] -# Define `echoerr` function which outputs text to stderr - # Note: function copied from https://stackoverflow.com/a/2990533 -function echoerr { - echo "$@" 1>&2; -} + EXAMPLE: + bknpass 128 +EOF +}; # Display information on how to use this script. +main() { + # Desc: main program + # Usage: main "$@" + # Input: $1: integer bits of entropy + # Output: stdout: string passphrase + # Depends: bash 5.1.8, GNU coreutils 8.32, bc 1.07.1, awk 5.1.0 + # Ref/Attrib: [1] "Check existence of input argument in a Bash shell script". https://stackoverflow.com/a/6482403 + # [2] "How do I test if a variable is a number in Bash?". https://stackoverflow.com/a/806923 + # [3] "Round a Number in a Bash Script". https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html + # [4] "Logarithms in GNU bc". https://web.archive.org/web/20210211050732/http://phodd.net/gnu-bc/bcfaq.html#bclog + # [5] "BIP 173". Note: bech32 charset. https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki + # [6] "Bash Function to generate random password". https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ + charset="qpzry9x8gf2tvdw0s3jn54khce6mua7l"; # bech32. See [5]. + charset="$(tr -s "$charset" < <(echo -n "$charset"))"; # remove repeat chars + alphabet_size="$(echo -n "$charset" | wc -c)"; # number of unique chars + log_base=2; # entropy unit base (i.e. 2 for "bits of entropy") -# Define `rpass` function which generates a base32 passphrase of length $1 (ex: `rpass 20` generates a 20-char string) -# Note: function adapted from https://www.thegeekstuff.com/2010/04/unix-bash-function-examples/ -# Note: base32 charset uses bech32 charset -function rpass { - cat /dev/urandom | LC_ALL=C tr -cd "qpzry9x8gf2tvdw0s3jn54khce6mua7l" | head -c ${1:-22} -} + # Check inputs + if [[ $# -gt 1 ]]; then showUsage; die "ERROR:Too many arguments:$#"; fi; + # Check for bc + if ! command -v bc 1>/dev/null 2>&1; then die "ERROR:bc not available"; fi; -#==Main Program== + ## Get entropy_bit_count + if [[ -z "$1" ]]; then + ### prompt user + showUsage; + echo -n "Please specify the required strength of the password in bits of entropy (ex: 256):" 1>&2; # prompt via stderr + read -r entropy_bit_count + else + entropy_bit_count="$1"; + fi; # See [1]. -# Define $ENTROPY_BIT_COUNT1 as argument $1 or prompt user if $1 is not defined. -# Note: argument test adapted from https://stackoverflow.com/a/6482403 -if [ -z "$1" ] -then - echo "Entropy bit count argument (\$1) not supplied." - # Get from user the number of bits of entropy. - echoerr -n "Please specify the required strength of the password in bits of entropy (ex: 256):" # prompt via stderr - read ENTROPY_BIT_COUNT1 -else - ENTROPY_BIT_COUNT1="$1" -fi + ## Check entropy_bit_count + pattern="^[0-9]+$"; + if ! [[ $entropy_bit_count =~ $pattern ]] ; then die "ERROR:Not an integer:$entropy_bit_count"; fi; # See [2] -# Check if $ENTROPY_BIT_COUNT1 is an non-negative integer -# Note: Regular expression test is adapted from https://stackoverflow.com/a/806923 -RETEST1='^[0-9]+$' -if ! [[ $ENTROPY_BIT_COUNT1 =~ $RETEST1 ]] ; then - echo "error: Not an integer." >&2; exit 1 -fi + ## Calculate minimum count of chars needed for passphrase as float using 'bc' + ### Solve ln(a^n)/ln(2)=b for n where: + ### a=$alphabet_size + ### n=$char_count_float + ### b=$entropy_bit_count + char_count_float="$(echo "$entropy_bit_count*l($log_base)/l($alphabet_size)" | bc -l)"; # See [4] -# Calculate minimum count of chars needed to encode $ENTROPY_BIT_COUNT1 with alphabet size of $ALPHABET_SIZE as float -# Solve ln(a^n)/ln(2)=b for n using `bc` where -# a=$ALPHABET_SIZE -# n=$CHAR_COUNT1_FLOAT -# b=$ENTROPY_BIT_COUNT1 -# Note: `bc` logarithm usage adapted from http://phodd.net/gnu-bc/bcfaq.html#bashlog -CHAR_COUNT1_FLOAT=$(echo "$ENTROPY_BIT_COUNT1*l($LOG_BASE)/l($ALPHABET_SIZE)" | bc -l) -# Note: Float will be of form "21.49744370650136860806". -# Note: This particular example float should be rounded to "22" later. + ## Round $char_count_float to next highest integer + char_count="$(echo "$char_count_float" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')"; # See [3] -# Round $CHAR_COUNT1_FLOAT1 up to next highest integer for use as argument in later bash functions. -# Note: awk expression from https://bits.mdminhazulhaque.io/linux/round-number-in-bash-script.html -CHAR_COUNT1=$(echo "$CHAR_COUNT1_FLOAT" | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}') - -# Generate passphrase -PASS1=$(rpass "$CHAR_COUNT1") -echo -e "$PASS1" + ## Generate and output passphrase + echo "$(LC_ALL=C tr -cd "$charset" < /dev/urandom | head -c "$char_count")"; # See [6] +}; # main program +( main "$@" ); # run 'main()' in subshell for 'die()' portability of script as sourced bash function. #==References== # @@ -159,53 +137,53 @@ echo -e "$PASS1" # License: BSD-2-Clause # Date: Accessed: 2021-01-23 # -# - Dependencies: bash, echo, bc, awk, tr, head. +# - Dependencies: bash, bc, echo, awk, tr, head # -# - GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) -# Copyright (C) 2019 Free Software Foundation, Inc. +# - GNU bash, version 5.1.8(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later # This is free software; you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# -# - echo (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# +# - bc 1.07.1 +# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. +# +# - echo (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. # # Written by Brian Fox and Chet Ramey. -# -# - bc 1.07.1 -# Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006, 2008, 2012-2017 Free Software Foundation, Inc. -# -# - GNU Awk 4.2.1, API: 2.0 (GNU MPFR 4.0.2, GNU MP 6.1.2) -# Copyright (C) 1989, 1991-2018 Free Software Foundation. -# +# +# - GNU Awk 5.1.0, API: 3.0 (GNU MPFR 4.1.0, GNU MP 6.2.1) +# Copyright (C) 1989, 1991-2020 Free Software Foundation. +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. # -# - tr (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# - tr (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# +# # Written by Jim Meyering. -# -# - head (GNU coreutils) 8.30 -# Copyright (C) 2018 Free Software Foundation, Inc. +# +# - head (GNU coreutils) 8.32 +# Copyright (C) 2020 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later . # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. -# +# # Written by David MacKenzie and Jim Meyering. diff --git a/user/bkots b/user/bkots index baffb08..3c9705c 100755 --- a/user/bkots +++ b/user/bkots @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Define variables +declare -g max_job_count="2"; # default max job count +declare -g age_threshold="60"; # min age to add file; seconds; declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status @@ -10,13 +12,11 @@ calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); -declare -a commands # array for storing assembled commands -age_threshold="60"; # min age to add file; seconds; # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status -try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails checkapp() { # Desc: If arg is a command, save result in assoc array 'appRollCall' # Usage: checkapp arg1 arg2 arg3 ... @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 1.0.3 +bkots 2.1.1 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -239,6 +239,8 @@ showUsage() { --include-dotfiles Include files and directories starting with '.' (not included by default). + -j, --jobs + Specify simultaneous job count (default: 2) -r, --recursive Consider files in dirs recursively. --version @@ -268,6 +270,32 @@ showUsage() { bkots foo.txt bar.pdf /home/username/Pictures/ EOF } # Display information on how to use this script. +count_jobs() { + # Desc: Count and return total number of jobs + # Usage: count_jobs + # Input: None. + # Output: stdout integer number of jobs + # Depends: Bash 5.1.16 + # Example: while [[$(count_jobs) -gt 0]]; do echo "Working..."; sleep 1; done; + # Version: 0.0.1 + + local job_count; + job_count="$(jobs -r | wc -l | tr -d ' ' )"; + #yell "DEBUG:job_count:$job_count"; + if [[ -z $job_count ]]; then job_count="0"; fi; + echo "$job_count"; +}; # Return number of background jobs +wait_for_jobslot() { + # Desc: Does not return until count_jobs() falls below $max_job_count + # Input: var max_job_count + # Output: return code 0 + # Depends: count_jobs(), yell(); + while [[ $(count_jobs) -ge $max_job_count ]]; do + printf "\r%d:Working..." "$SECONDS"; + sleep 1; + done; + return 0; +}; processArgs() { # Desc: Processes arguments provided to script. # Usage: processArgs "$@" @@ -302,6 +330,15 @@ processArgs() { --include-dotfiles) # Include dotfiles option_include_dotfiles="true"; vbm "DEBUG:Option enabled:include dotfiles";; + -j | --jobs) # Specify max_job_count + if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then + max_job_count="$2"; + vbm "STATUS:Max job count set to:$max_job_count"; + shift; + else + showUsage; + die "FATAL:Invalid job count:$2"; + fi;; -r | --recursive) # Specify recursive option option_recursive="true"; vbm "DEBUG:option enabled:include files in dirs recursively";; @@ -336,7 +373,7 @@ get_parent_dirnames() { # Input: arg1 input path # Output: stdout newline-delimited list of parent dirs # Version: 0.0.1 - # Depends: yell(), die(), try() + # Depends: yell(), die(), must() local path # Check input @@ -354,14 +391,6 @@ get_parent_dirnames() { echo "$name_base"; done; }; # Output parent dirnames to stdout -cmdwrap() { - # print command to stderr - echo "$@" 1>&2; - - # execute command - "$@"; -}; # print and execute string together -export -f cmdwrap; # export cmdwrap for use in other functions main() { # Desc: Creates `.ots` file: # - for each file specified in arrayPosArgs array @@ -377,7 +406,6 @@ main() { # Ref/Attrib: [1] How to create an array of unique elements from a string/array in bash https://unix.stackexchange.com/a/167194 # [2] How to find files containing newlines in their names https://stackoverflow.com/a/21727028 # [3] Get mtime of specific file using Bash? https://stackoverflow.com/a/4774377 - # [4] Search/Replace with string substitution instead of sed. https://www.shellcheck.net/wiki/SC2001 local -a file_list file_list_pruned; local -a files_to_verify files_to_upgrade files_to_stamp local -a files_to_verify_pruned files_to_upgrade_pruned files_to_stamp_pruned @@ -638,86 +666,108 @@ main() { vbm "DEBUG:files_to_stamp_pruned:"; printf "%s\n" "${files_to_stamp_pruned[@]}"; fi; - + # Act on files - ## Assemble upgrade file commands + ## Assemble and execute upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do - path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. - if [[ -z "$path_prf" ]]; then - yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try upgrade with known calendars in random order - while read -r url; do - vbm "DEBUG:Upgrading with calendar:url:$url"; - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; - else - commands+=("cmdwrap ots -l $url --no-default-whitelist upgrade \"$path_prf_sesc\"") && break; - fi; - #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; - fi; + wait_for_jobslot && { + path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_prf" ]]; then + yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; + return 1; # would have been `continue` were it not in a subshell + fi; + vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try upgrade with known calendars in random order + while read -r url; do + vbm "DEBUG:Upgrading with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("upgrade" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; + fi; + } & done; - ## Assemble verify file commands + ## Assemble and execute verify file commands for item in "${files_to_verify_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. - path_prf_sesc="${path_prf//\"/\\\"}"; # escape path double quotes. See [4]. - if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then - yell "ERROR:blank verify item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; - vbm "DEBUG: against proof file: path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try verify with known calendars in random order - while read -r url; do - vbm "DEBUG:Verifying with calendar:url:$url"; - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; - else - commands+=("cmdwrap ots -l $url --no-default-whitelist verify -f \"$path_src_sesc\" \"$path_prf_sesc\"") && break; - fi; - #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; - fi; + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then + yell "ERROR:blank verify item encountered. Skipping:item:$item"; + return 1; # would have been `continue` were it not in a subshell + fi; + vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; + vbm "DEBUG: against proof file: path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try verify with known calendars in random order + while read -r url; do + vbm "DEBUG:Verifying with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; + fi; + } & done; - - ## Assemble stamp file commands + + ## Assemble and execute stamp file commands for item in "${files_to_stamp_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_src_sesc="${path_src//\"/\\\"}"; # escape path double quotes. See [4]. - if [[ -z "$path_src" ]]; then - yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; - if [[ ! $option_dry_run == "true" ]]; then - if [[ "$opVerbose" = "true" ]]; then - commands+=("cmdwrap ots -v stamp \"$path_src_sesc\""); + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_src" ]]; then + yell "ERROR:blank stamp item encountered. Skipping:item:$item"; + return 1; # would have been `continue` were it not in a subshell + fi; + vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; + if [[ ! $option_dry_run == "true" ]]; then + + #### assemble command + local -a cmd_temp; + cmd_temp=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("stamp" "$path_src"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + #ots stamp "$path_src"; else - commands+=("cmdwrap ots stamp \"$path_src_sesc\""); + yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; fi; - #ots stamp "$path_src"; - else - yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; - fi; + } & done; - ## Run commands - #yell "DEBUG:commands:$(printf "%s\n" "${commands[@]}")"; - printf "%s\n" "${commands[@]}" | parallel --group --jobs 25%; - + ## Wait for jobs to finish. + wait; }; # main program # Run program diff --git a/user/bkytpldl-generic b/user/bkytpldl-generic new file mode 100644 index 0000000..ae79832 --- /dev/null +++ b/user/bkytpldl-generic @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Desc: Download YouTube videos +# Usage: $ ./bkytpldl-generic +# Version: 4.1.1 + +declare -a args; # array for yt-dlp arguments +declare -a urls urls_rand; # array for YouTube playlist URLs + +# Settings +dir_out="~/Videos/"; +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here +urls+=("https://www.youtube.com/playlist?list=PLxxx"); # Adjust me. YouTube playlist URL goes here + +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails + +# check dependencies +if ! command -v yt-dlp 1>/dev/random 2>&1; then die "FATAL:yt-dlp not found."; fi; + +# Donʼt run multiple yt-dlp instances +if pgrep "^yt-dlp$" 1>/dev/random 2>&1; then die "FATAL:yt-dlp already running."; fi; + +# Check directories +if [[ ! -d $dir_out ]]; then mkdir -p "$dir_out"; fi; + +# == Assemble options == + +# yt-dlp output options +## Restrict file name character set +#args+=("--restrict-filenames"); # Remove non-ASCII characters +args+=("--trim-filenames=120"); # Use in tandem with `%(title).120B` + +## Request to write accompanying files +args+=("--write-subs"); # Write subtitles file +args+=("--write-auto-subs"); # Write subtitles file +#args+=("--all-subs"); # Download all available subtitles (causes many requests) +#subLangs="en.*,ja.*,id.*,es.*,zh-Hans.*,zh-Hant.*,sv.*,el.*,hi.*,ru.*,bn.*,fr.*,ko.*,ar.*,nv.*"; # custom language list +subLangs="en,en-orig,en.*"; # custom language list +args+=("--sub-langs" "$subLangs"); +args+=("--write-info-json"); # Write accompanying json file +args+=("--no-overwrites"); # Don't overwrite files +args+=("--write-thumbnail"); # Write thumbnail + +## Only download metadata +#args+=("--no-download"); # Don't download video file. + +## Save meta-data +args+=("--write-comments"); # Get comments +### Limit comments +### comment_sort values: +### top : use YouTube top comment algorithm +### new : get newest comments (default) +### max_comments values: +### max-comments : max number of parent comments or replies +### max-parents : max number of comment threads +### max-replies : max number of replies across all threads +### max-replies-per-thread : max number of replies per thread +args+=("--extractor-args" "youtube:comment_sort=top;max_comments=10000,100,10000,100"); + +## Randomize order in which playlist items are downloaded +args+=("--playlist-random"); + +## Delay between downloads +minSleep="30"; +maxSleep="$(( minSleep + (RANDOM + RANDOM + RANDOM) / ( 3 * 400) ))"; # roughly 60 seconds +args+=("--min-sleep-interval" "$minSleep"); +args+=("--max-sleep-interval" "$maxSleep"); +args+=("--sleep-requests" "2"); # delay on metadata requests +args+=("--sleep-subtitles" "10"); # delay for subtitles + +## Remember downloaded videos to avoid redownload attempts +pathDA="$dir_out"/.bkytpldl_history.txt; +args+=("--download-archive" "$pathDA"); + +## Use firefox 'default-release' profile cookies +## Example: Linux: from ~/.mozilla/firefox/deadbeef.default-release/ +#args+=("--cookies-from-browser"); +#args+=("firefox:deadbeef.default-release"); Default Firefox profile name + +## Specify output filename format +## Note: `$(title).120B` shortens title to 120 bytes (useful for +## titles with UTF-8 characters. +args+=("-o"); +args+=("%(playlist)s/%(upload_date)s.%(channel)s.%(channel_id)s.%(title).120B.%(id)s.%(ext)s"); + +## Limit download resolution to 1080p +args+=("-S" "res:1080"); + +## Specify playlist URLs to download +### Shuffle playlist download order +mapfile -t urls_rand < <(printf "%s\n" "${urls[@]}" | shuf); +for url in "${urls_rand[@]}"; do + args+=("$url"); +done; + +# Change working directory to output dir +pushd "$dir_out" || die "FATAL:Failed to change pwd to:dir_out:$dir_out"; + +# == Download videos == + +#yell "DEBUG:args:$(declare -p args)"; # debug command +must yt-dlp "${args[@]}"; # execute command +popd || die "FATAL:Failed to return from dir_out:$dir_out"; + +# Author: Steven Baltakatei Sandoval +# License; GPLv3+ diff --git a/user/check_order.sh b/user/check_order.sh new file mode 100755 index 0000000..bce5066 --- /dev/null +++ b/user/check_order.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Desc: Checks stdin for lines out of order. +# Input: stdin +# Output: stdout +# Version: 0.0.1 + +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +read_stdin() { + # Desc: Consumes stdin; outputs as stdout lines + # Input: stdin (consumes) + # Output: stdout (newline delimited) + # return 0 stdin read + # return 1 stdin not present + # Example: printf "foo\nbar\n" | read_stdin + # Depends: GNU bash (version 5.1.16), GNU Coreutils 8.32 (cat) + # Version: 0.1.1 + # Attrib: Steven Baltakatei Sandoval (2024-01-29). reboil.com + local input_stdin output; + + # Store stdin + if [[ -p /dev/stdin ]]; then + input_stdin="$(cat -)" || { + echo "FATAL:Error reading stdin." 1>&2; return 1; }; + else + return 1; + fi; + + # Store as output array elements + ## Read in stdin + if [[ -n $input_stdin ]]; then + while read -r line; do + output+=("$line"); + done < <(printf "%s\n" "$input_stdin") || { + echo "FATAL:Error parsing stdin."; return 1; }; + fi; + + # Print to stdout + printf "%s\n" "${output[@]}"; + + return 0; +}; # read stdin to stdout lines +main() { + previous=""; + n=1; + while IFS= read -r line; do + current="$line"; + if [[ -n "$previous" ]] && [[ "$previous" > "$current" ]]; then + yell "STATUS:Line $n is out of order: $line"; + fi; + previous="$current"; + ((n++)); + done < <(read_stdin || die "FATAL:No stdin."); +}; # main program +export -f read_stdin; + +main "$@"; + +# Author: Steven Baltakatei Sandoval +# License: GPLv3+ diff --git a/user/elisp/remove-single-newlines.el b/user/elisp/remove-single-newlines.el new file mode 100644 index 0000000..22da713 --- /dev/null +++ b/user/elisp/remove-single-newlines.el @@ -0,0 +1,24 @@ +(defun remove-single-newlines () + ;; Desc: Replace all single newlines with spaces. + ;; Date: 2024-02-24 + "Remove single newlines, joining lines that are separated by a single newline, while preserving paragraphs separated by multiple newlines." + (interactive) + (let ((start (point-min)) (end (point-max))) + (save-excursion + (goto-char start) + (while (re-search-forward "\\([^\n]\\)\n\\([^\n]\\)" end t) + ;; Check if the newline is followed by another newline (indicating a paragraph break) + (unless (looking-at-p "\n") + (replace-match "\\1 \\2" nil nil)))))) + +(defun remove-single-newlines-region (start end) + ;; Desc: Replace all single newlines in a selected region with spaces. + ;; Date: 2024-02-24 + "Remove single newlines in the selected region, joining lines separated by a single newline, while preserving paragraphs separated by multiple newlines." + (interactive "r") ; "r" means this function should use the region start and end as arguments when called interactively + (save-excursion + (goto-char start) + (while (re-search-forward "\\([^\n]\\)\n\\([^\n]\\)" end t) + ;; Check if the newline is followed by another newline (indicating a paragraph break) + (unless (looking-at-p "\n") + (replace-match "\\1 \\2" nil nil))))) diff --git a/user/mp3s_to_mkv.sh b/user/mp3s_to_mkv.sh index 144e34b..b788edb 100755 --- a/user/mp3s_to_mkv.sh +++ b/user/mp3s_to_mkv.sh @@ -1,9 +1,10 @@ #!/bin/bash -# Desc: Converts a directory of mp3s into a single mkv file +# Desc: Converts a directory of mp3s into a single mkv file with chapters # Usage: mp3s_to_mkv.sh [DIR in] [DIR out] [BITRATE] # Example: mp3s_to_mkv.sh ./dir_source ./dir_output 48k -# Depends: GNU Coretils 8.32 (date) -# Version: 0.0.2 +# Depends: GNU Coreutils 8.32 (date), ffmpeg, ffprobe +# Ref/Attrib: [1] FFmpeg Formats Documentation https://ffmpeg.org/ffmpeg-formats.html#toc-Metadata-2 +# Version: 0.1.2 # plumbing opus_bitrate="$3"; # e.g. "48k" @@ -14,6 +15,7 @@ dir_out="$(readlink -f "$2")"; file_flist="$dir_tmp"/"$script_rundate"..flist.txt; file_out_mkv="$dir_out"/output.mkv; file_albumart="$dir_out"/albumart.png; +file_chapters="$dir_tmp"/"$script_rundate"..chapters.txt; yell() { echo "$0: $*" >&2; } # print script path and all args to stderr die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status @@ -35,43 +37,89 @@ EOF } # Display information on how to use this script. check_depends() { if ! command -v ffmpeg; then show_usage; die "FATAL:Missing ffmpeg."; fi; + if ! command -v ffprobe; then show_usage; die "FATAL:Missing ffprobe."; fi; }; # check dependencies check_plumbing() { if [[ $# -ne 3 ]]; then show_usage; die "FATAL:Invalid arg count:$#"; fi; if [[ ! -d "$dir_in" ]]; then show_usage; die "FATAL:Not a dir:$dir_in"; fi; if [[ ! -d "$dir_out" ]]; then mkdir -p "$2"; fi; }; # check arguments -build_filelist() { +get_media_length() { + # Use ffprobe to get media container length in seconds (float) + # Usage: get_media_length arg1 + # Input: arg1: path to file + # Output: stdout: seconds (float) + # Depends: ffprobe 4.1.8 + # BK-2020-03: die() + local file_in; + file_in="$1"; + if [[ ! -f $file_in ]]; then + die "ERROR:Not a file:$file_in"; + fi; + ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$file_in"; +} # Get media container length in seconds via stdout +build_filelist_and_chapters() { # Depends: var dir_tmp temporary directory # var dir_in input dir # var file_flist path file list - # Output: file: $file_flist list of mp3 files for ffmpeg + # Output: file: $file_flist list of mp3 files for ffmpeg + # file: $file_chapters chapters file for ffmpeg # Change directory to input dir pushd "$dir_in" || die "FATAL:Directory error:$(pwd)"; + + local chapter_start=0; + local chapter_end; + local duration; + + + # Initialize chapter ffmetadata file. See [1]. + { + echo ";FFMETADATA1"; + } >> "$file_chapters"; + - while read -r line; do - yell "$(printf "file '%s'\n" "${line#./}")"; - printf "file '%s'\n" "${line#./}" >> "$file_flist"; - done < <(find "$dir_in" -type f -iname "*.mp3" | sort); + find "$dir_in" -type f -iname "*.mp3" | sort | while read -r line; do + local filename="${line#./}"; + yell "$(printf "file '%s'\n" "$filename")"; + printf "file '%s'\n" "$filename" >> "$file_flist"; + + # Get duration of the current file + duration=$(get_media_length "$filename"); + chapter_end=$(echo "$chapter_start + $duration" | bc -l); + + # Write chapter info + { + echo "[CHAPTER]"; + echo "TIMEBASE=1/1000"; + echo "START=$(echo "scale=0; $chapter_start * 1000" | bc -l)"; + echo "END=$(echo "scale=0; $chapter_end * 1000" | bc -l)"; + echo "title=$(basename "$filename" .mp3)"; + } >> "$file_chapters"; + + chapter_start=$chapter_end; + done # Return to original dir popd || die "FATAL:Directory error:$(pwd)"; - -}; # build file list for ffmpeg +}; # build file list and chapters for ffmpeg ffmpeg_convert() { # Depends: var dir_tmp # dir_in # dir_out - # Input: file $file_flist list of mp3 files for ffmpeg + # Input: file $file_flist list of mp3 files for ffmpeg + # file $file_chapters chapters file for ffmpeg # Change directory to input dir pushd "$dir_in" || die "FATAL:Directory error:$(pwd)"; # Concatenate mp3 files into a single WAV file - # # Convert WAV to 48 kbps opus mkv file + # Convert WAV to opus mkv file ffmpeg -nostdin -f concat -safe 0 -i "$file_flist" -c:a pcm_s24le -rf64 auto -f wav - | \ - ffmpeg -i - -c:a libopus -b:a "$opus_bitrate" "$file_out_mkv"; + ffmpeg -i - -i "$file_chapters" \ + -map_metadata 1 -map_chapters 1 \ + -map 0 -c:a libopus -b:a "$opus_bitrate" \ + "$file_out_mkv"; # Return to original dir popd || die "FATAL:Directory error:$(pwd)"; @@ -85,10 +133,12 @@ save_albumart() { main() { check_depends && yell "DEBUG:check_depends OK"; check_plumbing "$@" && yell "DEBUG:check_plumbing OK"; - build_filelist "$@" && yell "DEBUG:build_filelist OK"; + build_filelist_and_chapters "$@" && yell "DEBUG:build_filelist_and_chapters OK"; ffmpeg_convert "$@" && yell "DEBUG:ffmpeg_convert OK"; save_albumart "$@" && yell "DEBUG:save_albumart OK"; }; # main program main "$@"; +# Author: Steven Baltakatei Sandoval +# License: GPLv3+ diff --git a/user/mp4_make480.sh b/user/mp4_make480.sh new file mode 100644 index 0000000..8799a2e --- /dev/null +++ b/user/mp4_make480.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Desc: Reëncodes .mp4 videos in a specified directory. +# Usage: mp4_make480.sh [dir in] [dir out] +# Input: arg1 dir input dir +# arg2 dir output dir (optional) +# Output: dir ./out/ +# file ./out/[files]_480.mp4 +# Version: 0.1.2 +# Depends: GNU parallel 20161222, ffmpeg 4.3.6-0 +# Ref/Attrb: [1]: FFmpeg wiki: https://trac.ffmpeg.org/wiki/Scaling +declare -g dir_in dir_out; + +check_input() { + dir_in="$1"; + if [[ $# -le 0 ]]; then echo "FATAL:Insufficient args:$#" 1>&2; exit 1; fi; + if [[ $# -eq 2 ]]; then + dir_out="$2"; + else + dir_out="./out_480"; + fi; + export dir_out; + if [[ ! -d "$dir_in" ]]; then echo "FATAL:Not a dir:$dir_in" 1>&2; exit 1; fi; + mkdir -p "$dir_out"; + declare -p dir_in dir_out; # debug +}; +convert_video() { + find "$dir_in" -mindepth 1 -maxdepth 1 -type f -iname "*.mp4" | \ + parallel job_ffmpeg "{}" "$path_out"; +}; +job_ffmpeg() { + path_in="$1"; + path_out="$2"; + file_in="$(basename "$path_in")"; + path_out=./"$dir_out"/"${file_in%.mp4}"_480.mp4; + opt_scale="scale=-2:480"; # See [1] + declare -p path_in path_out file_in dir_out path_out opt_scale; + ffmpeg -nostdin -i "$path_in" -vf "$opt_scale" "$path_out"; +}; +export -f job_ffmpeg; + +main() { + check_input "$@"; + convert_video; +}; + +main "$@"; + +# Author: Steven Baltakatei Sandoval +# License: GPLv3+ diff --git a/user/mw_create_subpage_navlinks.sh b/user/mw_create_subpage_navlinks.sh index de37365..5cdf74a 100755 --- a/user/mw_create_subpage_navlinks.sh +++ b/user/mw_create_subpage_navlinks.sh @@ -3,8 +3,8 @@ # Input: file text file with list of chapters # stdin text with list of chapters # Output: [[../Chapter 4|Next]], [[../Chapter 2|Previous]], [[../|Up]] -# Version: 0.0.1 -# Attrib: Steven Baltakatei Sandoval. (2024-01-29). reboil.com +# Version: 0.1.0 +# Attrib: Steven Baltakatei Sandoval. (2024-07-17). reboil.com # License: GPLv3+ yell() { echo "$0: $*" >&2; } # print script path and all args to stderr @@ -63,6 +63,129 @@ read_stdin() { return 0; }; # read stdin to stdout lines +get_path_fork_level() { + # Desc: Get fork level from two paths + # Input: arg1 str path + # arg2 str path + # Output: stdout int fork level + # Version: 0.0.1 + local path1="$1"; + local path2="$2"; + + # Squeeze multiple slashes and remove trailing slashes + path1="$(echo "$path1" | tr -s '/' | sed 's:/*$::' )"; + path2="$(echo "$path2" | tr -s '/' | sed 's:/*$::' )"; + + # Check for mixed absolute/relative paths + if [[ "$path1" =~ ^/ ]] && [[ "$path2" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path1="$(echo "$path1" | sed -e 's:^/::' )"; + path2="$(echo "$path2" | sed -e 's:^/::' )"; + elif [[ ! "$path1" =~ ^/ ]] && [[ ! "$path2" =~ ^/ ]]; then + flag_root=false; + else + declare -p path1 path2 flag_root; + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; + return 1; + fi; + + # Save path as arrays with `/` as element delimiter + local IFS='/'; + read -ra parts1 <<< "$path1"; + read -ra parts2 <<< "$path2"; + + # Get fork level by counting identical path elements from rootside + local fork_level=0; + for (( i=0; i<${#parts1[@]} && i<${#parts2[@]}; i++ )); do + if [[ "${parts1[i]}" != "${parts2[i]}" ]]; then break; fi; + ((fork_level++)); + done; + + echo "$fork_level"; + #declare -p path1 path2 flag_root parts1 parts2 fork_level; # debug + return 0; +}; # Get fork level int from two paths +prune_path_rootside() { + # Desc: Prunes a path from the root-side to a specified prune level. + # Input: arg1 str path + # arg2 int prune level (0-indexed) + # Depends: GNU sed 4.8 + # Version: 0.0.1 + local path="$1"; + local prune_level="$2"; + + # Check for absolute or relative path + if [[ "$path" =~ ^/ ]]; then + flag_root=true; + # Remove initial / + path="$(echo "$path" | sed -e 's:^/::' )"; + else + flag_root=false; + fi; + + # Save path as array with `/` as element delimiter + local IFS='/'; + read -ra parts <<< "$path"; + + # Assemble pruned path from prune_level + local pruned_path=""; + for (( i=prune_level; i<${#parts[@]}; i++ )); do + pruned_path+="${parts[i]}/"; + done; + + # Trim trailing `/` delimiter + pruned_path=$(echo "$pruned_path" | sed 's:/*$::'); + + # Restore initial / if appropriate + if [[ "$flag_root" == "true" ]] && [[ "$prune_level" -eq 0 ]]; then + pruned_path=/"$pruned_path"; + fi; + + # Output pruned path + echo "$pruned_path"; + #declare -p path prune_level parts pruned_path && printf "========\n"; # debug + return 0; +}; # prune path rootside to int specified level +get_path_hierarchy_level() { + # Desc: Outputs hierarchy level of input paths + # Example: $ cat lines.txt | get_path_hierarchy_level + # Input: stdin str lines with /-delimited paths + # Output: stdout int hierarchy level of each path + # Version: 0.0.1 + + local line level; + local flag_root; + local -a output; + + n=0; + while read -r line; do + # Check for mixed absolute/relative paths. + if [[ $n -le 0 ]] && [[ "$line" =~ ^/ ]]; then + flag_root=true; + else + flag_root=false; + fi; + if { [[ "$flag_root" == "true" ]] && [[ ! "$line" =~ ^/ ]]; } || \ + { [[ "$flag_root" == "false" ]] && [[ "$line" =~ ^/ ]]; } then + echo "FATAL:Mixed relative and absolute paths not supported." 1>&2; return 1; + fi; + + # Squeeze multiple slashes and remove trailing slashes + line="$(echo "$line" | tr -s '/' | sed 's:/*$::' )"; + + # Count the number of slashes to determine hierarchy level + level="$(echo "$line" | awk -F'/' '{print NF-1}' )"; + if [[ "$flag_root" == "true" ]]; then ((level--)); fi; + + # Append to output + output+=("$level"); + #declare -p flag_root level; # debug + ((n++)); + done; + # Print output + printf "%s\n" "${output[@]}"; +}; # return hierarchy level of lines as integers validate_subpage_list() { # Desc: Check for illegal characters in subpage titles # Input: stdin unvalidated subpage list @@ -97,18 +220,27 @@ validate_subpage_list() { echo "FATAL:Error reading stdin." 1>&2; return 1; }; }; generate_wikicode() { + # Desc: Generates navigational link wikicode for subpages # Input: stdin validated subpage list # Output: stdout wikicode - local lprev lnext; + # Depends: get_path_fork_level() + # prune_path_rootside() + # get_path_hierarchy_level() n=0; while read -r line; do #yell "$n:Processing line:$line"; # debug + # Advance input lines lprev="$lcurr"; lcurr="$lnext"; lnext="$line"; #declare -p lprev lcurr lnext; # debug + # Update hierarchy tracker states + lprev_hier="$lcurr_hier"; + lcurr_hier="$lnext_hier"; + lnext_hier="$(echo "$lnext" | get_path_hierarchy_level)"; + # Skip first iteration if [[ "$n" -eq 0 ]]; then # yell "$n:DEBUG:Skipping first iteration."; # debug @@ -116,30 +248,42 @@ generate_wikicode() { #printf -- "----\n" 1>&2; # debug continue; fi; - # Handle first valid input set - # yell "$n:DEBUG:Handling first valid input set."; # debug - if [[ "$n" -eq 1 ]]; then - printf "[[../%s|Next]], [[../|Up]]\n" \ - "$lnext"; - #printf -- "----\n" 1>&2; # debug - ((n++)); continue; fi; - - # Handle middle lines - # yell "$n:DEBUG:Handling middle lines."; # debug - printf "[[../%s|Next]], [[../%s|Previous]], [[../|Up]]\n" \ - "$lnext" "$lprev"; - ((n++)); - #printf -- "----\n" 1>&2; # debug - done; + # Get path fork levels + fork_level_next="$(get_path_fork_level "$lcurr" "$lnext")"; + fork_level_prev="$(get_path_fork_level "$lcurr" "$lprev")"; - # Handle last line - lprev="$lcurr"; - lcurr="$lnext"; - lnext="$line"; - printf "[[../%s|Previous]], [[../|Up]]\n" \ - "$lprev"; - ((n++)); -}; + # Count relative ups needed (`../`) + relups_next="$((lcurr_hier - fork_level_next + 1))"; + relups_prev="$((lcurr_hier - fork_level_prev + 1))"; + + # Initialize Next and Prev links with relative ups to fork. + link_next=""; + for (( i=0; i&2; + fi; + + #declare -p n lprev lcurr lnext lprev_hier lcurr_hier lnext_hier; # debug + #declare -p fork_level_next fork_level_prev relups_next relups_prev; # debug + #declare -p link_next link_prev; # debug + ((n++)); + done < <(read_stdin); # read stdin plus one more blank line +}; # Generate wikicode from validated subpage lines main() { read_input "$@" | validate_subpage_list | generate_wikicode; }; # main program diff --git a/user/mw_get_audiobook_chapters.sh b/user/mw_get_audiobook_chapters.sh new file mode 100755 index 0000000..2d1ff38 --- /dev/null +++ b/user/mw_get_audiobook_chapters.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Print mediawiki code chapter headings and timecodes from audiobook file +# Usage: mw_get_audiobook_chapters.sh [file] ([int offset]) +# Input: arg1: path to audiobook +# Example: mw_get_audiobook_chapters.sh input.m4b +# Version: 0.1.0 + + +fin="$1"; +n=0 +while read -r line; do + if [[ $((n % 2)) -eq 0 ]]; then + chap="$(sed -e 's/^0*//' <<< "$line")"; # remove leading zero + printf "====%s====\n" "$chap"; # output wikicode header + elif [[ $((n % 2)) -eq 1 ]]; then + printf "%s\n\n" "$line"; # output timecode + fi; + #declare -p n line; # debug + ((n++)); +done < <(ffprobe -i "$fin" -print_format json -show_chapters -sexagesimal | \ + jq \ + -r '.chapters[] | (.tags.title) + "\n" + (.start_time | gsub("\\.\\d+$"; "") )' + ); diff --git a/user/rsync_tranches.sh b/user/rsync_tranches.sh index 4e27765..bcae90e 100644 --- a/user/rsync_tranches.sh +++ b/user/rsync_tranches.sh @@ -7,7 +7,7 @@ function rsync_tranches() { # Desc: Runs rsync in parallel across different files size ranges # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/ # Depends: rsync 3.2.7 - # Version: 0.0.2 + # Version: 0.1.1 local -a rsync_opts=(); local source dest; @@ -20,7 +20,13 @@ function rsync_tranches() { shift; ;; *) - if [ -z "$source" ]; then + ## If not a file or directory, assume option + if [[ ! -f "$1" ]] && [[ ! -d "$1" ]]; then + rsync_opts+=("$1"); + shift; + fi; + ## If valid file or directory, assume source or dest path + if [[ -z "$source" ]]; then source="$1"; else dest="$1"; @@ -38,15 +44,40 @@ function rsync_tranches() { # Tranche 1: 0 to 1MiB-1 rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + sleep 2; + + # Tranche 2: 1MiB to 2MiB-1 + rsync --min-size='1MiB' --max-size='2MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 3: 2MiB to 4MiB-1 + rsync --min-size='2MiB' --max-size='4MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 4: 4MiB to 8MiB-1 + rsync --min-size='4MiB' --max-size='8MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 5: 8MiB to 16MiB-1 + rsync --min-size='8MiB' --max-size='16MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 6: 16MiB to 32MiB-1 + rsync --min-size='16MiB' --max-size='32MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 7: 32MiB to 64MiB-1 + rsync --min-size='32MiB' --max-size='64MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 8: 64MiB to 128MiB-1 + rsync --min-size='64MiB' --max-size='128MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 9: 128MiB to 256MiB-1 + rsync --min-size='128MiB' --max-size='256MiB-1' "${rsync_opts[@]}" "$source" "$dest" & - # Tranche 2: 1MiB to 10MiB-1 - rsync --min-size='1MiB' --max-size='10MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + # Tranche 10: 256MiB to 512MiB-1 + rsync --min-size='256MiB' --max-size='512MiB-1' "${rsync_opts[@]}" "$source" "$dest" & - # Tranche 3: 10MiB to 100MiB-1 - rsync --min-size='10MiB' --max-size='100MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + # Tranche 11: 512MiB to 1024MiB-1 + rsync --min-size='512MiB' --max-size='1024MiB-1' "${rsync_opts[@]}" "$source" "$dest" & - # Tranche 4: Greater than 100MiB - rsync --min-size='100MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" & + # Tranche 12: Greater than 1024MiB + rsync --min-size='1024MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" & wait # Wait for all rsync processes to complete }; diff --git a/user/transcribe_whisper.sh b/user/transcribe_whisper.sh new file mode 100755 index 0000000..00dc313 --- /dev/null +++ b/user/transcribe_whisper.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Desc: Runs OpenAI Whisper on working directory media files +# Usage: ./transcribe_whisper.sh [dir] 3 +# Input: arg1 input dir +# arg2 CUDA graphics card number (zero-indexed) +# Version: 0.4.0 +# Depends: whisper ( https://github.com/openai/whisper ) + +# Find settings +firegex=".+\(aac\|aif\|aiff\|flac\|m4a\|m4b\|mkv\|mp3\|mp4\|ogg\|opus\|wav\)$"; # update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u` +fsize="10k"; # default: minimum "10k" +fdepth="10"; # find depth + +yell() { echo "$0: $*" >&2; } # print script path and all args to stderr +die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status +must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails +checkInt() { + # Desc: Checks if arg is integer + # Usage: checkInt arg + # Input: arg: integer + # Output: - return code 0 (if arg is integer) + # - return code 1 (if arg is not integer) + # Example: if ! checkInt $arg; then echo "not int"; fi; + # Version: 0.0.2 + local returnState + + #===Process Arg=== + if [[ $# -ne 1 ]]; then + die "ERROR:Invalid number of arguments:$#"; + fi; + + RETEST1='^[0-9]+$'; # Regular Expression to test + if [[ ! "$1" =~ $RETEST1 ]] ; then + returnState="false"; + else + returnState="true"; + fi; + + #===Determine function return code=== + if [ "$returnState" = "true" ]; then + return 0; + else + return 1; + fi; +} # Checks if arg is integer +find_flist() { + # Desc: print file list to stdout via `find` using script parameters + # Input: arg1: path to dir + # var: fdepth find depth + # var: firegex pattern find iregex + # var: fsize find size + if [[ ! -d "$1" ]]; then return 1; fi; + must find "$1" -maxdepth "$fdepth" -type f -iregex "$firegex" -size +"$fsize"; +}; # print file list to stdout from dir with script parameters +main() { + # Input: arg1: dir_in input dir + # arg2: cuda_num cuda GPU index + # var: fdepth (find_flist) find depth + # var: firegex (find_flist) pattern find iregex + # var: fsize (find_flist) find size + dir_in="$1"; + cuda_num="$2"; + if ! checkInt "$cuda_num"; then die "FATAL:No graphics card selected."; fi; + while read -r line; do + echo "STATUS:Processing:$line" 1>&2; + SECONDS=0; + dir_out="$(dirname "$line"; )"; + ftmp="$line".tmp; + #declare -p line dir_out ftmp; # debug + if [[ ! -f "$ftmp" ]] && \ + [[ ! -f "${line%.*}".srt ]] && \ + [[ ! -f "${line%.*}".vtt ]] && \ + [[ ! -f "${line%.*}".txt ]] && \ + [[ ! -f "${line%.*}".tsv ]] && \ + [[ ! -f "${line%.*}".json ]]; then + touch "$ftmp"; + yell "STATUS:No conflicts detected."; + else + yell "STATUS:Skipping:$line"; + continue; + fi; + whisper "$line" \ + --model large-v3 \ + --output_format all \ + --output_dir "$dir_out" \ + --language en \ + --device cuda:"$cuda_num" && \ + ( + echo "STATUS:$SECONDS:Finished:$line" 1>&2; + rm "$ftmp"; # remove .tmp file + ); + done < <(find_flist "$dir_in" | shuf); +}; # main program +export -f yell die must find_flist; + +main "$@"; + + diff --git a/user/zeropad.sh b/user/zeropad.sh new file mode 100755 index 0000000..242b30b --- /dev/null +++ b/user/zeropad.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Desc: Zero-pad working dir files with initial digits for sorting +# Usage: zeropad.sh [str fext] +# Example: zeropad.sh; +# zeropad.sh ".jpg"; +# Input: arg1 str file extension of files to apply changes to +# Version: 0.2.0 + +declare fext; # file extension + +list_files() { + # Desc: Lists working directory file basenames + # Usage: list_files + # Input: fext var file extension + # Output: stdout a line-delimited list of file names + if [[ -z "$fext" ]]; then + find . -mindepth 1 -maxdepth 1 -type f -exec basename '{}' \; ; + else + find . -mindepth 1 -maxdepth 1 -type f -exec basename '{}' \; \ + | grep -E -- "${fext}$" ; + fi; +}; # list pwd file basenames +remove_leading_zeroes() { + # Desc: Removes leading zeroes from lines + # Input: stdin + # Output: stdout + # Depends: BK-2020-03 read_stdin() + # Version: 0.0.1 + while read -r line; do + printf "%s\n" "$line" | sed -E -e 's/(^0*)([0-9].*)/\2/'; + done; +}; +get_max_digits() { + # Desc: Returns max leading digits of pwd filenames + # Depends: list_files() + # BK-2020-03: remove_leading_zeroes() + # GNU sed 4.8 + # GNU awk 5.1.0 + local output; + + # Find the maximum number of leading digits in the filenames of working dir + output="$(list_files | remove_leading_zeroes \ + | awk '{ match($0, /^[0-9]+/); if (RLENGTH > max) max=RLENGTH } END { print max }' )"; + # declare -p max_digits; # debug + printf "%s" "$output"; +}; # return max digits to stdout +main () { + # Read file extension if provided + if [[ -n "$1" ]]; then + fext="$1"; + fi; + + max_digits="$(get_max_digits)"; + + # Loop over the files and rename them + while read -r file; do + # Skip files with no leading digits + re='^[0-9]+'; + if [[ ! "$file" =~ $re ]]; then continue; fi; + + # Extract the leading digits + digits="$(echo "$file" | sed -E 's/([0-9]*).*/\1/')"; # all leading digits + digits_nlz="$(echo "$digits" | remove_leading_zeroes)"; # no leading zeroes + # Zero-pad the digits + padded_digits="$(printf "%0${max_digits}d" "$digits_nlz")"; + # Construct the new filename + new_file="${padded_digits}${file#${digits}}"; + # Rename the file + if [[ "$file" == "$new_file" ]]; then continue; fi; + mv -n "$file" "$new_file"; + #declare -p max_digits file digits digits_nlz padded_digits new_file # debug + done < <(list_files); +}; # main program + +main "$@"; + +# Author: Steven Baltakatei Sandoval +# License: GPLv3+