From: Steven Baltakatei Sandoval Date: Thu, 25 Jul 2024 18:32:45 +0000 (+0000) Subject: feat(user/bkots):Use subshells to speed up jobs X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/commitdiff_plain/21beb7be61fdf90ea0df0a542c5f29f70a4a2434?ds=sidebyside feat(user/bkots):Use subshells to speed up jobs --- diff --git a/user/bkots b/user/bkots index a7f61ee..ec371f6 100755 --- a/user/bkots +++ b/user/bkots @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Define variables +declare -g max_job_count="2"; # default max job count +declare -g age_threshold="60"; # min age to add file; seconds; declare -Ag appRollCall # Associative array for storing app status declare -Ag fileRollCall # Associative array for storing file status declare -Ag dirRollCall # Associative array for storing dir status @@ -10,8 +12,6 @@ calendars+=("https://finney.calendar.eternitywall.com"); calendars+=("https://btc.calendar.catallaxy.com"); calendars+=("https://alice.btc.calendar.opentimestamps.org"); calendars+=("https://bob.btc.calendar.opentimestamps.org"); -age_threshold="60"; # min age to add file; seconds; -max_job_count="2"; # default max job count # Declare functions yell() { echo "$0: $*" >&2; } # print script path and all args to stderr @@ -200,7 +200,7 @@ showVersion() { vbm "DEBUG:showVersion function called." cat <<'EOF' -bkots 2.0.2 +bkots 2.1.0 Copyright (C) 2022 Steven Baltakatei Sandoval License GPLv3: GNU GPL version 3 This is free software; you are free to change and redistribute it. @@ -666,99 +666,108 @@ main() { vbm "DEBUG:files_to_stamp_pruned:"; printf "%s\n" "${files_to_stamp_pruned[@]}"; fi; - + # Act on files ## Assemble and execute upgrade file commands for item in "${files_to_upgrade_pruned[@]}"; do - path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_prf" ]]; then - yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try upgrade with known calendars in random order - while read -r url; do - vbm "DEBUG:Upgrading with calendar:url:$url"; - - #### assemble command - local -a cmd_temp; - cmd_temp+=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("upgrade" "$path_prf"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & - unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; - fi; + wait_for_jobslot && { + path_prf="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_prf" ]]; then + yell "ERROR:blank upgrade item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to upgrade proof file:path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try upgrade with known calendars in random order + while read -r url; do + vbm "DEBUG:Upgrading with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("upgrade" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist upgrade "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots upgrade $path_prf\""; + fi; + } & done; ## Assemble and execute verify file commands for item in "${files_to_verify_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; - if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then - yell "ERROR:blank verify item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; - vbm "DEBUG: against proof file: path_prf:$path_prf"; - if [[ ! $option_dry_run == "true" ]]; then - ### Try verify with known calendars in random order - while read -r url; do - vbm "DEBUG:Verifying with calendar:url:$url"; + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + path_prf="$(cut -d $'\n' -f2 < <(echo "$item"))"; + if [[ -z "$path_src" ]] || [[ -z "$path_prf" ]]; then + yell "ERROR:blank verify item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to verify source file:path_src:$path_src"; + vbm "DEBUG: against proof file: path_prf:$path_prf"; + if [[ ! $option_dry_run == "true" ]]; then + ### Try verify with known calendars in random order + while read -r url; do + vbm "DEBUG:Verifying with calendar:url:$url"; + + #### assemble command + local -a cmd_temp; + cmd_temp+=("ots"); + if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; + cmd_temp+=("-l" "$url" "--no-default-whitelist"); + cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; + + #### execute command + "${cmd_temp[@]}"; + unset cmd_temp; + break; + #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; + done < <(printf "%s\n" "${calendars[@]}" | shuf); + else + yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; + fi; + } & + done; + + ## Assemble and execute stamp file commands + for item in "${files_to_stamp_pruned[@]}"; do + wait_for_jobslot && { + path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; + if [[ -z "$path_src" ]]; then + yell "ERROR:blank stamp item encountered. Skipping:item:$item"; + continue; + fi; + vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; + if [[ ! $option_dry_run == "true" ]]; then #### assemble command local -a cmd_temp; cmd_temp+=("ots"); if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("-l" "$url" "--no-default-whitelist"); - cmd_temp+=("verify" "-f" "$path_src" "$path_prf"); + cmd_temp+=("stamp" "$path_src"); if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & + "${cmd_temp[@]}"; unset cmd_temp; - break; - #ots -l "$url" --no-default-whitelist verify -f "$path_src" "$path_prf" && break; - done < <(printf "%s\n" "${calendars[@]}" | shuf); - else - yell "DEBUG:DRY RUN:Not running:\"ots verify -f $path_src $path_prf\""; - fi; - done; - - ## Assemble and execute stamp file commands - for item in "${files_to_stamp_pruned[@]}"; do - path_src="$(cut -d $'\n' -f1 < <(echo "$item"))"; - if [[ -z "$path_src" ]]; then - yell "ERROR:blank stamp item encountered. Skipping:item:$item"; - continue; - fi; - vbm "DEBUG:Attempting to stamp source file:path_src:$path_src"; - if [[ ! $option_dry_run == "true" ]]; then - - #### assemble command - local -a cmd_temp; - cmd_temp+=("ots"); - if [[ "$opVerbose" = "true" ]]; then cmd_temp+=("-v"); fi; - cmd_temp+=("stamp" "$path_src"); - if [[ "$opVerbose" = "true" ]]; then declare -p cmd_temp; fi; - - #### execute command - wait_for_jobslot && "${cmd_temp[@]}" & - unset cmd_temp; - #ots stamp "$path_src"; - else - yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; - fi; + #ots stamp "$path_src"; + else + yell "DEBUG:DRY RUN:Not running:\"ots stamp $path_src\""; + fi; + } & done; + + ## Wait for jobs to finish. + wait; }; # main program # Run program