From fdf917e1ee70b612202fe10fab2e73d0ea077017 Mon Sep 17 00:00:00 2001 From: Steven Baltakatei Sandoval Date: Thu, 4 Jun 2020 02:04:22 +0000 Subject: [PATCH] feature(unitproc) Add legacy unit process scripts Add legacy bash scripts that fall under the "unit process" criteria for the `unitproc` directory. --- unitproc/bkbtclog | 59 ++++ unitproc/bkcsvjoin | 206 ++++++++++++ unitproc/bkdstcountdown | 59 ++++ unitproc/bkfind | 121 +++++++ unitproc/bkgenpass | 71 +++++ unitproc/bkgetbtchash | 21 ++ unitproc/bkgettext | 84 +++++ unitproc/bkhashwatch | 43 +++ {user => unitproc}/bklu | 0 unitproc/bkpoe | 678 ++++++++++++++++++++++++++++++++++++++++ unitproc/bktemplate.sh | 379 ++++++++++++++++++++++ 11 files changed, 1721 insertions(+) create mode 100755 unitproc/bkbtclog create mode 100755 unitproc/bkcsvjoin create mode 100755 unitproc/bkdstcountdown create mode 100755 unitproc/bkfind create mode 100755 unitproc/bkgenpass create mode 100755 unitproc/bkgetbtchash create mode 100755 unitproc/bkgettext create mode 100755 unitproc/bkhashwatch rename {user => unitproc}/bklu (100%) create mode 100755 unitproc/bkpoe create mode 100755 unitproc/bktemplate.sh diff --git a/unitproc/bkbtclog b/unitproc/bkbtclog new file mode 100755 index 0000000..cd2353a --- /dev/null +++ b/unitproc/bkbtclog @@ -0,0 +1,59 @@ +#!/bin/bash + +# Date: 2020-03-08T03:48Z; baltakatei> + +# Description: Log bitcoin block stats to file every minute for one +# hour. Also adds stats to PRNG. + +# Usage: bkbtclog [path] + + # [path]: Path to write log file. + +# Dependencies: jq + +# Modify PATH to include some non-standard executable directories. +PATH="/usr/local/bin/:$PATH" # for bitcoind and bitcoin-cli +PATH="/usr/bin/:$PATH" # for jq + +# Exit if jq is unavailable or bitcoin-cli getblockcount not available. +if ! ( ( command -v jq 1>/dev/null 2>&1 ) && ( bitcoin-cli getblockcount 1>/dev/null 2>&1 ) ) ; then echo "ERROR:Commands jq or bitcoin-cli not available." 1>&2; exit 1; fi + +# Set script time-to-live to 1 hour in seconds. See https://stackoverflow.com/a/11198713 . +SCRIPT_TTL=3600 +SECONDS_END=$(( SECONDS + SCRIPT_TTL )) + +# Set output log file path +LOG_PATH="$1" +LOG_DIR=$(dirname "$LOG_PATH" ) + +# Exit if LOG_DIR doesn't exist. +if [ ! -d "$LOG_DIR" ]; then echo "ERROR:Specified log directory doesn't exist." 1>&2; + echo "LOG_DIR is:""$LOG_DIR" 1>&2; exit 1; fi + +# Loop until script age exceeds SCRIPT_END. +while [ $SECONDS -lt $SECONDS_END ]; do + # Create log file and first line header if file at LOG_PATH doesn't exist. + if [ ! -f "$LOG_PATH" ]; then + echo "TIME","BTC_BESTBLOCKCOUNT","BTC_BESTBLOCKTIME","BTC_BESTBLOCKHASH","BTC_MEDIANFEE","BTC_TOTALOUTPUT","BTC_TXTOTALSIZE","BTC_TXCOUNT","BTC_TXIN","BTC_TXOUT","BTC_MAXTXFEERATE","BTC_MINTXFEERATE","BITCOINCORE_VERSION" >> "$LOG_PATH"; + fi + TIME="$(date +%Y%m%dT%H%M%S%z)" + BITCOINCORE_VERSION="$(bitcoin-cli -version)" + BTC_BESTBLOCKSTATS_JSON="$(bitcoin-cli getblockstats "$(bitcoin-cli getblockcount)" '["height","time","blockhash","medianfee","total_out","total_size","txs","ins","outs","maxfeerate","minfeerate"]')" + BTC_BESTBLOCKCOUNT=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .height' ) + BTC_BESTBLOCKTIME_UNIX=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .time' ) + BTC_BESTBLOCKTIME=$(date --date=@"$BTC_BESTBLOCKTIME_UNIX" +%Y%m%dT%H%M%S%z ) + BTC_BESTBLOCKHASH=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .blockhash' | tr -dc "[:xdigit:]" ) + BTC_MEDIANFEE=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .medianfee' ) + BTC_TOTALOUTPUT=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .total_out' ) + BTC_TXTOTALSIZE=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .total_size' ) + BTC_TXCOUNT=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .txs' ) + BTC_TXIN=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .ins' ) + BTC_TXOUT=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .outs' ) + BTC_MAXTXFEERATE=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .maxfeerate' ) + BTC_MINTXFEERATE=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .minfeerate' ) + OUTPUT=$(echo "$TIME","$BTC_BESTBLOCKCOUNT","$BTC_BESTBLOCKTIME","$BTC_BESTBLOCKHASH","$BTC_MEDIANFEE","$BTC_TOTALOUTPUT","$BTC_TXTOTALSIZE","$BTC_TXCOUNT","$BTC_TXIN","$BTC_TXOUT","$BTC_MAXTXFEERATE","$BTC_MINTXFEERATE","$BITCOINCORE_VERSION") + echo "$OUTPUT" >> "$LOG_PATH"; + echo "$OUTPUT" >> /dev/random; + sleep 60 +done +exit 0 diff --git a/unitproc/bkcsvjoin b/unitproc/bkcsvjoin new file mode 100755 index 0000000..beb1fbf --- /dev/null +++ b/unitproc/bkcsvjoin @@ -0,0 +1,206 @@ +# Date: 2020-02-21T21:23Z; + +# Author: Steven Baltakatei Sandoval + +# License: This bash script, 'bkcsvjoin.sh', is licensed under GPLv3 or +# later by Steven Baltakatei Sandoval: +# +# 'bkcsvjoin.sh' sorts and joins two CSV files based on first field +# Copyright (C) 2020 Steven Baltakatei Sandoval (baltakatei.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# A copy of the GNU General Public License may be found at +# . + +# Description: Sorts and joins two CSV files based upon the contents +# of the first field ("field1") in each line of CSV file. It assumes +# the first line is a header labelling field names. It outputs several +# files into a new directory in the working directory. Set theory +# characters in output file names indicate file 1 elements with "A" +# and file 2 elements with "B". Input assumes UNIX-style line +# endings. The output files use the following numbers and definitions: +# +# 1. A⋃B The union of A and B. Contains all field1 elements from file 1 and 2. +# +# 2. A⋃(A⋂B) Contains all field1 elements of file 1 plus the matches between +# file 1 and 2. +# +# 3. B⋃(B⋂A) Contains all field1 elements of file 2 plus the matches between +# file 1 and 2. +# +# 4. A⋂B Contains only field1 elements present in both file 1 and file 2. +# +# 5. A-B Contains all field1 elements of file 1 except when a matching +# element is present in file 2. +# +# 6. B-A Contains all field1 elements of file 2 except when a matching +# element is present in file 1. +# +# 7. A⊖B Contains all field1 elements of file 1 and file 2 except for +# elements which are present in both file 1 and 2. + +# Usage: bkcsvjoin.sh [ file1 ] [ file2 ] > fileOut.csv + +# Dependencies: GNU Coreutils 8.30-3 (bash, join, sort, head, tail, +# mkdir) (see end of file for license details). Note: only tested on +# Debian GNU/Linux 10. + +echoerr() { echo "$@" 1>&2; } # Function for outputing text to stderr. + +SCRIPT_DATE=$(date +%Y%m%dT%H%M%S%z) +SCRIPT_PATH=$(pwd) + +# Create temporary working directory +TEMP_DIRPATH="$SCRIPT_PATH"/"$SCRIPT_DATE"..temp +mkdir "$TEMP_DIRPATH" + +# Read file names as input arguments +FILE1_PATH="$1" +FILE2_PATH="$2" +FILE1_BASENAME=$(basename "$FILE1_PATH") +FILE2_BASENAME=$(basename "$FILE2_PATH") +#[[ -f "$FILE1_PATH" ]] && echoerr "DEBUG:$FILE1_PATH is file with basename $FILE1_BASENAME." +#[[ -f "$FILE2_PATH" ]] && echoerr "DEBUG:$FILE2_PATH is file with basename $FILE2_BASENAME." + +# Warn and prompt if carriage return character is used. +if [[ $(grep -Uc $'\x0D' "$FILE1_PATH") -gt 0 ]] || [[ $(grep -Uc $'\x0D' "$FILE2_PATH") -gt 0 ]]; then + echoerr "ERROR: Carriage Return character(s) detected. Windows text file?" + read -p "Do you wish to attempt to convert DOS-style line endings to UNIX-style by removing all carriage returns? (probably okay) [y/n]" choice + case "$choice" in + y|Y ) echoerr "yes" + OPTION_DOS2UNIX="true" + ;; + n|N ) echoerr "no" + OPTION_DOS2UNIX="false" + echoerr "WARNING:Output may contain mix of DOS-style (CRLF) and UNIX-style (LF) line endings!" + echoerr " See https://stackoverflow.com/a/2613834 " + sleep 2 + ;; + * ) echo "Invalid selection." + echoerr "Exiting." && exit 1;; + esac +fi + +# Sort files on field1 and write to temporary directory. See [1], and [2]. +FILE1_SORTED_PATH="$TEMP_DIRPATH"/"$FILE1_BASENAME"_SORTED.csv +FILE2_SORTED_PATH="$TEMP_DIRPATH"/"$FILE2_BASENAME"_SORTED.csv +( head -n1 $FILE1_PATH && tail -n+2 $FILE1_PATH | sort -t, -k1,1 | uniq ) > $FILE1_SORTED_PATH +( head -n1 $FILE2_PATH && tail -n+2 $FILE2_PATH | sort -t, -k1,1 | uniq ) > $FILE2_SORTED_PATH + +# 1. A⋃B. Join files into Union of (file 1) + (file 2) (i.e. throwing nothing out). See [3], [6]. +FILES_JOINED_COMPLETE="$( join -t, -a1 -a2 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 2. A⋃(A⋂B). Join files, file 1 uniques only (omit unmatched field1 entries from file 2). See [6]. +FILES_JOINED_F1UO="$( join -t, -a1 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 3. B⋃(A⋂B). Join files, file 2 uniques only (omit unmatched field1 entries from file 1). See [6]. +FILES_JOINED_F2UO="$( join -t, -a2 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 4. A⋂B. Join files, only entries present in both file 1 and file 2 ( file 1 ⋂ file 2 ). See [6]. +FILES_JOINED_INTERSECTION="$( join -t, -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 5. A-B. Join files into Relative Complement of (file 1) - (file 2) field1 entries. See See [5], [6]. +FILES_JOINED_DIFFERENCE_AB="$( join -t, -v1 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 6. B-A. Join files into Relative Complement of (file 2) - (file 1) field1 entries. See [5], [6]. +FILES_JOINED_DIFFERENCE_BA="$( join -t, -v2 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# 7. A⊖B. Join files into Symmetric Difference of (file 2) ⊖ (file 1) field1 entries. See [5], [6]. +FILES_JOINED_DIFFERENCE_SYM="$( join -t, -v1 -v2 -o auto --header "$FILE1_SORTED_PATH" "$FILE2_SORTED_PATH" )" + +# Attempt to restore to UNIX-style line endings by removing all carriage return (CR) characters OPTION_DOS2UNIX set to true. See [7]. +if [[ $OPTION_DOS2UNIX == "true" ]]; then + FILES_JOINED_COMPLETE="$(echo "$FILES_JOINED_COMPLETE" | tr -d '\r')" + FILES_JOINED_F1UO="$(echo "$FILES_JOINED_F1UO" | tr -d '\r')" + FILES_JOINED_F2UO="$(echo "$FILES_JOINED_F2UO" | tr -d '\r')" + FILES_JOINED_INTERSECTION="$(echo "$FILES_JOINED_INTERSECTION" | tr -d '\r')" + FILES_JOINED_DIFFERENCE_AB="$(echo "$FILES_JOINED_DIFFERENCE_AB" | tr -d '\r')" + FILES_JOINED_DIFFERENCE_BA="$(echo "$FILES_JOINED_DIFFERENCE_BA" | tr -d '\r')" + FILES_JOINED_DIFFERENCE_SYM="$(echo "$FILES_JOINED_DIFFERENCE_SYM" | tr -d '\r')" +fi + +# Output CSV files to temp directory with carriage returns ('\r') removed. +echo "$FILES_JOINED_COMPLETE" > "$TEMP_DIRPATH"/"1.A⋃B..UNION.csv" && echoerr "1.A⋃B..UNION.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_F1UO" > "$TEMP_DIRPATH"/"2.A⋃(A⋂B)..FILE-1-UNIQUES-ONLY.csv" && echoerr "2.A⋃(A⋂B)..FILE-1-UNIQUES-ONLY.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_F2UO" > "$TEMP_DIRPATH"/"3.B⋃(A⋂B)..FILE-2-UNIQUES-ONLY.csv" && echoerr "3.B⋃(A⋂B)..FILE-2-UNIQUES-ONLY.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_INTERSECTION" > "$TEMP_DIRPATH"/"4.A⋂B..INTERSECTION.csv" && echoerr "4.A⋂B..INTERSECTION.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_DIFFERENCE_AB" > "$TEMP_DIRPATH"/"5.A-B..DIFFERENCE_A_MINUS_B.csv" && echoerr "5.A-B..DIFFERENCE_A_MINUS_B.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_DIFFERENCE_BA" > "$TEMP_DIRPATH"/"6.B-A..DIFFERENCE_B_MINUS_A.csv" && echoerr "6.B-A..DIFFERENCE_B_MINUS_A.csv written to $TEMP_DIRPATH" +echo "$FILES_JOINED_DIFFERENCE_SYM" > "$TEMP_DIRPATH"/"7.A⊖B..DIFFERENCE_SYM.csv" && echoerr "7.A⊖B..DIFFERENCE_SYM.csv written to $TEMP_DIRPATH" + +# ==References== +# [1]: How to sort a csv file while retaining the first line. https://stackoverflow.com/a/14562674 +# [2]: How to sort a csv file by sorting on a single field. https://stackoverflow.com/a/44744800 +# [3]: How to join multiple sorted csv files into one. https://stackoverflow.com/a/27606199 +# [4]: How to join a csv file retaining the first line, sorting by certain fields, and not omitting nonmatching fields. https://superuser.com/a/1527095 +# [5]: How to determine symmetric difference between two sorted files. https://www.gnu.org/software/coreutils/manual/html_node/Set-operations.html +# [6]: Set theory symbols https://www.rapidtables.com/math/symbols/Set_Symbols.html +# [7]: How to convert DOS-style to UNIX-style line endings. https://stackoverflow.com/a/2613834 + +# ==Dependencies== + + # GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) + # Copyright (C) 2019 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later + + # This is free software; you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + + # join (GNU coreutils) 8.30 + # Copyright (C) 2018 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later . + # This is free software: you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + # Written by Mike Haertel. + + + # sort (GNU coreutils) 8.30 + # Copyright (C) 2018 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later . + # This is free software: you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + # Written by Mike Haertel and Paul Eggert. + + + # head (GNU coreutils) 8.30 + # Copyright (C) 2018 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later . + # This is free software: you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + # Written by David MacKenzie and Jim Meyering. + + + # tail (GNU coreutils) 8.30 + # Copyright (C) 2018 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later . + # This is free software: you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + # Written by Paul Rubin, David MacKenzie, Ian Lance Taylor, + # and Jim Meyering. + + + # mkdir (GNU coreutils) 8.30 + # Copyright (C) 2018 Free Software Foundation, Inc. + # License GPLv3+: GNU GPL version 3 or later . + # This is free software: you are free to change and redistribute it. + # There is NO WARRANTY, to the extent permitted by law. + + # Written by David MacKenzie. + +# ==Metadata== +# - Created in task 20200220_90eb0c7d + diff --git a/unitproc/bkdstcountdown b/unitproc/bkdstcountdown new file mode 100755 index 0000000..ee3b094 --- /dev/null +++ b/unitproc/bkdstcountdown @@ -0,0 +1,59 @@ +#!/bin/bash + +# Date: 2020-02-13T23:14Z; baltakatei> + +# Description: This script outputs days until the next time zone +# discontinuity using 'zdump' and 'date'. + +#=================ADJUST ME======================= +SCRIPT_TZ="America/Los_Angeles" # Timezone to check for discontinuities +#================================================= + +echoerr() { + echo "$@" 1>&2; +} + +# Declare variables +declare -a datesDiscontArray #array + +SCRIPT_YEAR=$(date +%Y) +SCRIPT_YEAR_NEXT=$(( $(date +%Y) + 1)) +SCRIPT_RUN_DATE_SECONDS=$(date +%s) + +# Save zdump output for SCRIPT_YEAR (for how multiline data saved in variable, see https://stackoverflow.com/a/613580 ) +datesDiscont="$(zdump -c $SCRIPT_YEAR_NEXT -v $SCRIPT_TZ | grep $SCRIPT_YEAR | awk '{ print $2 " " $3 " " $4 " " $5 " " $6 " " $7 }'; echo)" +if [ -z "$datesDiscont" ]; then + echo "No time discontinuity this year." + exit 1 +fi + +###echoerr "datesDiscont:--""$datesDiscont""--" + +# Count lines in datesDiscont (for counting lines in a variable, see https://stackoverflow.com/a/6314682 ) +#datesDiscontLineCount=$(echo "$datesDiscont" | wc -l) + +# Convert datesDiscont into array (for how to process each line in a multiline variable, see https://superuser.com/a/284226 ) +while IFS= read -r line; do + #echo "$line" + #datesDiscontArray+="$line" + datesDiscontArray+=($(date --date="$line" +%s)) + #echo ${datesDiscontArray[-1]} +done <<< "$datesDiscont" +#echoerr ${datesDiscontArray[@]} + +# Sort datesDiscontArray (see https://stackoverflow.com/a/11789688 ) +IFS=$'\n' datesDiscontArray=($(sort <<<"${datesDiscontArray[*]}")) +unset IFS + +# Get earliest date in datesDiscontArray that isn't in the past. +for discontinuityDate in "${datesDiscontArray[@]}"; do + ###echoerr "Discontinuity date (seconds):"$discontinuityDate + ###echoerr "SCRIPT_RUN_DATE_SECONDS :"$SCRIPT_RUN_DATE_SECONDS + if [ $discontinuityDate -gt $SCRIPT_RUN_DATE_SECONDS ]; then + ###echoerr "DEBUG:Date in future." + nextDiscontDate="$discontinuityDate" + break + fi +done + +echo $(( ( $nextDiscontDate - $SCRIPT_RUN_DATE_SECONDS )/86400 ))" days" # Days until next discontinuity date diff --git a/unitproc/bkfind b/unitproc/bkfind new file mode 100755 index 0000000..bfba95d --- /dev/null +++ b/unitproc/bkfind @@ -0,0 +1,121 @@ +#!/bin/bash + +# Date: 2020-01-20T17:08Z +# +# Author: Steven Baltakatei Sandoval (baltakatei.com) +# +# License: This bash script, `bkfind`, is licensed under GPLv3 or +# later by Steven Baltakatei Sandoval: +# +# `bkfind`, a duplicate file finder +# Copyright (C) 2020 Steven Baltakatei Sandoval (baltakatei.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# A copy of the GNU General Public License may be found at +# . +# +# Description: This is a script that searches a specified directory +# for files with a file name containing a specified string. It works +# as follows: +# +# - Search specified directory tree for files that have filenames +# that contain the specified file's filename. List groups of files +# sharing the same hash first then list files with unique hashes. +# +# Dependencies: find, rhash, uniq, cut, cat, bash. See end of file +# +# Tested on: +# +# - GNU/Linux Debian 10 +# + + +#==Initialization== +# Use input arguments to define internal script variables. +DIR1="$1" # Specified directory +FILE1="$2" # Specified file +DUPLICATES1="" +DUPLICATES2="" +DUPLICATES3="" +UNIQUES1="" +UNIQUES2="" +UNIQUES3="" +RHASH_HASH_TYPE="sha512" +HASH_DISP_LENGTH=16 +let HASH_CHAR_LENGTH="512 / 4" # The number of characters returned by the chosen hash function (ex: `rhash --sha512 {}` produces 512/4=128 hexadecimal chars) + +# Strip path information from provided file name. +FILEBASE1=$(basename "$FILE1") + + +#==Main Program== +# Generate list of sha512 hashes and filepaths, save to $HASHLIST1 +HASHLIST1="$(find $DIR1 -type f -iname "*$FILEBASE1*" -exec rhash --"$RHASH_HASH_TYPE" {} \;)" + +# Specify character position before which characters are dropped from each line with `cut`. +let CUT_POSITION="1 + $HASH_CHAR_LENGTH - $HASH_DISP_LENGTH" + +#====Files with duplicate hashes==== +# Generate sublist of duplicate entries, save to $DUPLICATES1 +DUPLICATES1="$(echo -e "$HASHLIST1" | sort | uniq -D --check-chars=128)" + +# Format $DUPLICATES1 for readability by grouping, truncating sha512 hash; save to $DUPLICATES2 +DUPLICATES2="$(echo -e "$DUPLICATES1" | uniq --check-chars=128 --group | cut --characters=$CUT_POSITION-)" + +#====Files with unique hashes==== +# Generate sublist of unique entries, save to $UNIQUES1 +UNIQUES1="$(echo -e "$HASHLIST1" | sort | uniq --unique --check-chars=128)" + +# Format $UNIQUES1 for readability by truncating sha512 hash; save to $UNIQUES2 +UNIQUES2="$(echo -e "$UNIQUES1" | cut --characters=$CUT_POSITION-)" + + +# List results +echo -e "$DUPLICATES2" +echo -e "$UNIQUES2" + +# Dependencies: +# +# - find (GNU findutils) 4.6.0.225-235f +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# Written by Eric B. Decker, James Youngman, and Kevin Dalley. +# +# - RHash v1.3.8 +# License: RHash License +# +# - uniq (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Richard M. Stallman and David MacKenzie. +# +# - cut (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by David M. Ihnat, David MacKenzie, and Jim Meyering. +# +# - cat (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Torbjorn Granlund and Richard M. Stallman. +# +# - GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later +# This is free software; you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. diff --git a/unitproc/bkgenpass b/unitproc/bkgenpass new file mode 100755 index 0000000..077a88b --- /dev/null +++ b/unitproc/bkgenpass @@ -0,0 +1,71 @@ +#!/bin/bash + +# 2019-08-23T00:36Z; baltakatei> + +# This bash script should prompt the user for a URL, email address, +# user name, text notes, and then generate a password, push this +# password into the x clipboard, and then save a text file containing +# the prompted details. This script should be run by Debian-based +# GNU/Linux machines owned by baltakatei. + +# This file by Steven Baltakatei Sandoval is licensed under CC BY-SA 4.0. + +# Ask user for registration fields +echo "What is the URL for whose website you are registering? (omit https://)" +read input_URL +echo "input_URL is:"$input_URL + +echo "What is your email address?" +read input_EMAIL +echo "input_EMAIL is:"$input_EMAIL + +echo "What is your user name (if any)?" +read input_USERNAME +echo "input_USERNAME is:"$input_USERNAME + +echo "Notes?" +read input_NOTE +echo "input_NOTE is:"$input_NOTE + +# Determine long date +LONGDATE=$(date +%Y%m%dT%H%M%S.%N%z) +echo "Current time is:"$LONGDATE + +# Determine hostname +HOSTNAME=$(hostname) +echo "HOSTNAME is:"$HOSTNAME + +# Perform superstitious PRNG spells +echo $LONGDATE$HOSTNAME > /dev/random +echo $(cat /proc/sys/kernel/random/entropy_avail)" bits of entropy in /dev/random" + +# Determine random 128-bit entropy password +PASS1=$(LC_ALL=C tr -dc "[:graph:]" < /dev/urandom | head -c ${1:-8}) +PASS2=$(LC_ALL=C tr -dc "[:graph:]" < /dev/urandom | head -c ${1:-8}) +PASS3=$(LC_ALL=C tr -dc "[:graph:]" < /dev/urandom | head -c ${1:-4}) +PASSPHRASE=$PASS1" "$PASS2" "$PASS3 +echo "PASSPHRASE is:"$PASSPHRASE + +# Print fields to temporary file +#echo "PASSPHRASE="$PASSPHRASE >> $PATHFILENAME +#echo "URL="$input_URL >> $PATHFILENAME +#echo "EMAIL="$input_EMAIL >> $PATHFILENAME +#echo "USERNAME="$input_USERNAME >> $PATHFILENAME +#echo "NOTE="$input_NOTE >> $PATHFILENAME +#echo "DATE="$LONGDATE >> $PATHFILE + +# Push passphrase to clipboard +echo -n $PASSPHRASE | xclip -selection clipboard + +echo "Passphrase copied to clipboard via xclip." + +# Push passphrase to `pass` +#printf "$PASSPHRASE\nURL=$input_URL\nEMAIL=$input_EMAIL\nUSERNAME=$input_USERNAME\nNOTE=$input_NOTE\nDATE=$LONGDATE\n" | pass insert -m 01\ Online\ Services/$input_URL + +echo "$PASSPHRASE +URL=$input_URL +EMAIL=$input_EMAIL +USERNAME=$input_USERNAME +NOTE=$input_NOTE +DATE=$LONGDATE +" | pass insert -m 01\ Online\ Services/$input_URL diff --git a/unitproc/bkgetbtchash b/unitproc/bkgetbtchash new file mode 100755 index 0000000..23537eb --- /dev/null +++ b/unitproc/bkgetbtchash @@ -0,0 +1,21 @@ +#!/bin/bash + +# Date: 2020-03-07T22:18Z; baltakatei> + +# Description: Return latest bitcoin block stats in comma-delimited +# line + +# Exit if jq is unavailable or bitcoin-cli getblockcount not available. +if ! ( command -v jq 1>/dev/null 2>&1 ) && ! ( bitcoin-cli getblockcount 1>/dev/null 2>&1 ); then echo "Commands jq or bitcoin-cli not available." 1>&2; exit 1; fi + +TIME="$(date +%Y%m%dT%H%M%S%z)" +BTC_BESTBLOCKSTATS_JSON="$(bitcoin-cli getblockstats "$(bitcoin-cli getblockcount)" '["height","time","blockhash"]')" +BTC_BESTBLOCKCOUNT=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .height') +BTC_BESTBLOCKTIME=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .time') +BTC_BESTBLOCKHASH=$(echo "$BTC_BESTBLOCKSTATS_JSON" | jq '. | .blockhash') + +if [ ! -f "$DIR_LOG_PATH" ]; then + echo "EVENT_DATE,TARGET_FILEMTIME,DIGEST_ALGO,TARGET_DIGEST,TARGET_FILESIZE,TARGET_FILEPATH" >> "$DIR_LOG_PATH" # print field names in first row +fi + +echo "$TIME","$BTC_BESTBLOCKCOUNT","$BTC_BESTBLOCKTIME","$BTC_BESTBLOCKHASH" >> $LOG_PATH; diff --git a/unitproc/bkgettext b/unitproc/bkgettext new file mode 100755 index 0000000..b220a20 --- /dev/null +++ b/unitproc/bkgettext @@ -0,0 +1,84 @@ +#!/bin/bash + +# Date: 2020-02-15T22:23Z + +# Author: Steven Baltakatei Sandoval + +# Description: `bkgettext` prints text between and including two +# specified tags. + +# Usage: +# $ bkgettext [ -s tagStart ] [ -e tagEnd ] [ -f file ] + +echoerr() { echo "$@" 1>&2; } # function for outputing to stderr; see [3] +vc() { [ "$OPTION_VERBOSE" == "true" ] && return 0 || return 1; } # returns true if verbose mode active +print_help() { echoerr "Usage: [ -s tagStart ] [ -e tagEnd ] [ -f file ] "; } # print help +process_inputs() { + # echoerr "process_inputs() \$1:""===""$1""===" + # echoerr "process_inputs() \$2:""===""$2""===" + # echoerr "process_inputs() \$3:""===""$3""===" + # echoerr "process_inputs() \$4:""===""$4""===" + # echoerr "process_inputs() \$5:""===""$5""===" + # echoerr "process_inputs() \$6:""===""$6""===" + # echoerr "process_inputs() \$7:""===""$7""===" + # echoerr "process_inputs() \$8:""===""$8""===" + + while getopts "vhs:e:f:" options_array; do + case "${options_array}" in + v) + OPTION_VERBOSE="true" + vc && echoerr "DEBUG:Verbose mode active." + ;; + h) + print_help; exit 1 + ;; + s) + tagStart=${OPTARG} + vc && echoerr "DEBUG:tagStart:""$tagStart" + ;; + e) + tagEnd=${OPTARG} + vc && echoerr "DEBUG:tagEnd:""$tagEnd" + ;; + f) + [ ! -f ${OPTARG} ] && echoerr "ERROR: Invalid file name provided." && exit 1; + fileInput=${OPTARG} + vc && echoerr "DEBUG:fileInput:""$fileInput" + ;; + *) + echoerr "Error." + ;; + esac + done + shift $((OPTIND-1)) +} + +main() { + + process_inputs "$@" # Define variables: tagStart tagEnd fileInput + # echoerr "main() \$1:""===""$1""===" + # echoerr "main() \$2:""===""$2""===" + # echoerr "main() \$3:""===""$3""===" + # echoerr "main() \$4:""===""$4""===" + # echoerr "main() \$5:""===""$5""===" + # echoerr "main() \$6:""===""$6""===" + # echoerr "main() \$7:""===""$7""===" + # echoerr "main() \$8:""===""$8""===" + # echoerr "main() \$tagStart:""===""$tagStart""===" + # echoerr "main() \$tagEnd:""===""$tagEnd""===" + # echoerr "main() \$fileInput:""===""$fileInput""===" + #OUTPUT=$(cat "$fileInput" | awk "/$tagStart/,/$tagEnd/") # get text between and including tag lines; see [1] + #OUTPUT=$(cat "$fileInput" | awk "/$tagStart/{f=1;next} /$tagEnd/{f=0} f") # get text between tag lines; see [1] + OUTPUT=$(cat "$fileInput" | awk "/$tagStart/{f=1} /$tagEnd/{f=0;print} f") # get text between and including tag lines; see [1] + echo "$OUTPUT" +} + +main "$@" # pass arguments to function "main" and execute "main" (for why `"$@"`, see [2]) +exit 0 # exit normally + + + +#===References=== +# [1]: How to get text between and including specified strings: https://stackoverflow.com/a/22222219 +# [2]: How to correctly pass bash script arguments to functions: https://stackoverflow.com/a/8198970 +# [3]: How to print text to stderr instead of stdout: https://stackoverflow.com/a/2990533 diff --git a/unitproc/bkhashwatch b/unitproc/bkhashwatch new file mode 100755 index 0000000..63aeabc --- /dev/null +++ b/unitproc/bkhashwatch @@ -0,0 +1,43 @@ +#!/bin/bash + +# Date: 2020-04-08T23:37Z + +# Description: A bash script that hashes files immediately after they +# are modified. Watches for 1 hour then exits. + +# Dependencies: inotifywait, timeout, awk, b2sum, date + +echoerr() { echo "$@" 1>&2; } + +DIR_TO_WATCH="/home/baltakatei/Sync" +DIR_LOG="/home/baltakatei/Sync/kodawkuori-07/2020/archive-PERS/logs/files" +DIGEST_ALGO="b2sum" +DIR_LOG_PATH=$DIR_LOG"/""$(date +%Y%m%d)""..""$(hostname)""_""$DIGEST_ALGO""_filewrites.log" # The .log extension is important for inotifywait "--exclude" option. +TIMEOUT="1h" # Limit inotifywait process to 1 hour. +MIN_FILE_SIZE=1 # smallest file size to log (in bytes) + +if ! command -v "$DIGEST_ALGO" 1>/dev/null 2>/dev/null; then echoerr "ERROR: $0 could not find command $DIGEST_ALGO ."; exit 1; fi + +if [ ! -d "$DIR_TO_WATCH" ]; then echoerr "ERROR: $0 could not parse $DIR_TO_WATCH as directory."; exit 1; fi + +timeout $TIMEOUT inotifywait -m -e close_write -e moved_to --exclude ".tmp$" --exclude ".log$" -r --format "%w%f" "$DIR_TO_WATCH" | + while read line; do + # note: make sure to exclude the $DIR_LOG_PATH via ".log$" (or equivalent means) to avoid endless logging of log writes. + EVENT_DATE="$(date +%Y%m%dT%H%M%S.%N%z )" + TARGET_FILEPATH="$(echo -n $line )" + TARGET_FILENAME="$(basename "$TARGET_FILEPATH" )" + TARGET_FILESIZE="$(du -b "$TARGET_FILEPATH" | awk '{print $1}' )" + TARGET_FILEMTIME="$(date -r "$TARGET_FILEPATH" +%Y%m%dT%H%M%S.%N%z)" + TARGET_DIGEST="$(cat "$TARGET_FILEPATH" | $DIGEST_ALGO | awk '{print $1}' )" + EVENT_LOG_ENTRY="$EVENT_DATE","$TARGET_FILEMTIME","$DIGEST_ALGO","$TARGET_DIGEST","$TARGET_FILESIZE","$TARGET_FILEPATH" + echo "$EVENT_LOG_ENTRY" >> /dev/random # Mix written file's digest with system PRNG. + if [ -d "$DIR_LOG" ] && [ $(( "$TARGET_FILESIZE" - "$MIN_FILE_SIZE" )) -ge 0 ]; then + if [ ! -f "$DIR_LOG_PATH" ]; then + echo "EVENT_DATE,TARGET_FILEMTIME,DIGEST_ALGO,TARGET_DIGEST,TARGET_FILESIZE,TARGET_FILEPATH" >> "$DIR_LOG_PATH" # print field names in first row + fi + echo "$EVENT_LOG_ENTRY" >> "$DIR_LOG_PATH"; # Log written file's digest if TARGET_FILESIZE greater or equal to MIN_FILE_SIZE (in bytes). + #echo "$EVENT_LOG_ENTRY" >> /tmp/bkhashwatch.log #debug + fi + done +echo "Timeout of $TIMEOUT elapsed. Exiting." +exit 0 diff --git a/user/bklu b/unitproc/bklu similarity index 100% rename from user/bklu rename to unitproc/bklu diff --git a/unitproc/bkpoe b/unitproc/bkpoe new file mode 100755 index 0000000..ff4c738 --- /dev/null +++ b/unitproc/bkpoe @@ -0,0 +1,678 @@ +#!/bin/bash +# +# Date: 2020-02-18T20:06Z +# +# Author: Steven Baltakatei Sandoval (baltakatei.com) +# +# License: This bash script, 'bkpoe', is licensed under GPLv3 or +# later by Steven Baltakatei Sandoval: +# +# 'bkpoe', a file system proof of existence generator +# Copyright (C) 2020 Steven Baltakatei Sandoval (baltakatei.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# A copy of the GNU General Public License may be found at +# . +# +# Description: This is a simple script that uses GNU Coreutils +# OpenTimestamps to generate a proof that permits a user to prove the +# existence of any individual file within a directory. A proof +# consists of: +# +# - An OpenTimestamps 'ots' file (if '-t' option provided) +# +# - Public proof: text file(s) containing a list of salted file +# hashes. +# +# - Private proof: text file(s) containing: +# +# - A list of unsalted file hashes +# +# - A list of nonces used to generate the list of salted file hashes. +# +# - The list of salted file hashes +# +# - File attributes (size, relative path) +# +# The proof is produced by performing the following steps: +# +# - Use 'find' and GNU Coreutils digest algorithms to create +# numbered lists of file digests at specified directories. +# +# - Save these numbered lists of digests into text files within a +# proof directory. +# +# - If more than one specified directory is provided also process +# the files within the proof directory. +# +# - Use OpenTimestamps to create an 'ots' timestamp file +# for the final digest list file. (if '-t' option provided). +# +# - Use OpenTimestamps to upgrade each 'ots' timestamp file after a +# delay. (if '-w' option provided). +# +# Dependencies: coreutils, ots. See end of file +# +# Tested on: +# +# - GNU/Linux Debian 10 + + +# === VARIABLE INITIALIZATION AND FUNCTION DEFINITIONS === + +PATH="/usr/local/bin/:$PATH" # Add default OpenTimestamps path to PATH (necessary if cron used to call this script) +scriptHostname=$(hostname) # Save hostname of system running this script. +OTS_TIMEOUT="6h" # maximum time to run `ots --wait` (used via `timeout` command) + +# Declare array for storing directories whose contents will be hashed. +declare -a directoriesToProcess + +# Declare arrays for storing file paths, file index, file hashes, file +# hash nonces, the salted hashes, file sizes, file paths, file +# modification times, and proof file paths. +declare -a filePathsFull # array +declare -a filePathsIndex # array +declare -a fileHashes # array +declare -a fileHashNonces # array +declare -a fileHashesSalted # array +declare -a fileSizes # array +declare -a fileModTime # array +declare -g PROOF_DIRECTORY # global +declare -ag prvProofPaths # array, global +declare -ag pubProofPaths # array, global + +# Define 'echoerr' function which outputs text to stderr + # Note: function copied from https://stackoverflow.com/a/2990533 +echoerr() { + echo "$@" 1>&2; +} + +# Define script version. +SCRIPT_VERSION="bkpoe 0.1.0" + +# Save current date. +runDate=$(date +%Y%m%dT%H%M%S%z) +sleep 1 # Space out program run times by at least one second. +###echoerr "DEBUG: Current date is:"$runDate + +# Save working directory +runPath=$(pwd) +###echoerr "DEBUG: Current working directory is:"$runPath + +# Define proof directory basename +PROOF_DIRECTORY_BASE="$runDate""..""$scriptHostname""_bkpoe_proofs" + +# Define default proof directory +PROOF_DIRECTORY="$runPath"/"$PROOF_DIRECTORY_BASE" + +# Print information on how to use this bash script. +usage() { + + echo "USAGE:" + echo " bkpoe [ options ] DIRECTORIES" + echo + echo "OPTIONS:" + echo " -h, --help" + echo " Display help information." + echo + echo " -r, --recursive" + echo " Perform recursive hashing of directories." + echo + echo " -v, --verbose" + echo " Display debugging info." + echo + echo " -a, --algorithm [ algo ]" + echo " Specify GNU Coreutils hash command. Options include: " + echo " b2sum md5sum sha1sum sha224sum sha256sum sha384sum " + echo " sha512sum" + echo + echo " -m, --message [ string ]" + echo " Specify message to be included in filehash files." + echo + echo " -d, --delimiter [ string ]" + echo " Specify character to serve as delimiter in proof files. " + echo " (Default: ,) " + echo " -t, --timestamp" + echo " Create openTimestmaps proof file." + echo " -w, --wait" + echo " Pass '--wait' option to OpenTimestamps to 'wait " + echo " until a complete timestamp committed in the " + echo " Bitcoin blockchain is available instead of " + echo " returning immediately. " + echo " -o, --output" + echo " Specify output directory to save proof. Default is working" + echo " directory where script was run." +} + + +# Check that a string matches one of GNU Coreutils hash commands. +isValidDigestAlgo() { + # Syntax: isValidDigestAlgo [string] + # Description: Returns 1 if [string] is a valid GNU Coreutils hash function command. Returns 0 otherwise. + + # Exit if more than one argument provided. + ###echoerr "DEBUG: Arguments provided to isValidDigestAlgo() are:$@" + if [[ $# -ne 1 ]]; then + echoerr "ERROR: Illegal number of digest arguments provided." + exit 1 + fi + + # Exit if argument contains non-alphanumeric characters + if [[ "$@" =~ [^a-zA-Z0-9$] ]]; then + echoerr "ERROR: Illegal characters for digest argument provided." + exit 1 + fi + + local input=$1 #Save first argument provided to function as input for further processing. + + ###echoerr "DEBUG: Starting function to check validity of string as valid digest command." + + # Define array of valid hash algorithm commands present in by GNU coreutils package. List is current as of Debian Buster. List taken from https://packages.debian.org/buster/amd64/coreutils/filelist ). + local gnuCoreutilsAlgos=(b2sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum) + ###echoerr "DEBUG:Contents of \$gnuCoreutilsAlgos array is:${gnuCoreutilsAlgos[@]}" + + # Make input string lowercase + #input=${input,,} # Make all letters in $1 lowercase (see https://stackoverflow.com/questions/2264428/how-to-convert-a-string-to-lower-case-in-bash ) and save as $input + ###echoerr "DEBUG: User-specified hash algorithm is:$input" + + # Return true (0) if $gnuCoreutilsAlgos array contains string $input (see https://unix.stackexchange.com/a/177139 ) + for item in "${gnuCoreutilsAlgos[@]}"; do + ###echoerr "DEBUG: \$item is:$item" + ###echoerr "DEBUG: \$input is:$input" + if [ "$input" == "$item" ]; then + ###echoerr "DEBUG: $input present in \$gnuCoreutilsAlgos array." + return 0 + fi + done + + # Return false (1) otherwise. + echoerr "DEBUG: $input not recognized as valid hash algorithm name." + return 1 +} + +writeProofs() { + # Syntax: writeProofs [ loopIndex ] [ dirPath ] + + # Description: Construct private and public proofs for files + # contained within [ dirPath ]. Save file with date, [ loopIndex ], + # and base directory name within filename. + + # Required variables: + # - $runDate: used in creating proof directory + # - $PROOF_DIRECTORY: used as directory where proof files are saved + # - $OPTION_ALGORITHM: used to decide which hash algorithm to use + # - $OPTION_DELIMITER: used to determine which char to use as a delimiter in proof files + # - $OPTION_MESSAGE: used to determine message included in proof files + # - $OPTION_RECURSIVE: used to determine if files within subdirectories under [ dir Path ] directory should be included + + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG:This is the writeProofs() function. Provided arguments are:$@" + + # Determine scope of 'find' file search (recursive search or not) + if [[ $OPTION_RECURSIVE == "true" ]]; then + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Performing recursive file search on directory $directoryToHash" + findMaxDepthOption="" # Do not specify -maxdepth option for find (causes 'find' to include all files recursively within [ dirPath ] + else + findMaxDepthOption="-maxdepth 1" # Specify "-maxdepth 1 " as option for 'find' so only files immediately within [ dirPath ] directory are included (no subdirectories). + fi + + # Save current date of loop + proofLoopRunDate=$(date +%Y%m%dT%H%M%S%z) + ###echoerr "DEBUG: Date and time of current loop is:"$proofLoopRunDate + + # Reset arrays. + filePathsFull=() + filePathsIndex=() + fileHashes=() + fileHashNonces=() + fileHashesSalted=() + fileSizes=() + fileModTime=() + + # Pass first argument provided to function as directory index. + local loopIndex=$1 + + # Pass second argument as directory to process + if [[ -d $2 ]]; then + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Valid directory argument provided to writeProofs() function." + local directoryToHash=$2 + ## local directoryToHash=$(readlink -f $2) # Use absolutely full path (not used for privacy). + else + echoerr "ERROR: Invalid directory argument provided to writeProofs() function." + exit 1 + fi + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG:Processing directory:"$directoryToHash + local directoryToHashShort="$(basename "$directoryToHash")" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG:Directory basename:"$directoryToHashShort + + # Populate filePathsFull array with file paths from 'find' results (see https://stackoverflow.com/a/54561526 ). + # Note: For creating array from output of find via readarray, see https://unix.stackexchange.com/a/263885 + # Note: For sorting output of find, see https://unix.stackexchange.com/a/34328 + readarray -d '' filePathsFull < <(find $directoryToHash $findMaxDepthOption -readable -type f -print0 | sort -z) # Populate filePathsFull array with sorted list of files present in $directoryToHash (and subdirectories if $OPTION_RECURSIVE set to 'true'. + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#filePathsFull[@]} elements of filePathsFull array are:${filePathsFull[@]}" # Show array length (see https://www.cyberciti.biz/faq/finding-bash-shell-array-length-elements/ ) + + # Construct index array from filePathsFull array for the for loops used to write proofs. + filePathsIndex=(${!filePathsFull[@]}) + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#filePathsIndex[@]} elements of filePathsIndex array are:${filePathsIndex[@]}" + + # Populate fileHashes with hashes of files listed in filePathsFull array + # Iterate through all elements of filePathsFull array using an integer index. (see https://stackoverflow.com/a/6723516 ) + for i in "${!filePathsIndex[@]}"; do + ###[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG:The $i th element of the filePathsFull array is:${filePathsFull[i]}" + fileHashes[i]=$($OPTION_ALGORITHM "${filePathsFull[i]}" | awk '{ print $1 }') # Get hash value from GNU Coreutils hash command (see https://stackoverflow.com/a/3679803 ) + done + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#fileHashes[@]} elements of fileHashes array are:${fileHashes[@]}" + + # Populate fileHashNonces array with same number of elements as filePathsFull array. Each element is a random hexadecimal nonce of length matching OPTION_ALGORITHM. Hex nonce generated from 64-byte (512-bits) block from /dev/urandom. + for i in "${!filePathsIndex[@]}"; do + fileHashNonces[i]=$(dd bs=64 count=1 if=/dev/urandom 2>/dev/null | $OPTION_ALGORITHM | awk '{ print $1 }') + done + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#fileHashNonces[@]} elements of fileHashNonces array are:${fileHashNonces[@]}" + + # Populate fileHashesSalted array with salted digest for each file hash corresponding nonce. + for i in "${!filePathsIndex[@]}"; do + fileHashesSalted[i]=$(echo -n "${fileHashes[i]}${fileHashNonces[i]}" | $OPTION_ALGORITHM | awk '{ print $1 }') + done + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#fileHashesSalted[@]} elements of fileHashesSalted array are:${fileHashesSalted[@]}" + + # Populate fileSizes array with file sizes (in bytes). + for i in "${!filePathsIndex[@]}"; do + fileSizes[i]=$(wc -c "${filePathsFull[i]}" | awk '{ print $1 }') + done + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#fileSizes[@]} elements of fileSizes array are:${fileSizes[@]}" + + # Populate fileModTime array with file modification times (in ISO-8601 format) + for i in "${!filePathsIndex[@]}"; do + fileModTime[i]=$(date --iso-8601=seconds -r "${filePathsFull[i]}" | awk '{ print $1 }') + done + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#fileModTime[@]} elements of fileModTime array are:${fileModTime[@]}" + + # ==== GENERATE PROOF FILES FROM HASH LIST(S) ==== + + # Define output file names and paths. + local PROOF_PRIVATE_FILENAME=$proofLoopRunDate"_"$loopIndex"_prv.."$scriptHostname"_"$directoryToHashShort"_"$OPTION_ALGORITHM"_proof_private.txt" + local PROOF_PUBLIC_FILENAME=$proofLoopRunDate"_"$loopIndex"_pub.."$scriptHostname"_"$directoryToHashShort"_"$OPTION_ALGORITHM"_proof_public.txt" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PRIVATE_FILENAME is $PROOF_PRIVATE_FILENAME" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PUBLIC_FILENAME is $PROOF_PUBLIC_FILENAME" + PROOF_PRIVATE_FILEPATH="$PROOF_DIRECTORY"/"$PROOF_PRIVATE_FILENAME" + PROOF_PUBLIC_FILEPATH="$PROOF_DIRECTORY"/"$PROOF_PUBLIC_FILENAME" + + # Create private proof and public proof files. + touch "$PROOF_PRIVATE_FILEPATH" + touch "$PROOF_PUBLIC_FILEPATH" + + # Abbreviate OPTION_DELIMITER to local variable DELIM + local DELIM=$OPTION_DELIMITER + + # Define private proof first-line headers (date, version, algorithm, message) + local PROOF_PRIVATE_HEADER1="$proofLoopRunDate$DELIM$SCRIPT_VERSION$DELIM$OPTION_ALGORITHM$DELIM$OPTION_MESSAGE" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PRIVATE_HEADER1 is $PROOF_PRIVATE_HEADER1" + # Define public proof first-line headers (date, version, algorithm, message) + local PROOF_PUBLIC_HEADER1="$proofLoopRunDate$DELIM$SCRIPT_VERSION$DELIM$OPTION_ALGORITHM$DELIM$OPTION_MESSAGE" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PUBLIC_HEADER1 is $PROOF_PUBLIC_HEADER1" + + # Define private proof second-line column labels (index, digest, nonce, salted digest, mdate, size, filepath) + local PROOF_PRIVATE_HEADER2="index"$DELIM"digest"$DELIM"nonce"$DELIM"digest_salted"$DELIM"mdate"$DELIM"size_bytes"$DELIM"file_path" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PRIVATE_HEADER2 is $PROOF_PRIVATE_HEADER2" + # Define public proof second-line column labels (index, salted digest) + local PROOF_PUBLIC_HEADER2="index"$DELIM"digest_salted" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: \$PROOF_PUBLIC_HEADER2 is $PROOF_PUBLIC_HEADER2" + + # Write headers to proof files + echo $PROOF_PRIVATE_HEADER1 >> "$PROOF_PRIVATE_FILEPATH" + echo $PROOF_PRIVATE_HEADER2 >> "$PROOF_PRIVATE_FILEPATH" + echo $PROOF_PUBLIC_HEADER1 >> "$PROOF_PUBLIC_FILEPATH" + echo $PROOF_PUBLIC_HEADER2 >> "$PROOF_PUBLIC_FILEPATH" + + # Loop to append array contents. + for i in "${!filePathsIndex[@]}"; do + # Append to private proof: filePathsIndex[i],fileHashes[i],fileHashNonces[i],fileHashesSalted[i],fileModTime[i],fileSizes[i],filePathsFull[i] + ###[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Writing line to $PROOF_PRIVATE_FILEPATH" + echo ${filePathsIndex[i]}$DELIM${fileHashes[i]}$DELIM${fileHashNonces[i]}$DELIM${fileHashesSalted[i]}$DELIM${fileModTime[i]}$DELIM${fileSizes[i]}$DELIM${filePathsFull[i]} >> "$PROOF_PRIVATE_FILEPATH" + # Append to public proof: filePathsIndex[i],fileHashesSalted[i] + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Writing line to $PROOF_PUBLIC_FILEPATH" + echo ${filePathsIndex[i]}$DELIM${fileHashesSalted[i]} >> $PROOF_PUBLIC_FILEPATH + done + + # Save file paths to global variables prvProofPaths and pubProofPaths to allow other functions to manipulate files. + prvProofPaths+=($PROOF_PRIVATE_FILEPATH) + pubProofPaths+=($PROOF_PUBLIC_FILEPATH) + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#prvProofPaths[@]} elements of prvProofPaths array are:${prvProofPaths[@]}" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: The ${#pubProofPaths[@]} elements of pubProofPaths array are:${pubProofPaths[@]}" + + # Reset arrays. + filePathsFull=() + filePathsIndex=() + fileHashes=() + fileHashNonces=() + fileHashesSalted=() + fileSizes=() + fileModTime=() +} + + +# === INPUT PROCESSING === +# Check for presence of options. +###echoerr "DEBUG: === INPUT PROCESSING START ===" + +# Process initial option arguments (see https://jonalmeida.com/posts/2013/05/26/different-ways-to-implement-flags-in-bash/ and https://likegeeks.com/linux-bash-scripting-awesome-guide-part3/ ) +while [ ! $# -eq 0 ] # While number of arguments ($#) is not (!) equal to (-eq) zero (0). +do + case "$1" in # Check first of remaining arguments to see if it matches one of strings below. + --help | -h) + # Code to run if $1 matched "--help" or "-h": + ###echoerr "DEBUG: This is the help information." + usage # Run usage function to display helpful info to user. + exit 1 + ;; + --recursive | -r) + # Code to run if $1 matched "--recursive" or "-r": + ###echoerr "DEBUG: Recursive option activated." + OPTION_RECURSIVE="true" + ;; + --verbose | -v) + # Code to run if $1 matched "--verbose" or "-v": + ###echoerr "DEBUG: Verbose option activated." + OPTION_VERBOSE="true" + ;; + --algorithm | -a) + # Code to run if $1 matched "--algorithm" or "-a": + + ###echoerr "DEBUG: Selecting hash algorithm." + # Check that OPTION_ALGORITHM variable hasn't already been set. + # Check that OPTION_ALGORITHM contains only alphanumeric characters. + # Check that argument following '-a' or '--algorithm' ($2) is valid algorithm. + if [[ ! -v OPTION_ALGORITHM ]] && [[ ! "$2" =~ [^a-zA-Z0-9$] ]] && isValidDigestAlgo $2; then + ###echoerr "DEBUG: Valid hash algorithm provided." + OPTION_ALGORITHM=$2 # Save argument following '-a' or '--algorithm' to $OPTION_ALGORITHM + shift # Remove an additional argument so the additional algorithm argument $2 is properly removed at the end of the while loop. + else + echoerr "ERROR: Invalid hash algorithm argument provided:""$2" + exit 1 + fi + ;; + --message | -m) + # Code to run if $1 matched "--message" or "-m": + ###echoerr "DEBUG: Specifying header message." + # Check that OPTION_MESSAGE variable hasn't already been set. + if [[ ! -v OPTION_MESSAGE ]]; then + OPTION_MESSAGE=$2 # Save argument following '-m' or '--message' to $OPTION_MESSAGE + shift # Remove an additional argument so the additional algorithm argument $2 is properly removed at the end of the while loop. + fi + ;; + --delimiter | -d) + # Code to run if $1 matched "--delimiter" or "-d": + ###echoerr "DEBUG: Specifying delimiter." + # Check that OPTION_DELIMITER variable hasn't already been set. + if [[ ! -v OPTION_DELIMITER ]]; then + OPTION_DELIMITER=$2 # Save argument following '-d' or '--delimiter' to $OPTION_DELIMITER + shift # Remove an additional argument so the additional delimiter argument $2 is properly removed at the end of the while loop. + fi + ;; + --timestamp | -t) + # Code to run if $1 matched "--timestamp" or "-t": + echoerr "DEBUG: OpenTimestamps option selected." + # Check that ots command exists. (see https://stackoverflow.com/a/677212 ) + if command -v ots >/dev/null 2>&1 ; then + echoerr "DEBUG: OpenTimestampscommand ('ots') detected." + OPTION_OPENTIMESTAMPS="true" + else + echo >&2 "I require the ots (OpenTimestamps) command but it's not installed. Aborting." + exit 1 + fi + ;; + --wait | -w) + # Code to run if $1 matched "--wait" or "-w": + echoerr "DEBUG: OpenTimestamps '--wait' option selected." + OPTION_OPENTIMESTAMPS_WAIT="true" + ;; + --output | -o) + # Code to run if $1 matched "--output" or "-o": + # Check that argument $2 is a directory. + if [[ -d $2 ]]; then + echoerr "DEBUG: Specified ""$PROOF_DIRECTORY_BASE"" in ""$2"" as directory for saving proof files." + PROOF_DIRECTORY="$2"/"$PROOF_DIRECTORY_BASE" + echoerr "DEBUG: Proof Directory is:""$PROOF_DIRECTORY" + shift # Remove an additional argument so the additional delimiter argument $2 is properly removed at the end of the while loop. + else + echoerr "ERROR: Invalid argument provided as output directory for proof files:""$2" + exit 1 + fi + ;; + *) + # Code to run if $1 isn't any of the above cases: + # Check that remaining argument is a directory. + if [[ -d $1 ]]; then + ###echoerr "DEBUG: Remaining argument is directory. Adding to \$directoriesToProcess array." + # Add directory argument $1 to directory array + directoriesToProcess+=($1) # (see https://stackoverflow.com/a/1951523 ) + else + echoerr "ERROR: Remaining argument is not a directory or valid option. Exiting." + exit 1 + fi + ;; + esac + shift # Remove first argument ($1) so remaining arguments can be processed on next loop. +done + +# If OPTION_ALGORITHM is not set then set it to 'sha256sum' by default. +if [[ ! -v OPTION_ALGORITHM ]]; then + echoerr "DEBUG: No hash algorithm set. Defaulting to sha256sum." + OPTION_ALGORITHM="sha256sum" +fi + +# If OPTION_DELIMITER is not set then set it to ',' by default. +if [[ ! -v OPTION_DELIMITER ]]; then + ###echoerr "DEBUG: No delimiter set. Defaulting to ','." + OPTION_DELIMITER="," +fi + +# Exit program if directoriesToProcess array is empty. +if [[ ${#directoriesToProcess[@]} -eq 0 ]]; then + echoerr "ERROR: No valid directory specified." + exit 1 +fi + +###echoerr "DEBUG: === INPUT PROCESSING END ===" + +# === MAIN PROGRAM === +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: === MAIN PROGRAM START ===" +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Verbose option active." +[[ $OPTION_VERBOSE == "true" ]] && [[ $OPTION_RECURSIVE == "true" ]] && echoerr "DEBUG: Recursive option active." +[[ $OPTION_VERBOSE == "true" ]] && [[ -v OPTION_ALGORITHM ]] && echoerr "DEBUG: Selected Hash algorithm is:""$OPTION_ALGORITHM" +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Directories to process are: ${directoriesToProcess[@]}" +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Message to add to proof is: $OPTION_MESSAGE" +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Proof directory is:""$PROOF_DIRECTORY" +[[ $OPTION_VERBOSE == "true" ]] && [[ $OPTION_OPENTIMESTAMPS == "true" ]] && echoerr "DEBUG: OpenTimestamps option active." +[[ $OPTION_VERBOSE == "true" ]] && [[ $OPTION_OPENTIMESTAMPS_WAIT == "true" ]] && echoerr "DEBUG: OpenTimestamps '--wait' option selected." +# Create proof directory +mkdir "$PROOF_DIRECTORY" +echoerr "Proof directory created at:""$PROOF_DIRECTORY" + +## TEMP REF: +# declare -a filePathsFull # array +# declare -a filePathsIndex # array +# declare -a fileHashes # array +# declare -a fileHashNonces # array +# declare -a fileHashesSalted # array +# declare -a fileSizes # array +# declare -a fileModTime # array +# declare -ag prvProofPaths # array, global +# declare -ag pubProofPaths # array, global + +# ==== GENERATE HASH LIST(S) ==== +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: ==== GENERATING HASH LIST(S) ====" + +# Assemble and write proofs to disk for all specified directories. + +# Generate proofs from all provided directories. +for j in "${!directoriesToProcess[@]}"; do + writeProofs $j "${directoriesToProcess[j]}" +done + +# Decide if superproof is required. +if [[ ${#directoriesToProcess[@]} -eq 1 ]]; then + # If only one directory provided then no further proofs required. + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: Only 1 directory processed. No superproof required." +elif [[ ${#directoriesToProcess[@]} -gt 1 ]]; then + # If multiple directories provided then create superproof from files in PROOF_DIRECTORY. + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: More than 1 directory processed. Proceeding to generate superproof from PROOF_DIRECTORY." + writeProofs "S" $PROOF_DIRECTORY # Create proof using proof files which are stored in PROOF_DIRECTORY +fi + +# ==== GENERATE HASH LIST(S) REPORT ===== +echoerr "Proof files written to $PROOF_DIRECTORY directory:" +for i in ${!prvProofPaths[@]}; do + echo $(basename "${prvProofPaths[i]}") + echo $(basename "${pubProofPaths[i]}") +done + +# ==== GENERATE OPENTIMESTAMPS TIMESTAMP FILE(S) ==== + +# Determine if OpenTimestamp actions to be performed. +if [[ $OPTION_OPENTIMESTAMPS == "true" ]]; then + # Identify target for OpenTimeStamps script. + otsTarget=${pubProofPaths[-1]} # Select final element in pubProofPaths array as target for 'ots' command. + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: otsTarget is:""$otsTarget" + + # Set wait option as determined by OPTION_OPENTIMESTAMPS_WAIT variable. Set timeout length. + if [[ $OPTION_OPENTIMESTAMPS_WAIT == "true" ]]; then + otsWaitOption="--wait" # Specify '--wait' option for ots command. + otsWaitTimeCmd="timeout ""$OTS_TIMEOUT" # Specify timeout length (6 hours) for ots command. + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: OpenTimestamps '--wait' option active." + else + otsWaitOption="" + otsWaitTimeCmd="" + [[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: OpenTimestamps '--wait' option not active." + fi + + # Generate OpenTimeStamps proof file. + pushd "$PROOF_DIRECTORY" 1>/dev/null 2>/dev/null # temporarily change working dir to PROOF_DIRECTORY, suppress stdout and stderr + $otsWaitTimeCmd ots $otsWaitOption stamp $otsTarget + echoerr "OpenTimestamps operation successful! $otsTarget created." + echoerr "Use 'ots info [ filename ]' for more info." + echoerr "Use 'ots upgrade' if '--wait' option not used to make .ots file independently-verifiable." + echoerr "See https://opentimestamps.org/ for more information." + popd 1>/dev/null 2>/dev/null # reverse temporary change to working dir, suppress stdout and stderr + +fi + + +[[ $OPTION_VERBOSE == "true" ]] && echoerr "DEBUG: === MAIN PROGRAM END ===" +# Exit program successfully. +exit 0 + + +# == Dependencies == +# +# - bash, find, sort, echo, mkdir, basename, awk, wc, date, dd, +# readlink, touch, ots, + +# - GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later +# This is free software; you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + +# - find (GNU findutils) 4.6.0.225-235f +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Eric B. Decker, James Youngman, and Kevin Dalley. +# Features enabled: D_TYPE O_NOFOLLOW(enabled) LEAF_OPTIMISATION FTS(FTS_CWDFD) CBO(level=2) + +# - sort (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Mike Haertel and Paul Eggert. + +# - echo (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Brian Fox and Chet Ramey. + +# - mkdir (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by David MacKenzie. + +# - basename (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by David MacKenzie. + +# - GNU Awk 4.2.1, API: 2.0 (GNU MPFR 4.0.2, GNU MP 6.1.2) +# Copyright (C) 1989, 1991-2018 Free Software Foundation. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +# - wc (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Paul Rubin and David MacKenzie. + +# - date (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by David MacKenzie. + +# - dd (coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Paul Rubin, David MacKenzie, and Stuart Kemp. + +# - readlink (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Dmitry V. Levin. + +# - touch (GNU coreutils) 8.30 +# Copyright (C) 2018 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. +# Written by Paul Rubin, Arnold Robbins, Jim Kingdon, +# David MacKenzie, and Randy Smith. + +# - ots v0.7.0 ( https://github.com/opentimestamps/opentimestamps-client ) +# The OpenTimestamps Client is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# The OpenTimestamps Client is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +# below for more details. diff --git a/unitproc/bktemplate.sh b/unitproc/bktemplate.sh new file mode 100755 index 0000000..44d3727 --- /dev/null +++ b/unitproc/bktemplate.sh @@ -0,0 +1,379 @@ +#!/bin/bash + +# Date: 2020-05-04T16:50Z +# Author: Steven Baltakatei Sandoval +# Description: Template for bash scripts. +# Note: Use hide-show-block function to aid readability. (ex: https://www.emacswiki.org/emacs/HideShow ). + +#== Variable Initialization == + +#== Global constants == +SCRIPT_TTL=10 # Limit script life to this in seconds. +PATH="/usr/local/bin/:$PATH" # Add user binary executable directory to PATH. +PATH="/opt/bktei:$PATH" # Add 'optional' executable directory to PATH. +SCRIPT_HOSTNAME=$(hostname) # Save hostname of system running this script. +SCRIPT_VERSION="bktemplate.sh 0.0.0" # Define version of script. Used by function 'showVersion'. +SCRIPT_TIME_SHORT="$(date +%Y%m%dT%H%M%S%z)" # Save current date & time in ISO-8601 format (YYYYmmddTHHMMSS+zzzz). +SCRIPT_DATE_SHORT="$(date +%Y%m%d)" # Save current date in ISO-8601 format. + +#== Function Definitions == +echoerr() { + # Usage: echo [ arguments ] + # Description: Prints provided arguments to stderr. + # Input: unspecified + # Output: stderr + # Script function dependencies: none + # External function dependencies: echo + # Last modified: 2020-04-11T21:16Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib: + # [1]: # Roth, James (2010-06-07). ["How to print text to stderr instead of stdout"](https://stackoverflow.com/a/2990533). Licensed CC BY-SA 4.0. + + echo "$@" 1>&2; # Define stderr echo function. See [1]. + return 0; # Function finished. +} # Define stderr message function. +vbm() { + # Usage: vbm "DEBUG:verbose message here" + # Description: Prints verbose message ("vbm") to stderr if OPTION_VERBOSE is set to "true". + # Input: + # - OPTION_VERBOSE variable set by processArguments function. (ex: "true", "false") + # - "$@" positional arguments fed to this function. + # Output: stderr + # Script function dependencies: echoerr + # External function dependencies: echo + # Last modified: 2020-04-11T23:57Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib: + + if [ "$OPTION_VERBOSE" == "true" ]; then + FUNCTION_TIME=$(date --iso-8601=ns); # Save current time in nano seconds. + echoerr "[$FUNCTION_TIME] ""$@"; # Display argument text. + fi + + # End function + return 0; # Function finished. +} # Verbose message display function. +showUsage() { + # Usage: showUsage + # Description: Displays script usage information. + # Input: none + # Output: stderr + # Script function dependencies: echoerr + # External dependencies: bash (5.0.3), echo + # Last modified: 2020-05-04T16:11Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib.: + + echoerr "USAGE:" + echoerr " bktemplate.sh [ options ]" + echoerr + echoerr "OPTIONS:" + echoerr " -h, --help" + echoerr " Display help information." + echoerr + echoerr " --version" + echoerr " Display script version." + echoerr + echoerr " -v, --verbose" + echoerr " Display debugging info." + echoerr + echoerr " -o, --output-file [ file ]" + echoerr " Specify output file." + echoerr + echoerr " -i, --input-file [ file ]" + echoerr " Specify input file." + echoerr + echoerr " -o, --output-dir [ directory ]" + echoerr " Specify output directory." + echoerr + echoerr " -i, --input-dir [ directory ]" + echoerr " Specify input directory." + echoerr + + # End function + return 0; # Function finished. +} # Display information on how to use this script. +showVersion() { + # Usage: showVersion + # Descriptoin: Displays script version and license information. + # Input: unspecified + # Output: stderr + # Script function dependencies: echoerr + # External function dependencies: echo + # Last modified: 2020-04-11T23:57Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib: + + # Initialize function + vbm "DEBUG:showVersion function called." + + # Perform work + OUTPUT="$SCRIPT_VERSION" + + # Display results + echoerr "$OUTPUT"; + + # End function + vbm "DEBUG:showVersion function ended." + return 0; # Function finished. +} # Display script version. +processArguments() { + # Usage: processArguments "$@" + # Description: Processes provided arguments in order to set script option variables useful for + # changing how other functions behave. For example, it may: + # 1. Activate verbose mode + # Input: "$@" (list of arguments provided to the function) + # Output: Sets following variables used by other functions: + # OPTION_VERBOSE Indicates verbose mode enable status. (ex: "true", "false") + # DIROUT1 Path to output directory. + # FILEOUT1 Path to output file. + # DIRIN1 Path to input directory. + # FILEIN1 Path to input file. + # OPTION_FILEOUT1_OVERWRITE Indicates whether file FILEOUT1 should be overwritten (ex: "true", "false') + # Script function dependencies: + # - echoerr Displays messages to stderr. + # - vbm Displays messsages to stderr if OPTION_VERBOSE set to "true". + # External dependencies: bash (5.0.3), echo + # Last modified: 2020-05-04T14:41Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib.: + # [1]: Marco Aurelio (2014-05-08). "echo that outputs to stderr". https://stackoverflow.com/a/23550347 + + # Initialize function + #vbm "DEBUG:processArguments function called." + + # Perform work + while [ ! $# -eq 0 ]; do # While number of arguments ($#) is not (!) equal to (-eq) zero (0). + #1>&2 echo "DEBUG:Starting processArguments while loop." # Debug stderr message. See [1]. + #1>&2 echo "DEBUG:Provided arguments are:""$@" # Debug stderr message. See [1]. + case "$1" in + -h | --help) showUsage; exit 1;; # Display usage. + --version) showVersion; exit 1;; # Show version + -v | --verbose) OPTION_VERBOSE="true"; vbm "DEBUG:Verbose mode enabled.";; # Enable verbose mode. See [1]. + -i | --input-file) # Define input file path + if [ -f "$2" ]; then # If $2 is file that exists, set FILEIN1 to $2, pop $2. + FILEIN1="$2"; + shift; + vbm "DEBUG:Input file FILEIN1 set to:""$2"; + else + echoerr "ERROR: Specified input file does not exist:""$2"; + echoerr "Exiting."; + exit 1; + fi ;; + -I | --input-dir) # Define input directory path + if [ -d "$2" ]; then # If $2 is dir that exists, set DIRIN1 to $2, pop $2. + DIRIN1="$2"; + shift; + vbm "DEBUG:Input directory DIRIN1 set to:""$2"; + else # Display error if $2 is not a valid dir. + echoerr "ERROR:Specified input directory does not exist:""$2"; + echoerr "Exiting."; + exit 1; + fi ;; + -o | --output-file) # Define output file path + if [ -f "$2" ]; then # If $2 is file that exists, prompt user to continue to overwrite, set FILEOUT1 to $2, pop $2. + echoerr "Specified output file $2 already exists. Overwrite? (y/n):" + read m; case $m in + y | Y | yes) OPTION_FILEOUT1_OVERWRITE="true";; + n | N | no) OPTION_FILEOUT1_OVERWRITE="false";; + *) echoerr "Invalid selection. Exiting."; exit 1;; + esac + if [ "$OPTION_FILEOUT1_OVERWRITE" == "true" ]; then + FILEOUT1="$2"; + shift; + vbm "DEBUG:Output file FILEOUT1 set to:""$2"; + else + echoerr "ERORR:Exiting in order to not overwrite output file:""$FILEOUT1"; + exit 1; + fi + else + FILEOUT1="$2"; + shift; + vbm "DEBUG:Output file FILEOUT1 set to:""$2"; + fi ;; + -O | --output-dir) # Define output directory path + if [ -d "$2" ]; then # If $2 is dir that exists, set DIROUT1 to $2, pop $2 + DIROUT1="$2"; + shift; + vbm "DEBUG:Output directory DIROUT1 set to:""$2"; + else + echoerr "ERROR:Specified output directory is not valid:""$2"; + echoerr "Exiting."; + exit 1; + fi ;; + *) echoerr "ERROR: Unrecognized argument."; exit 1;; # Handle unrecognized options. See [1]. + esac + shift + done + + # End function + vbm "DEBUG:processArguments function ended." + return 0; # Function finished. +} # Evaluate script options from positional arguments (ex: $1, $2, $3, etc.). +checkExecutables() { + # Usage: checkExecutables [ command1 ] [ command2 ] [...] [ commandN ] + # Description: Checks that provided commands exist and displays those that do not exist. + # Input: + # - command names (arguments) + # Output: commands that don't exist (stderr) + # Script function dependencies: + # - echoerr for displaying errors via stderr + # - processArguments for setting OPTION_VERBOSE + # - vbm for displaying verbose messages if OPTION_VERBOSE is "true" + # External dependencies: bash (5.0.3), command + # Last modified: 2020-04-11T23:59Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib.: + # [1]: SiegeX (2010-12-12). ["Difference between return and exit in Bash functions."](https://stackoverflow.com/a/4419971). Licensed CC BY-SA 4.0. + # [2]: Darryl Hein (2009-12-23). ["Add a new element to an array without specifying the index in Bash"](https://stackoverflow.com/a/1951523). Licensed CC BY-SA 4.0. + # [3]: kojiro (2012-10-03). ["Convert command line arguments into an array in Bash"](https://stackoverflow.com/a/12711853) + # [4]: niieani (2016-01-12). ["How to copy an array in Bash?"](https://stackoverflow.com/a/34733375/10850071). Licensed CC BY-SA 4.0. + + # Initialize function + vbm "DEBUG:checkExecutables function called." + declare -a candidateCommandsNames # Initialize array for storing positional arguments provided to this function. + candidateCommandsNames=("$@") # Save positional arguments to variable as string. See [3]. + vbm "DEBUG:candidateCommandsNames:""$@" + vbm "DEBUG:candidateCommandsNames[0]:""${candidateCommandsNames[0]}" + vbm "DEBUG:candidateCommandsNames[1]:""${candidateCommandsNames[1]}" + vbm "DEBUG:candidateCommandsNames[2]:""${candidateCommandsNames[2]}" + declare -a outputInvalidCommandsArray # Initialize arary for storing names of invalid commands. + declare -a outputValidCommandsArray # Initialize array for storing names of valid commands. + + # Perform work + for candidateCommandName in "${candidateCommandsNames[@]}"; do # Search through all space-delimited text for valid commands. + if command -v "$candidateCommandName" 1>/dev/null 2>/dev/null; then # Check if a command is valid or not. + outputValidCommandsArray+=("$candidateCommandName") ; # See [2]. + vbm "DEBUG:Adding $candidateCommandName to outputValidCommandsArray." + else + outputInvalidCommandsArray+=("$candidateCommandName") ; # See [2]. + vbm "DEBUG:Adding $candidateCommandName to outputInvalidCommandsArray." + fi + done + + # Output results + if [ ${#outputInvalidCommandsArray[@]} -gt 0 ]; then # if number of elements in outputInvalidCommandsArray greater than 0, then display offending commands and exit 1. + echoerr "ERROR: Invalid commands found:""${outputInvalidCommandsArray[@]}"; # display invalid commands as error + exit 1; # See [1]. + elif [ ${#outputInvalidCommandsArray[@]} -eq 0 ]; then # if number of elements in outputInvalidCommandsArray equals zero, then return 0. + vbm "DEBUG: Valid commands are:""${outputValidCommandsArray[@]}"; # display valid commands if verbose mode enabled + return 0; # See [1]. + else + echoerr "ERROR: Check outputInvalidCommandsArray."; + fi + + # End function + vbm "DEBUG:checkExecutables function ended." + return 0; # Function finished. +} # Check that certain executables exist. +updateTimeConstants() { + # Usage: updateTimeConstants + # Description: Updates time-related variables for use by other scripts. + # Input: (none) + # Output: Sets following variables: + # TIME_CURRENT Current time in long ISO-8601 format. (ex: YYYY-mm-ddTHH:MM:SS+ZZZZ) + # TIME_CURRENT_SHORT Current time in short ISO-8601 format. (ex: YYYYmmddTHHMMSS+ZZZZ) + # DATE_CURRENT Current date in ISO-8601 format. (ex: YYYY-mm-dd) + # DATE_CURRENT_SHORT Current date in short ISO-8601 format. (ex: YYYYmmdd) + # DATE_TOMORROW Tomorrow's date in ISO-8601 format. (ex: YYYY-mm-dd) + # TIME_NEXT_MIDNIGHT Time of tomorrow's midnight in long (ex: YYYY-mm-ddTHH:MM:SS+ZZZZ) + # ISO-861 format. + # SEC_TIL_MIDNIGHT Seconds until next midnight. + # Script function dependencies: + # - echoerr for displaying errors via stderr + # - processArguments for setting OPTION_VERBOSE + # - vbm for displaying verbose messages if OPTION_VERBOSE is "true" + # External dependencies: bash (5.0.3), date, echo + # Last modified: 2020-04-11T23:59Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib.: + + # Initialize function + vbm "DEBUG:updateTimeConstants function called." + + # Perform work + TIME_CURRENT="$(date --iso-8601=seconds)" ; + TIME_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%dT%H%M%S%z)" + DATE_CURRENT="$(date -d "$TIME_CURRENT" --iso-8601=date)" ; + DATE_CURRENT_SHORT="$(date -d "$TIME_CURRENT" +%Y%m%d)" ; + DATE_TOMORROW="$(date -d "$TIME_CURRENT next day" --iso-8601=date)" ; + TIME_NEXT_MIDNIGHT="$(date -d "$DATE_TOMORROW" --iso-8601=seconds)" ; + SECONDS_UNTIL_NEXT_MIDNIGHT="$(( $(date +%s -d "$TIME_NEXT_MIDNIGHT") - $(date +%s -d "$TIME_CURRENT") ))" ; + + # End function + vbm "DEBUG:updateTimeConstants function ended." + return 0; # Function finished. +} # Update time constants + +#== Main Program Definition == +main() { + # Usage: main "$@" # See [1]. + # Input: unspecified (note: all Global Constants declared at beginning of script assumed to be available) + # OPTION_VERBOSE (used by vbm, set by processArguments) (ex: "true", "false") + # Output: unspecified + # Script function dependencies: + # - echoerr for displaying errors via stderr + # - vbm for displaying verbose messages if OPTION_VERBOSE is "true" + # - processArguments for setting OPTION_VERBOSE + # - checkExecutables for checking that specified commands are available + # External dependencies: bash (5.0.3), echo + # Last modified: 2020-05-04T16:50Z + # Last modified by: Steven Baltakatei Sandoval + # License: GPLv3+ + # Ref./Attrib.: + # [1]: ErichBSchulz (2011-11-20). [How to correctly pass bash script arguments to functions](https://stackoverflow.com/a/8198970). Licensed CC BY-SA 4.0. + + # Initialize function + processArguments "$@" # Process arguments. + vbm "DEBUG:main function called." + vbm "DEBUG:main function arguments are:""$@" + checkExecutables "echo" "date" "cut" "awk" # Confirm that executables necessary to run are available. Exit if any fail. + + # Process inputs + if [ -v FILEIN1 ]; then # VERBOSE: Display contents of FILEIN1 if FILEIN1 has been set. + vbm "DEBUG:main function detects input file:""$FILEIN1" ; + vbm "DEBUG:contents of following file is displayed below:""$FILEIN1" ; + if [ "$OPTION_VERBOSE" == "true" ]; then # display FILEIN1 contents via cat if verbose mode specified + echoerr "========""$FILEIN1"" START""========" ; + cat "$FILEIN1" 1>&2 ; + echoerr "========""$FILEIN1"" END""========" ; + fi + fi + + # Perform work + OUTPUT="$OUTPUT""Hello world.\n" + if [ -v FILEIN1 ]; then + OUTPUT="$OUTPUT""The b2sum hash of $FILEIN1 is: $(b2sum "$FILEIN1" | awk '{print $1}').\n"; + fi + OUTPUT="$OUTPUT""The current time is:""$SCRIPT_TIME_SHORT\n" + + # Output results + echo -e "$OUTPUT" + + if [ -v FILEOUT1 ]; then # if FILEOUT1 set, write to this file. + vbm "DEBUG:main function detects output file:""$FILEOUT1" ; + echo -e "$OUTPUT" > "$FILEOUT1" ; + vbm "DEBUG:main funtion wrote OUTPUT to:""$FILEOUT1" ; + elif [ -v DIROUT1 ]; then # else if DIROUT1 set, set FILEOUT1 to timestamped filename, combine into PATHOUT1, write output to PATHOUT1. + vbm "DEBUG:main function detects output directory:""$DIROUT1" ; + FILEOUT1="$SCRIPT_TIME_SHORT"..output + PATHOUT1="$DIROUT1"/"$FILEOUT1" + echo -e "$OUTPUT" > $PATHOUT1 ; + vbm "DEBUG:main function wrote output file to:""$PATHOUT1" ; + fi + + # End function + vbm "DEBUG:main function ended." + return 0; # Function finished. +} + +#== Perform work and exit == +main "$@" # Run main function. +exit 0; -- 2.30.2