From: Steven Baltakatei Sandoval Date: Mon, 18 Dec 2023 22:59:31 +0000 (+0000) Subject: feat(user/rsync_tranches.sh):Add bash function for parallel rsync X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/commitdiff_plain/f061ac6b6e47ad2eba92e5eb09687cae055daa02?ds=inline;hp=--cc feat(user/rsync_tranches.sh):Add bash function for parallel rsync --- f061ac6b6e47ad2eba92e5eb09687cae055daa02 diff --git a/user/rsync_traunches.sh b/user/rsync_traunches.sh new file mode 100644 index 0000000..23e5242 --- /dev/null +++ b/user/rsync_traunches.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Desc: A bash function for splitting up rsync jobs based on size + +echo "ERROR:This is a bash function, not a script." 2>&1; exit 1; + +function rsync_tranches() { + # Desc: Runs rsync in parallel across different files size ranges + # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/ + # Depends: rsync 3.2.7 + # Version: 0.0.1 + local -a rsync_opts=(); + local source; + local dest; + + # Parse arguments until source and destination are found + while [[ $# -gt 0 ]]; do + case "$1" in + # If it's not an option, assume it's the source, then the destination + -*) + rsync_opts+=("$1"); + shift; + ;; + *) + if [ -z "$source" ]; then + source="$1"; + else + dest="$1"; + fi; + shift; + ;; + esac; + done; + + # Validate that source and destination are set + if [ -z "$source" ] || [ -z "$dest" ]; then + echo "Error: Source and destination directories must be specified." 1>&2; + return 1; + fi; + + # Tranche 1: 0 to 1MiB-1 + rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 2: 1MiB to 10MiB-1 + rsync --min-size='1MiB' --max-size='10MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 3: 10MiB to 100MiB-1 + rsync --min-size='10MiB' --max-size='100MiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + # Tranche 4: Greater than 100MiB + rsync --min-size='100MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" & + + wait # Wait for all rsync processes to complete +}; +export -f rsync_tranches;