From: Steven Baltakatei Sandoval <baltakatei@gmail.com>
Date: Mon, 18 Dec 2023 22:59:31 +0000 (+0000)
Subject: feat(user/rsync_tranches.sh):Add bash function for parallel rsync
X-Git-Url: https://zdv2.bktei.com/gitweb/BK-2020-03.git/commitdiff_plain/f061ac6b6e47ad2eba92e5eb09687cae055daa02?ds=inline;hp=--cc

feat(user/rsync_tranches.sh):Add bash function for parallel rsync
---

f061ac6b6e47ad2eba92e5eb09687cae055daa02
diff --git a/user/rsync_traunches.sh b/user/rsync_traunches.sh
new file mode 100644
index 0000000..23e5242
--- /dev/null
+++ b/user/rsync_traunches.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Desc: A bash function for splitting up rsync jobs based on size
+
+echo "ERROR:This is a bash function, not a script." 2>&1; exit 1;
+
+function rsync_tranches() {
+    # Desc: Runs rsync in parallel across different files size ranges
+    # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/
+    # Depends: rsync 3.2.7
+    # Version: 0.0.1
+    local -a rsync_opts=();
+    local source;
+    local dest;
+
+    # Parse arguments until source and destination are found
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            # If it's not an option, assume it's the source, then the destination
+            -*)
+                rsync_opts+=("$1");
+                shift;
+                ;;
+            *)
+                if [ -z "$source" ]; then
+                    source="$1";
+                else
+                    dest="$1";
+                fi;
+                shift;
+                ;;
+        esac;
+    done;
+
+    # Validate that source and destination are set
+    if [ -z "$source" ] || [ -z "$dest" ]; then
+        echo "Error: Source and destination directories must be specified." 1>&2;
+        return 1;
+    fi;
+
+    # Tranche 1: 0 to 1MiB-1
+    rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
+
+    # Tranche 2: 1MiB to 10MiB-1
+    rsync --min-size='1MiB' --max-size='10MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
+
+    # Tranche 3: 10MiB to 100MiB-1
+    rsync --min-size='10MiB' --max-size='100MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
+
+    # Tranche 4: Greater than 100MiB
+    rsync --min-size='100MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" &
+
+    wait # Wait for all rsync processes to complete
+};
+export -f rsync_tranches;