feat(user/rsync_tranches.sh):Add bash function for parallel rsync
[BK-2020-03.git] / user / rsync_traunches.sh
1 #!/usr/bin/env bash
2 # Desc: A bash function for splitting up rsync jobs based on size
3
4 echo "ERROR:This is a bash function, not a script." 2>&1; exit 1;
5
6 function rsync_tranches() {
7 # Desc: Runs rsync in parallel across different files size ranges
8 # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/
9 # Depends: rsync 3.2.7
10 # Version: 0.0.1
11 local -a rsync_opts=();
12 local source;
13 local dest;
14
15 # Parse arguments until source and destination are found
16 while [[ $# -gt 0 ]]; do
17 case "$1" in
18 # If it's not an option, assume it's the source, then the destination
19 -*)
20 rsync_opts+=("$1");
21 shift;
22 ;;
23 *)
24 if [ -z "$source" ]; then
25 source="$1";
26 else
27 dest="$1";
28 fi;
29 shift;
30 ;;
31 esac;
32 done;
33
34 # Validate that source and destination are set
35 if [ -z "$source" ] || [ -z "$dest" ]; then
36 echo "Error: Source and destination directories must be specified." 1>&2;
37 return 1;
38 fi;
39
40 # Tranche 1: 0 to 1MiB-1
41 rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
42
43 # Tranche 2: 1MiB to 10MiB-1
44 rsync --min-size='1MiB' --max-size='10MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
45
46 # Tranche 3: 10MiB to 100MiB-1
47 rsync --min-size='10MiB' --max-size='100MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
48
49 # Tranche 4: Greater than 100MiB
50 rsync --min-size='100MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" &
51
52 wait # Wait for all rsync processes to complete
53 };
54 export -f rsync_tranches;