user/rsync_traunches.sh

   1 #!/usr/bin/env bash
   2 # Desc: A bash function for splitting up rsync jobs based on size
   3
   4 echo "ERROR:This is a bash function, not a script." 2>&1; exit 1;
   5
   6 function rsync_tranches() {
   7     # Desc: Runs rsync in parallel across different files size ranges
   8     # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/
   9     # Depends: rsync 3.2.7
  10     # Version: 0.0.1
  11     local -a rsync_opts=();
  12     local source;
  13     local dest;
  14
  15     # Parse arguments until source and destination are found
  16     while [[ $# -gt 0 ]]; do
  17         case "$1" in
  18             # If it's not an option, assume it's the source, then the destination
  19             -*)
  20                 rsync_opts+=("$1");
  21                 shift;
  22                 ;;
  23             *)
  24                 if [ -z "$source" ]; then
  25                     source="$1";
  26                 else
  27                     dest="$1";
  28                 fi;
  29                 shift;
  30                 ;;
  31         esac;
  32     done;
  33
  34     # Validate that source and destination are set
  35     if [ -z "$source" ] || [ -z "$dest" ]; then
  36         echo "Error: Source and destination directories must be specified." 1>&2;
  37         return 1;
  38     fi;
  39
  40     # Tranche 1: 0 to 1MiB-1
  41     rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  42
  43     # Tranche 2: 1MiB to 10MiB-1
  44     rsync --min-size='1MiB' --max-size='10MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  45
  46     # Tranche 3: 10MiB to 100MiB-1
  47     rsync --min-size='10MiB' --max-size='100MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  48
  49     # Tranche 4: Greater than 100MiB
  50     rsync --min-size='100MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  51
  52     wait # Wait for all rsync processes to complete
  53 };
  54 export -f rsync_tranches;