user/rsync_tranches.sh

   1 #!/usr/bin/env bash
   2 # Desc: A bash function for splitting up rsync jobs based on size
   3
   4 echo "ERROR:This is a bash function, not a script." 2>&1; exit 1;
   5
   6 function rsync_tranches() {
   7     # Desc: Runs rsync in parallel across different files size ranges
   8     # Example: rsync_tranches -avu --progress --dry-run ./SOURCE/ ./DEST/
   9     # Depends: rsync 3.2.7
  10     # Version: 0.1.0
  11     local -a rsync_opts=();
  12     local source dest;
  13
  14     # Parse arguments until source and destination are found
  15     while [[ $# -gt 0 ]]; do
  16         case "$1" in
  17             # If it's not an option, assume it's the source, then the destination
  18             -*)
  19                 rsync_opts+=("$1");
  20                 shift;
  21                 ;;
  22             *)
  23                 ## If not a file or directory, assume option
  24                 if [[ ! -f "$1" ]] && [[ ! -d "$1" ]]; then
  25                     rsync_opts+=("$1");
  26                     shift;
  27                 fi;
  28                 ## If valid file or directory, assume source or dest path
  29                 if [[ -z "$source" ]]; then
  30                     source="$1";
  31                 else
  32                     dest="$1";
  33                 fi;
  34                 shift;
  35                 ;;
  36         esac;
  37     done;
  38
  39     # Validate that source and destination are set
  40     if [[ -z "$source" ]] || [[ -z "$dest" ]]; then
  41         echo "Error: Source and destination directories must be specified." 1>&2;
  42         return 1;
  43     fi;
  44
  45     # Tranche 1: 0 to 1MiB-1
  46     rsync --min-size='0' --max-size='1MiB-1' "${rsync_opts[@]}" "$source" "$dest";
  47
  48     # Tranche 2: 1MiB to 2MiB-1
  49     rsync --min-size='1MiB' --max-size='2MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  50     sleep 1;
  51
  52     # Tranche 3: 2MiB to 4MiB-1
  53     rsync --min-size='2MiB' --max-size='4MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  54
  55     # Tranche 4: 4MiB to 8MiB-1
  56     rsync --min-size='4MiB' --max-size='8MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  57
  58     # Tranche 5: 8MiB to 16MiB-1
  59     rsync --min-size='8MiB' --max-size='16MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  60
  61     # Tranche 6: 16MiB to 32MiB-1
  62     rsync --min-size='16MiB' --max-size='32MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  63
  64     # Tranche 7: 32MiB to 64MiB-1
  65     rsync --min-size='32MiB' --max-size='64MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  66
  67     # Tranche 8: 64MiB to 128MiB-1
  68     rsync --min-size='64MiB' --max-size='128MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  69
  70     # Tranche 9: 128MiB to 256MiB-1
  71     rsync --min-size='128MiB' --max-size='256MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  72
  73     # Tranche 10: 256MiB to 512MiB-1
  74     rsync --min-size='256MiB' --max-size='512MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  75
  76     # Tranche 11: 512MiB to 1024MiB-1
  77     rsync --min-size='512MiB' --max-size='1024MiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  78
  79     # Tranche 12: Greater than 1024MiB
  80     rsync --min-size='1024MiB' --max-size='8192PiB-1' "${rsync_opts[@]}" "$source" "$dest" &
  81
  82     wait # Wait for all rsync processes to complete
  83 };
  84 export -f rsync_tranches;
  85
  86 # Author: Steven Baltakatei Sandoval
  87 # License: GPLv3+