feat(user/bkfeh):Save up to 100MB or 100 sample images if envvar set

[BK-2020-03.git] / user / bkfeh

diff --git a/user/bkfeh b/user/bkfeh
old mode 100644 (file)
new mode 100755 (executable)
index bdc543a..c0410ef
--- a/user/bkfeh
+++ b/user/bkfeh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 #!/usr/bin/env bash

-# Version: 0.0.3
+# Version: 0.1.0

 # Ref/Attrib: [1] Tange, Ole. GNU Parallel with Bash Array. 2019-03-24. https://unix.stackexchange.com/a/508365/411854
 # Depends: GNU Parallel, GNU Bash v5.1.16, feh 3.6.3
 
 # Ref/Attrib: [1] Tange, Ole. GNU Parallel with Bash Array. 2019-03-24. https://unix.stackexchange.com/a/508365/411854
 # Depends: GNU Parallel, GNU Bash v5.1.16, feh 3.6.3
 


@@ -155,33 +155,46 @@ check_depends() {
     if ! checkapp feh parallel; then displayMissing; die "FATAL:Missing apps."; fi;
     return 1;
 }; # check dependencies
     if ! checkapp feh parallel; then displayMissing; die "FATAL:Missing apps."; fi;
     return 1;
 }; # check dependencies

-read_stdin_psarg() {
-    # Desc: Consumes stdin and reads arguments; outputs as stdout lines
+read_stdin() {
+    # Desc: Consumes stdin; outputs as stdout lines

     # Input: stdin (consumes)
     # Input: stdin (consumes)

-    # Input: args

     # Output: stdout (newline delimited)
     # Output: stdout (newline delimited)

-    # Example: read_stdin_psarg "$@"
+    # Example: printf "foo\nbar\n" | read_stdin

     # Depends: GNU bash (version 5.1.16)
     # Version: 0.0.1
     # Depends: GNU bash (version 5.1.16)
     # Version: 0.0.1

-    local input_stdin input_psarg output;
+    local input_stdin output;

 
     # Store stdin
     if [[ -p /dev/stdin ]]; then
         input_stdin="$(cat -)";
 
     # Store stdin
     if [[ -p /dev/stdin ]]; then
         input_stdin="$(cat -)";

-    fi;
-
-    # Store arguments
-    if [[ $# -gt 0 ]]; then
-        input_psarg="$@";
-    fi;
-
-    # Combine as output array elements
+    fi; 
+    
+    # Store as output array elements

     ## Read in stdin
     if [[ -n $input_stdin ]]; then
         while read -r line; do
             output+=("$line");
         done < <(printf "%s\n" "$input_stdin");
     fi;
     ## Read in stdin
     if [[ -n $input_stdin ]]; then
         while read -r line; do
             output+=("$line");
         done < <(printf "%s\n" "$input_stdin");
     fi;

+
+    # Print to stdout
+    printf "%s\n" "${output[@]}";
+}; # read stdin to stdout lines
+read_psarg() {
+    # Desc: Reads arguments; outputs as stdout lines
+    # Input: args
+    # Output: stdout (newline delimited)
+    # Example: read_psarg "$@"
+    # Depends: GNU bash (version 5.1.16)
+    # Version: 0.0.1
+    local input_psarg output;
+    
+    # Store arguments
+    if [[ $# -gt 0 ]]; then
+        input_psarg="$*";
+    fi;
+    
+    # Store as output array elements

     ## Read in positional arguments
     if [[ -n $input_psarg ]]; then
         for arg in "$@"; do
     ## Read in positional arguments
     if [[ -n $input_psarg ]]; then
         for arg in "$@"; do


@@ -191,43 +204,125 @@ read_stdin_psarg() {
 
     # Print to stdout
     printf "%s\n" "${output[@]}";
 
     # Print to stdout
     printf "%s\n" "${output[@]}";

-}; # read stdin and positional argument to stdout lines
-print_filelist() {
+}; # read positional argument to stdout lines
+find_flist() {

     # Desc: print file list to stdout via `find` using script parameters
     # Input: arg1: path to dir
     #        var: find_depth
     #        var: pattern_find_iregex
     #        var: find_size
     if [[ ! -d "$1" ]]; then return 1; fi;
     # Desc: print file list to stdout via `find` using script parameters
     # Input: arg1: path to dir
     #        var: find_depth
     #        var: pattern_find_iregex
     #        var: find_size
     if [[ ! -d "$1" ]]; then return 1; fi;

-    must find "$1" -maxdepth "$find_depth" -type f -iregex "$pattern_find_iregex" -size +"$find_size";
+    must find "$1" -maxdepth "$fdepth" -type f -iregex "$firegex" -size +"$fsize";

 }; # print file list to stdout from dir with script parameters
 }; # print file list to stdout from dir with script parameters

+save_sample() {
+    # Usage: save_sample arg1
+    # Input: arg1   list_paths        (list of files to take samples from)
+    #        envvar BKFEH_SAMPLE_DIR  (environment variable set outside of this script)
+    # Depends: yell(), GNU Parallel, GNU find, GNU Coreutils 8.32 (cut, find, du)
+    local list_paths
+    sample_count="100";
+    sample_max_space="100000000"; # bytes
+    
+    if [[ ! -v BKFEH_SAMPLE_DIR ]]; then return 0; fi; # return early if environment var not set.
+    
+    if [[ ! -z "$1" ]]; then
+        list_paths="$1"; # newline-delimited list of file paths to sample from
+    else
+        yell "ERROR:NO paths available to sample.";
+    fi;
+    
+    if [[ -d "$BKFEH_SAMPLE_DIR" ]]; then
+        sample_dir="$BKFEH_SAMPLE_DIR";
+        yell "STATUS:Environment variable BKFEH_SAMPLE_DIR set. Clearing and saving samples...";
+        
+        ## clear previous sample
+        count_samples="$(find $BKFEH_SAMPLE_DIR -maxdepth 1 -type f | wc -l)";
+        find "$BKFEH_SAMPLE_DIR" -maxdepth 1 -type f -exec rm '{}' \; ;
+        
+        ## save random sample
+        yell "STATUS:Saving random sample of size $sample_count to $BKFEH_SAMPLE_DIR...";
+        list_paths_sample="$(echo "$list_paths" | shuf | head -n"$sample_count")";
+        while read -r line; do
+            if [[ -z "$line" ]]; then continue; fi;
+            ### check size limit
+            sample_act_space="$(du -bd1 "$BKFEH_SAMPLE_DIR" | cut -f1 )"; # actual used space
+            cand_space="$(du -bd1 "$line" | cut -f1 )"; # size of candidate file to add
+            sample_req_space="$((sample_act_space + cand_space))";
+            if [[ "$sample_req_space" -lt "$sample_max_space" ]]; then
+                #### add file to sample dir
+                cp -n "$line" "$BKFEH_SAMPLE_DIR" ;
+            fi;
+        done < <( echo "$list_paths_sample" );
+    else
+        yell "ERROR:Does not exist: $BKFEH_SAMPLE_DIR";
+    fi;    
+}; # save sample of files
+

 main() {
     # Depends: read_stdin_psarg() v0.0.1, check_depends()
 main() {
     # Depends: read_stdin_psarg() v0.0.1, check_depends()

-    declare -a main_dirs;
+    local re_dotfile;
+    declare -a dirs_stdin dirs_psarg;

     declare -a paths_images;
     declare list_paths_images;
     check_depends;
 
     declare -a paths_images;
     declare list_paths_images;
     check_depends;
 

-    # Find settings
-    find_depth=10; # default: 10
-    find_size="10k"; # default: minimum "10k"
-    #Find files ending in .jpg, .gif, etc.
-    pattern_find_iregex=".+\(jpg\|jpeg\|gif\|png\|webm\)$"; # update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u`
-    export find_depth find_size pattern_find_iregex; # export for parallel
-
-    #Populate main_dirs array
-    ## Read stdin and positional arguments as lines
+    #Populate dirs_stdin and dirs_psarg arrays
+    ## Read stdin as lines
+    re_dotfile="^\."; # first char is a dot
+    while read -r line; do
+        line="$(readlink -e "$line")";
+        # Check if dir
+        if [[ ! -d "$line" ]]; then
+            echo "ERROR:Not a dir:$line" 1>&2;
+            continue;
+        fi;
+        dir_name="$(basename "$line")";
+        # Exclude dotdirs
+        if [[ "$dir_name" =~ $re_dotfile ]]; then
+            echo "ERROR:Is a dotdir:$line" 1>&2;
+            continue
+        fi;
+        dirs_stdin+=("$line");
+    done < <(read_stdin);
+    ## Read positional arguments as lines
+    re_dotfile="^\."; # first char is a dot

     while read -r line; do
     while read -r line; do

+        line="$(readlink -e "$line")";
+        # Check if dir

         if [[ ! -d "$line" ]]; then
             echo "ERROR:Not a dir:$line" 1>&2;
             continue;
         fi;
         if [[ ! -d "$line" ]]; then
             echo "ERROR:Not a dir:$line" 1>&2;
             continue;
         fi;

-        main_dirs+=("$line");
-    done < <(read_stdin_psarg "$@");
+        dir_name="$(basename "$line")";
+        # Exclude dotdirs
+        if [[ "$dir_name" =~ $re_dotfile ]]; then
+            echo "ERROR:Is a dotdir:$line" 1>&2;
+            continue
+        fi;
+        dirs_psarg+=("$line");
+    done < <(read_psarg "$@");
+    
+    # Catch all arrays empty
+    if [[ "${#dirs_stdin[@]}" -le 0 ]] && [[ "${#dirs_psarg[@]}" -le 0 ]]; then
+        die "FATAL:No valid directories provided.";
+    fi;

     
     # Generate file list
     
     # Generate file list

-    paths_images+=("$( parallel print_filelist {} ::: "${main_dirs[@]}" )"); # See [1]
-
+    # Find settings
+    firegex=".+\(jpg\|jpeg\|gif\|png\|webm\)$"; # update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u`
+    fsize="10k"; # default: minimum "10k"
+    export firegex fsize ; # export for parallel
+    ## Call find_filelist() in parallel for positional argument input
+    if [[ "${#dirs_psarg[@]}" -gt 0 ]]; then
+        fdepth=10; export fdepth; # 10 for dirs from positional arguments
+        paths_images+=("$( parallel find_flist {} "$fdepth" "$firegex" "$fsize" ::: "${dirs_psarg[@]}" )"); # See [1]
+    fi;
+    ## Call find_filelist() in parallel for stdin input
+    if [[ "${#dirs_stdin[@]}" -gt 0 ]]; then
+        fdepth=1; export fdepth; # 1 ofr dirs from stdin
+        paths_images+=("$( parallel find_flist {} "$fdepth" "$firegex" "$fsize" ::: "${dirs_stdin[@]}" )"); # See [1]
+    fi;
+    

     # Convert paths_images array into file list
     for i in "${!paths_images[@]}"; do
         list_paths_images="$(printf "%s\n%s" "${paths_images[$i]}" "$list_paths_images")";
     # Convert paths_images array into file list
     for i in "${!paths_images[@]}"; do
         list_paths_images="$(printf "%s\n%s" "${paths_images[$i]}" "$list_paths_images")";


@@ -236,13 +331,18 @@ main() {
         file_count="$(wc -l < <(echo -n "$list_paths_images"))";
         echo "$DEBUG:file_count:$file_count"
     done;
         file_count="$(wc -l < <(echo -n "$list_paths_images"))";
         echo "$DEBUG:file_count:$file_count"
     done;

-
-    # Sort and remove duplicates
+    
+    # Sort, remove duplicate paths

     list_paths_images="$(echo "$list_paths_images" | sort -u | tr -s '\n')";
 
     list_paths_images="$(echo "$list_paths_images" | sort -u | tr -s '\n')";
 

+    # Remove paths with dotfiles
+    list_paths_images="$(echo "$list_paths_images" | grep -viE "/\." )";
+

     # Write
     list_paths_images_tmp="/dev/shm/$(date +%Y%m%dT%H%M%S.%N%z)"..feh_paths.txt;
     must echo -n "$list_paths_images" > "$list_paths_images_tmp";
     # Write
     list_paths_images_tmp="/dev/shm/$(date +%Y%m%dT%H%M%S.%N%z)"..feh_paths.txt;
     must echo -n "$list_paths_images" > "$list_paths_images_tmp";

+    ## Save sample to path in env. var. BKFEH_SAMPLE_DIR if set
+    save_sample "$list_paths_images";

 
     # Print stats
     yell "STATUS:Built file list in $SECONDS seconds.";
 
     # Print stats
     yell "STATUS:Built file list in $SECONDS seconds.";


@@ -253,7 +353,7 @@ main() {
     # Cleanup
     must rm "$list_paths_images_tmp";
 };
     # Cleanup
     must rm "$list_paths_images_tmp";
 };

-export -f yell die must read_stdin_psarg print_filelist;
+export -f yell die must read_stdin read_psarg find_flist;

 #==END Define local functions==
 
 main "$@";
 #==END Define local functions==
 
 main "$@";