feat(user/bk-copy-rand-music):Optimize file processing

- Now ignores blacklist of extensions.

- Increase minimum file duration to 30s.

- Now adds files until 95% of target size or duration is reached
  instead of stopping as soon as a large or long file is considered.

- Now skips `ffprobe` of files (slow) if `du` size check (fast) fails.

diff --git a/user/bk-copy-rand-music b/user/bk-copy-rand-music
index b807286e8b75c30042e0283f59ed7eba51e324ba..2303d50196a76e2b1a7c7a3ef6fa0e22bd81cc8d 100755 (executable)
--- a/user/bk-copy-rand-music
+++ b/user/bk-copy-rand-music
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Desc: Copies random audio files
 # Usage: bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION] ([int BYTES])
 #!/usr/bin/env bash
 # Desc: Copies random audio files
 # Usage: bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION] ([int BYTES])

-# Version: 0.2.0
+# Version: 0.3.0

 # Depends: BK-2020-03: bkshuf v0.1.0
 
 declare -Ag appRollCall # Associative array for storing app status
 # Depends: BK-2020-03: bkshuf v0.1.0
 
 declare -Ag appRollCall # Associative array for storing app status


@@ -11,8 +11,9 @@ declare -a music_codecs # Array for storing valid codec names (e.g. "aac" "mp3")
 
 # Adjustable parameters
 music_codecs=("vorbis" "aac" "mp3" "flac" "opus"); # whitelist of valid codec_names ffprobe might return
 
 # Adjustable parameters
 music_codecs=("vorbis" "aac" "mp3" "flac" "opus"); # whitelist of valid codec_names ffprobe might return

+ext_ignore=".ots\$|.mid\$|.json\$|.gz\$|.jpg\$|.png\$|.asc\$|.pdf\$|.txt\$|.vtt\$|\.SUM|.zip\$|.xz\$|.org\$|.txt\$"; # blacklist of file extensions for 'grep -Evi'

 max_filename_length="255"; # max output filename length
 max_filename_length="255"; # max output filename length

-min_file_duration="10"; # minimum duration per music file
+min_file_duration="30"; # minimum duration per music file

 max_file_duration="3600"; # maximum duration per music file
 min_file_size="100000"; # minimum size per music file (bytes)
 max_file_size="100000000"; # maximum size per music file (bytes)
 max_file_duration="3600"; # maximum duration per music file
 min_file_size="100000"; # minimum size per music file (bytes)
 max_file_size="100000000"; # maximum size per music file (bytes)


@@ -438,7 +439,9 @@ main() {
     # Populate list_files array
     while read -r line; do
         list_files+=("$line");
     # Populate list_files array
     while read -r line; do
         list_files+=("$line");

-    done < <(find -L "$dir_source" -maxdepth "$max_find_depth" -type f | sort);
+    done < <(find -L "$dir_source" -maxdepth "$max_find_depth" -type f | \
+                 grep -Ev "$ext_ignore" | \
+                 sort);

 
     # Test and add random elements of list_files to list_copy
     dur=0; # Initialize duration
 
     # Test and add random elements of list_files to list_copy
     dur=0; # Initialize duration


@@ -449,12 +452,23 @@ main() {
     ## Get element count of list_files array
     file_count="${#list_files[@]}";
     while read -r line && \
     ## Get element count of list_files array
     file_count="${#list_files[@]}";
     while read -r line && \

-            [[ $dur -le $dur_dest ]] && \
-            [[ $siz -le $siz_dest ]] && \
+            [[ $dur -le $((dur_dest * 95 / 100)) ]] && \
+            [[ $siz -le $((siz_dest * 95 / 100)) ]] && \

             [[ $n -le $file_count ]]; do
             [[ $n -le $file_count ]]; do

-        #yell "DEBUG:list_copy building loop:$n";
+        ((n++));
+
+        yell "DEBUG:list_copy building loop:$n/$file_count"; # debug
+        printf "DEBUG:%8d,%8d,%8d/%8d,%8d/%8d\n" "$dur_cand" "$siz_cand" "$dur" "$dur_dest" "$siz" "$siz_dest"; # debug
+

         path_candfile="$line"; # path of candidate file
 
         path_candfile="$line"; # path of candidate file
 

+        ### Check size
+        siz_cand="$(du -b "$path_candfile" | awk '{ print $1 }')"; # size in bytes
+       if ! checkInt "$siz_cand"; then continue; fi; # reject
+        if [[ "$((siz + siz_cand))" -gt "$siz_dest" ]]; then continue; fi; # reject
+       if [[ "$siz_cand" -lt "$min_file_size" ]]; then continue; fi; # reject
+        if [[ "$siz_cand" -gt "$max_file_size" ]]; then continue; fi; # reject
+        

        ### Check if has valid codec
        if ! check_parsable_audio_ffprobe "$path_candfile"; then continue; fi; # reject
        
        ### Check if has valid codec
        if ! check_parsable_audio_ffprobe "$path_candfile"; then continue; fi; # reject
        


@@ -462,31 +476,28 @@ main() {
        file_format="$(get_audio_format "$path_candfile")";
        if ! checkIsInArray "$file_format" "${music_codecs[@]}"; then continue; fi; # reject
 
        file_format="$(get_audio_format "$path_candfile")";
        if ! checkIsInArray "$file_format" "${music_codecs[@]}"; then continue; fi; # reject
 

-       ### Check and save duration
+        ### Check duration

        dur_cand="$(get_media_length "$path_candfile")";
        dur_cand="${dur_cand%%.*}"; # convert float to int
        dur_cand="$(get_media_length "$path_candfile")";
        dur_cand="${dur_cand%%.*}"; # convert float to int

-        if [[ "$((dur + dur_cand))" -gt "$dur_dest" ]]; then break; fi; # no more
-        dur_cand_wnow="$(printf "%s" "$dur_cand" | wc -m)"; # duration width count
-        if [[ $dur_cand_wnow -gt $dur_cand_w ]]; then
-            dur_cand_w="$dur_cand_wnow"; fi;
-       if ! checkInt "$dur_cand"; then continue; fi; # reject
+        if ! checkInt "$dur_cand"; then continue; fi; # reject
+        if [[ "$((dur + dur_cand))" -gt "$dur_dest" ]]; then continue; fi; # reject

        if [[ "$dur_cand" -lt "$min_file_duration" ]]; then continue; fi; # reject
         if [[ "$dur_cand" -gt "$max_file_duration" ]]; then continue; fi; # reject
 
        if [[ "$dur_cand" -lt "$min_file_duration" ]]; then continue; fi; # reject
         if [[ "$dur_cand" -gt "$max_file_duration" ]]; then continue; fi; # reject
 

-        ### Check and save size
-        siz_cand="$(du -b "$path_candfile" | awk '{ print $1 }')"; # size in bytes
-        if [[ "$((siz + siz_cand))" -gt "$siz_dest" ]]; then break; fi; # no more
+        ### Update stats digits widths
+        #### duration
+        dur_cand_wnow="$(printf "%s" "$dur_cand" | wc -m)"; # duration width count
+        if [[ $dur_cand_wnow -gt $dur_cand_w ]]; then
+            dur_cand_w="$dur_cand_wnow"; fi;
+        #### size

         siz_cand_wnow="$(printf "%s" "$siz_cand" | wc -m)"; # size  width count
         if [[ $siz_cand_wnow -gt $siz_cand_w ]]; then
             siz_cand_w="$siz_cand_wnow"; fi;
         siz_cand_wnow="$(printf "%s" "$siz_cand" | wc -m)"; # size  width count
         if [[ $siz_cand_wnow -gt $siz_cand_w ]]; then
             siz_cand_w="$siz_cand_wnow"; fi;

-       if ! checkInt "$siz_cand"; then continue; fi; # reject
-       if [[ "$siz_cand" -lt "$min_file_size" ]]; then continue; fi; # reject
-        if [[ "$siz_cand" -gt "$max_file_size" ]]; then continue; fi; # reject
-
+        

        ### Add/update candfile to array:
         ###   list_copy (array with "duration, size, path")
        #yell "DEBUG:Adding $path_candfile";
        ### Add/update candfile to array:
         ###   list_copy (array with "duration, size, path")
        #yell "DEBUG:Adding $path_candfile";

-        #printf "DEBUG:%8d,%8d,%s\n" "$dur_cand" "$siz_cand" "$path_candfile" 1>&2;
+        printf "DEBUG:%8d,%8d,%s\n" "$dur_cand" "$siz_cand" "$path_candfile" 1>&2;

         #printf "DEBUG:dur:%s\n" "$dur" 1>&2;
         #printf "DEBUG:siz:%s\n" "$siz" 1>&2;
         list_copy+=("$dur_cand,$siz_cand,$path_candfile"); # for copying with order
         #printf "DEBUG:dur:%s\n" "$dur" 1>&2;
         #printf "DEBUG:siz:%s\n" "$siz" 1>&2;
         list_copy+=("$dur_cand,$siz_cand,$path_candfile"); # for copying with order


@@ -494,10 +505,8 @@ main() {
        ### Update total duration $dur and total size $siz
         dur="$((dur + dur_cand))";
         siz="$((siz + siz_cand))";
        ### Update total duration $dur and total size $siz
         dur="$((dur + dur_cand))";
         siz="$((siz + siz_cand))";

-       #yell "DEBUG:dur:$dur";
-       #yell "DEBUG:siz:$siz";
-
-       ((n++));
+       yell "DEBUG:dur:$dur";
+       yell "DEBUG:siz:$siz";

     done < <(printf "%s\n" "${list_files[@]}" | bkshuf);
 
     #yell "DEBUG:BKSHUF_PARAM_LINEC:$BKSHUF_PARAM_LINEC";
     done < <(printf "%s\n" "${list_files[@]}" | bkshuf);
 
     #yell "DEBUG:BKSHUF_PARAM_LINEC:$BKSHUF_PARAM_LINEC";


@@ -531,7 +540,7 @@ main() {
        path_output="$dir_dest"/"$file_name";
        
        ## Copy
        path_output="$dir_dest"/"$file_name";
        
        ## Copy

-       must cp "$fpath" "$path_output" && yell "NOTICE:Copied ($fdur seconds): $fpath ";
+       must cp "$fpath" "$path_output" && yell "NOTICE:Copied ($(printf "%""$dur_cand_w"d "$fdur") seconds): $fpath ";

        #yell "DEBUG:Copied $file_basename to $dur_dest.";
 
        ## Append log
        #yell "DEBUG:Copied $file_basename to $dur_dest.";
 
        ## Append log