fix(user/bkdatev):Make compatible with Bash 3.2.57
[BK-2020-03.git] / user / bk-copy-rand-music
1 #!/usr/bin/env bash
2 # Desc: Copies random audio files
3 # Usage: bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION] ([int BYTES])
4 # Version: 0.2.0
5 # Depends: BK-2020-03: bkshuf v0.1.0
6
7 declare -Ag appRollCall # Associative array for storing app status
8 declare -Ag fileRollCall # Associative array for storing file status
9 declare -Ag dirRollCall # Associative array for storing dir status
10 declare -a music_codecs # Array for storing valid codec names (e.g. "aac" "mp3")
11
12 # Adjustable parameters
13 music_codecs=("vorbis" "aac" "mp3" "flac" "opus"); # whitelist of valid codec_names ffprobe might return
14 max_filename_length="255"; # max output filename length
15 min_file_duration="10"; # minimum duration per music file
16 max_file_duration="3600"; # maximum duration per music file
17 min_file_size="100000"; # minimum size per music file (bytes)
18 max_file_size="100000000"; # maximum size per music file (bytes)
19 siz_dest="600000000"; # default destination size limit: 600 MB
20 max_find_depth="10"; # max find depth
21
22 # Load env vars (bkshuf defaults for typical music albums)
23 if [[ ! -v BKSHUF_PARAM_LINEC ]]; then export BKSHUF_PARAM_LINEC=1000000; fi;
24 if [[ ! -v BKSHUF_PARAM_GSIZE ]]; then export BKSHUF_PARAM_GSIZE=10; fi;
25
26 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
27 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
28 must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
29 checkapp() {
30 # Desc: If arg is a command, save result in assoc array 'appRollCall'
31 # Usage: checkapp arg1 arg2 arg3 ...
32 # Version: 0.1.1
33 # Input: global assoc. array 'appRollCall'
34 # Output: adds/updates key(value) to global assoc array 'appRollCall'
35 # Depends: bash 5.0.3
36 local returnState
37
38 #===Process Args===
39 for arg in "$@"; do
40 if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
41 appRollCall[$arg]="true";
42 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
43 else
44 appRollCall[$arg]="false"; returnState="false";
45 fi;
46 done;
47
48 #===Determine function return code===
49 if [ "$returnState" = "true" ]; then
50 return 0;
51 else
52 return 1;
53 fi;
54 } # Check that app exists
55 checkfile() {
56 # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
57 # Usage: checkfile arg1 arg2 arg3 ...
58 # Version: 0.1.1
59 # Input: global assoc. array 'fileRollCall'
60 # Output: adds/updates key(value) to global assoc array 'fileRollCall';
61 # Output: returns 0 if app found, 1 otherwise
62 # Depends: bash 5.0.3
63 local returnState
64
65 #===Process Args===
66 for arg in "$@"; do
67 if [ -f "$arg" ]; then
68 fileRollCall["$arg"]="true";
69 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
70 else
71 fileRollCall["$arg"]="false"; returnState="false";
72 fi;
73 done;
74
75 #===Determine function return code===
76 if [ "$returnState" = "true" ]; then
77 return 0;
78 else
79 return 1;
80 fi;
81 } # Check that file exists
82 checkdir() {
83 # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
84 # Usage: checkdir arg1 arg2 arg3 ...
85 # Version 0.1.2
86 # Input: global assoc. array 'dirRollCall'
87 # Output: adds/updates key(value) to global assoc array 'dirRollCall';
88 # Output: returns 0 if all args are dirs; 1 otherwise
89 # Depends: Bash 5.0.3
90 local returnState
91
92 #===Process Args===
93 for arg in "$@"; do
94 if [ -z "$arg" ]; then
95 dirRollCall["(Unspecified Dirname(s))"]="false"; returnState="false";
96 elif [ -d "$arg" ]; then
97 dirRollCall["$arg"]="true";
98 if ! [ "$returnState" = "false" ]; then returnState="true"; fi
99 else
100 dirRollCall["$arg"]="false"; returnState="false";
101 fi
102 done
103
104 #===Determine function return code===
105 if [ "$returnState" = "true" ]; then
106 return 0;
107 else
108 return 1;
109 fi
110 } # Check that dir exists
111 displayMissing() {
112 # Desc: Displays missing apps, files, and dirs
113 # Usage: displayMissing
114 # Version 1.0.0
115 # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
116 # Output: stderr: messages indicating missing apps, file, or dirs
117 # Output: returns exit code 0 if nothing missing; 1 otherwise
118 # Depends: bash 5, checkAppFileDir()
119 local missingApps value appMissing missingFiles fileMissing
120 local missingDirs dirMissing
121
122 #==BEGIN Display errors==
123 #===BEGIN Display Missing Apps===
124 missingApps="Missing apps :";
125 #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
126 for key in "${!appRollCall[@]}"; do
127 value="${appRollCall[$key]}";
128 if [ "$value" = "false" ]; then
129 #echo "DEBUG:Missing apps: $key => $value";
130 missingApps="$missingApps""$key ";
131 appMissing="true";
132 fi;
133 done;
134 if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing.
135 echo "$missingApps" 1>&2;
136 fi;
137 unset value;
138 #===END Display Missing Apps===
139
140 #===BEGIN Display Missing Files===
141 missingFiles="Missing files:";
142 #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
143 for key in "${!fileRollCall[@]}"; do
144 value="${fileRollCall[$key]}";
145 if [ "$value" = "false" ]; then
146 #echo "DEBUG:Missing files: $key => $value";
147 missingFiles="$missingFiles""$key ";
148 fileMissing="true";
149 fi;
150 done;
151 if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing.
152 echo "$missingFiles" 1>&2;
153 fi;
154 unset value;
155 #===END Display Missing Files===
156
157 #===BEGIN Display Missing Directories===
158 missingDirs="Missing dirs:";
159 #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
160 for key in "${!dirRollCall[@]}"; do
161 value="${dirRollCall[$key]}";
162 if [ "$value" = "false" ]; then
163 #echo "DEBUG:Missing dirs: $key => $value";
164 missingDirs="$missingDirs""$key ";
165 dirMissing="true";
166 fi;
167 done;
168 if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing.
169 echo "$missingDirs" 1>&2;
170 fi;
171 unset value;
172 #===END Display Missing Directories===
173
174 #==END Display errors==
175 #==BEGIN Determine function return code===
176 if [ "$appMissing" == "true" ] || [ "$fileMissing" == "true" ] || [ "$dirMissing" == "true" ]; then
177 return 1;
178 else
179 return 0;
180 fi
181 #==END Determine function return code===
182 } # Display missing apps, files, dirs
183 showUsage() {
184 # Desc: Display script usage information
185 # Usage: showUsage
186 # Version 0.0.1
187 # Input: none
188 # Output: stdout
189 # Depends: GNU-coreutils 8.30 (cat)
190 cat <<'EOF'
191
192 DESCRIPTION:
193 This script may be used to copy a random selection of files containing
194 audio tracks from SOURCE to DEST.
195
196 USAGE:
197 bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION] (int BYTES)
198
199 EXAMPLE:
200 bk-copy-rand-music ~/Music /tmp/music-sample 3600
201 bk-copy-rand-music ~/Music /tmp/music-sample 3600 680000000
202
203 DEPENDENCIES:
204 ffprobe
205 GNU Coreutils 8.30
206
207 ENVIRONMENT VARIABLES
208 BKSHUF_PARAM_LINEC (see `bkshuf` in BK-2020-03)
209 BKSHUF_PARAM_GSIZE (see `bkshuf` in BK-2020-03)
210 EOF
211 } # Display information on how to use this script.
212 check_parsable_audio_ffprobe() {
213 # Desc: Checks if ffprobe returns valid audio codec name for file
214 # Usage: check_parsable_audio_ffprobe [path FILE]
215 # Version: 0.0.1
216 # Input: arg1: file path
217 # Output: exit code 0 if returns valid codec name; 1 otherwise
218 # Depends: ffprobe, die()
219 local file_in ffprobe_out
220
221 if [[ $# -ne 1 ]]; then die "ERROR:Invalid number of args:$#"; fi;
222
223 file_in="$1";
224
225 # Check if ffprobe detects an audio stream
226 if ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in" 1>/dev/null 2>&1; then
227 return_state="true";
228 else
229 return_state="false";
230 fi;
231
232 # Fail if ffprobe returns no result
233 ffprobe_out="$(ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in")";
234 if [[ -z $ffprobe_out ]]; then
235 return_state="false";
236 fi;
237
238 # Report exit code
239 if [[ $return_state = "true" ]]; then
240 return 0;
241 else
242 return 1;
243 fi;
244 } # Checks if file has valid codec name using ffprobe
245 get_audio_format() {
246 # Desc: Gets audio format of file as string
247 # Usage: get_audio_format arg1
248 # Depends: ffprobe
249 # Version: 0.0.1
250 # Input: arg1: input file path
251 # Output: stdout (if valid audio format)
252 # exit code 0 if audio file; 1 otherwise
253 # Example: get_audio_format myvideo.mp4
254 # Note: Would return "opus" if full ffprobe report had 'Audio: opus, 48000 Hz, stereo, fltp'
255 # Note: Not tested with videos containing multiple video streams
256 # Ref/Attrib: [1] https://stackoverflow.com/questions/5618363/is-there-a-way-to-use-ffmpeg-to-determine-the-encoding-of-a-file-before-transcod
257 # [2] https://stackoverflow.com/questions/44123532/how-to-find-out-the-file-extension-for-extracting-audio-tracks-with-ffmpeg-and-p#comment88464070_50723126
258 local audio_format file_in;
259 local return_state;
260 file_in="$1";
261
262 # Return error exit code if not audio file
263 ## Return error if ffprobe itself exited on error
264 if ! ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in" 1>/dev/null 2>&1; then
265 return_state="false";
266 fi;
267
268 # Get audio format
269 audio_format="$(ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in")"; # see [1]
270
271 ## Return error if audio format is incorrectly formatted (e.g. reject if contains spaces)
272 pattern="^[[:alnum:]]+$"; # alphanumeric string with no spaces
273 if [[ $audio_format =~ $pattern ]]; then
274 return_state="true";
275 # Report audio format
276 echo "$audio_format";
277 else
278 return_state="false";
279 fi;
280
281 # Report exit code
282 if [[ $return_state = "true" ]]; then
283 return 0;
284 else
285 return 1;
286 fi;
287 } # Get audio format as stdout
288 get_media_length() {
289 # Use ffprobe to get media container length in seconds (float)
290 # Usage: get_media_length arg1
291 # Input: arg1: path to file
292 # Output: stdout: seconds (float)
293 # Depends: ffprobe 4.1.8
294 # Ref/Attrib: [1] How to get video duration in seconds? https://superuser.com/a/945604
295 local file_in
296 file_in="$1";
297 if [[ ! -f $file_in ]]; then
298 die "ERROR:Not a file:$file_in";
299 fi;
300 ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$file_in";
301 } # Get media container length in seconds via stdout
302 checkInt() {
303 # Desc: Checks if arg is integer
304 # Usage: checkInt arg
305 # Input: arg: integer
306 # Output: - return code 0 (if arg is integer)
307 # - return code 1 (if arg is not integer)
308 # Example: if ! checkInt $arg; then echo "not int"; fi;
309 # Version: 0.0.1
310 local returnState
311
312 #===Process Arg===
313 if [[ $# -ne 1 ]]; then
314 die "ERROR:Invalid number of arguments:$#";
315 fi;
316
317 RETEST1='^[0-9]+$'; # Regular Expression to test
318 if [[ ! $1 =~ $RETEST1 ]] ; then
319 returnState="false";
320 else
321 returnState="true";
322 fi;
323
324 #===Determine function return code===
325 if [ "$returnState" = "true" ]; then
326 return 0;
327 else
328 return 1;
329 fi;
330 } # Checks if arg is integer
331 checkIsInArray() {
332 # Desc: Checks if input arg is element in array
333 # Usage: checkIsInArray arg1 arg2
334 # Version: 0.0.1
335 # Input: arg1: test string
336 # arg2: array
337 # Output: exit code 0 if test string is in array; 1 otherwise
338 # Example: checkIsInArray "foo" "${myArray[@]}"
339 # Ref/Attrib: [1] How do I check if variable is an array? https://stackoverflow.com/a/27254437
340 # [2] How to pass an array as function argument? https://askubuntu.com/a/674347
341 local return_state input arg1 string_test
342 declare -a arg2 array_test
343 input=("$@") # See [2]
344 arg1="${input[0]}";
345 arg2=("${input[@]:1}");
346 #yell "DEBUG:input:${input[@]}";
347 #yell "DEBUG:arg1:${arg1[@]}";
348 #yell "DEBUG:arg2:${arg2[@]}";
349
350 string_test="$arg1";
351 array_test=("${arg2[@]}");
352
353 #yell "DEBUG:string_test:$string_test";
354 #yell "DEBUG:$(declare -p array_test)";
355 for element in "${array_test[@]}"; do
356 #yell "DEBUG:element:$element";
357 if [[ "$element" =~ ^"$string_test" ]]; then
358 return_state="true";
359 continue;
360 fi;
361 done;
362
363 # Report exit code
364 if [[ $return_state == "true" ]]; then
365 return 0;
366 else
367 return 1;
368 fi;
369 } # Check if string is element in array
370 main() {
371 # Desc: Main program
372 # Input: arg1: path to source tree
373 # arg2: path to destination tree
374 # arg3: cumulative duration (seconds) of audio files in destination tree
375 # arg4: cumulative size (bytes) of audio files in destination tree (optional)
376 # assoc arrays: appRollCall, fileRollCall, dirRollCall
377 # env.var: BKSHUF_PARAM_LINEC (bkshuf)
378 # BKSHUF_PARAM_GSIZE (bkshuf)
379 # arrays: music_codecs
380 # vars: max_filename_length, min_file_duration, max_file_duration,
381 # min_file_size, max_file_size, siz_dest, max_find_depth
382 # Output: [none]
383 # Depends: yell(), checkdir() 0.1.2, displayMissing() 1.0.0, GNU Coreutils 8.30
384 # BK-2020-03: bkshuf v0.1.0
385 local arg1 arg2 arg3 dur_dest dir_source dir_dest
386 declare -a list_files # array for files to be considered
387 declare -a list_copy # array for files to be copied (string: "$dur,$fsize,$path")
388
389 # Parse args
390 arg1="$1";
391 arg2="$2";
392 arg3="$3";
393 arg4="$4";
394 if ! { [[ $# -eq 3 ]] || [[ $# -eq 4 ]]; }; then
395 showUsage;
396 die "ERROR:Invalid number of args:$#"; fi;
397
398 # Check env vars
399 if ! checkInt "$BKSHUF_PARAM_LINEC"; then
400 die "FATAL:Not an int:BKSHUF_PARAM_LINEC:$BKSHUF_PARAM_LINEC"; fi;
401 if ! checkInt "$BKSHUF_PARAM_GSIZE"; then
402 die "FATAL:Not an int:BKSHUF_PARAM_LINEC:$BKSHUF_PARAM_GSIZE"; fi;
403
404 ## Check duration
405 if checkInt "$arg3"; then
406 dur_dest="$arg3";
407 else
408 die "FATAL:Duration (seconds) not an int:$arg3"
409 fi;
410
411 ## Check size
412 if [[ -n "$arg4" ]]; then
413 if checkInt "$arg4"; then
414 siz_dest="$arg4";
415 else
416 die "FATAL:Size (bytes) not an int:$arg4";
417 fi;
418 fi;
419
420 ## Check directories
421 if checkdir "$arg1" "$arg2"; then
422 dir_source="$arg1";
423 dir_dest="$arg2";
424 else
425 yell "ERROR:Directory error";
426 fi;
427
428 ## Check apps
429 checkapp ffprobe bkshuf;
430
431 if ! displayMissing; then
432 showUsage;
433 die "ERROR:Check missing resources.";
434 fi;
435
436 yell "STATUS:Working...";
437
438 # Populate list_files array
439 while read -r line; do
440 list_files+=("$line");
441 done < <(find -L "$dir_source" -maxdepth "$max_find_depth" -type f | sort);
442
443 # Test and add random elements of list_files to list_copy
444 dur=0; # Initialize duration
445 siz=0; # Initialize size
446 n=0; # Initialize loop counter
447 dur_cand_w=1; # Init duration digit width counter
448 siz_cand_w=1; # Init size digit width counter
449 ## Get element count of list_files array
450 file_count="${#list_files[@]}";
451 while read -r line && \
452 [[ $dur -le $dur_dest ]] && \
453 [[ $siz -le $siz_dest ]] && \
454 [[ $n -le $file_count ]]; do
455 #yell "DEBUG:list_copy building loop:$n";
456 path_candfile="$line"; # path of candidate file
457
458 ### Check if has valid codec
459 if ! check_parsable_audio_ffprobe "$path_candfile"; then continue; fi; # reject
460
461 ### Check if desired codec
462 file_format="$(get_audio_format "$path_candfile")";
463 if ! checkIsInArray "$file_format" "${music_codecs[@]}"; then continue; fi; # reject
464
465 ### Check and save duration
466 dur_cand="$(get_media_length "$path_candfile")";
467 dur_cand="${dur_cand%%.*}"; # convert float to int
468 if [[ "$((dur + dur_cand))" -gt "$dur_dest" ]]; then break; fi; # no more
469 dur_cand_wnow="$(printf "%s" "$dur_cand" | wc -m)"; # duration width count
470 if [[ $dur_cand_wnow -gt $dur_cand_w ]]; then
471 dur_cand_w="$dur_cand_wnow"; fi;
472 if ! checkInt "$dur_cand"; then continue; fi; # reject
473 if [[ "$dur_cand" -lt "$min_file_duration" ]]; then continue; fi; # reject
474 if [[ "$dur_cand" -gt "$max_file_duration" ]]; then continue; fi; # reject
475
476 ### Check and save size
477 siz_cand="$(du -b "$path_candfile" | awk '{ print $1 }')"; # size in bytes
478 if [[ "$((siz + siz_cand))" -gt "$siz_dest" ]]; then break; fi; # no more
479 siz_cand_wnow="$(printf "%s" "$siz_cand" | wc -m)"; # size width count
480 if [[ $siz_cand_wnow -gt $siz_cand_w ]]; then
481 siz_cand_w="$siz_cand_wnow"; fi;
482 if ! checkInt "$siz_cand"; then continue; fi; # reject
483 if [[ "$siz_cand" -lt "$min_file_size" ]]; then continue; fi; # reject
484 if [[ "$siz_cand" -gt "$max_file_size" ]]; then continue; fi; # reject
485
486 ### Add/update candfile to array:
487 ### list_copy (array with "duration, size, path")
488 #yell "DEBUG:Adding $path_candfile";
489 #printf "DEBUG:%8d,%8d,%s\n" "$dur_cand" "$siz_cand" "$path_candfile" 1>&2;
490 #printf "DEBUG:dur:%s\n" "$dur" 1>&2;
491 #printf "DEBUG:siz:%s\n" "$siz" 1>&2;
492 list_copy+=("$dur_cand,$siz_cand,$path_candfile"); # for copying with order
493
494 ### Update total duration $dur and total size $siz
495 dur="$((dur + dur_cand))";
496 siz="$((siz + siz_cand))";
497 #yell "DEBUG:dur:$dur";
498 #yell "DEBUG:siz:$siz";
499
500 ((n++));
501 done < <(printf "%s\n" "${list_files[@]}" | bkshuf);
502
503 #yell "DEBUG:BKSHUF_PARAM_LINEC:$BKSHUF_PARAM_LINEC";
504 #yell "DEBUG:BKSHUF_PARAM_GSIZE:$BKSHUF_PARAM_GSIZE";
505
506 n=0; # Initialize loop counter
507 num_w="$(printf "%s" "${#list_copy[@]}" | wc -m)"; # init file number format
508 num_fmt="%0""$num_w""d";
509 path_log_output="$dir_dest"/COPY.log;
510 printf "num,fingerprint,duration,size,original_path\n" >> "$path_log_output";
511 # Copy files in list_copy to dir_dest;
512 while read -r line; do
513 #yell "DEBUG:line:$line"; # debug
514 fdur="$(printf "%s" "$line" | cut -d',' -f1)";
515 fsize="$(printf "%s" "$line" | cut -d',' -f2)";
516 fpath="$(printf "%s" "$line" | cut -d',' -f3-)";
517 ## Get basename of path
518 file_basename="$(basename "$fpath")";
519 ### Get basename without unprintable non-ASCII characters
520 file_basename_compat="$(printf "%s" "$file_basename" | tr -dc '[:graph:][:space:]' )";
521
522 ## Get 16-character b2sum fingerprint (for different files that share basename)
523 fingerprint="$(b2sum -l32 "$fpath" | awk '{print $1}' )";
524
525 ## Form output filename
526 num="$(printf "$num_fmt" "$n")";
527 file_name="$num"_"$fingerprint".."$file_basename_compat";
528 file_name="${file_name:0:$max_filename_length}"; # Limit filename length (e.g. Windows has max of 255 characters)
529
530 ## Form output path
531 path_output="$dir_dest"/"$file_name";
532
533 ## Copy
534 must cp "$fpath" "$path_output" && yell "NOTICE:Copied ($fdur seconds): $fpath ";
535 #yell "DEBUG:Copied $file_basename to $dur_dest.";
536
537 ## Append log
538 fpath_can="$(readlink -f "$fpath")"; # resolve symlinks to canonical path
539 log_fmt="%s,%s,%""$dur_cand_w""d,%""$siz_cand_w""d,%s\n"; # e.g. "%s,%3d,%5d,%s" if dur_cand_w=3 and siz_cand_w=5
540 printf "$log_fmt" "$num" "$fingerprint" "$fdur" "$fsize" "$fpath_can" >> "$path_log_output";
541
542 ((n++));
543 unset file_basename file_basename_compat path_output;
544 done < <(printf "%s\n" "${list_copy[@]}");
545
546 # Report total duration and size
547 yell "NOTICE:Total duration (seconds):$dur";
548 yell "NOTICE:Total size (bytes):$siz";
549
550 } # Main program
551
552 main "$@";
553
554 # Author: Steven Baltakatei Sandoval
555 # License: GPLv3+
556
557 # bkshuf v0.1.0
558 # Author: Steven Baltakatei Sandoval
559 # License: GPLv3+
560 # URL: https://gitlab.com/baltakatei/baltakatei-exdev/-/blob/b9e8b771e985fcdf26ba8b9ccb8e31b62da757d3/unitproc/bkshuf