feat(unitproc/bktemp-count_jobs):Add count_jobs() function
[BK-2020-03.git] / user / bk-copy-rand-music.sh
1 #!/usr/bin/env bash
2 # Desc: Copies random audio files
3 # Usage: bk-copy-rand-music.sh [dir SOURCE] [dir DEST] [int DURATION]
4 # Version: 0.0.3
5
6 declare -Ag appRollCall # Associative array for storing app status
7 declare -Ag fileRollCall # Associative array for storing file status
8 declare -Ag dirRollCall # Associative array for storing dir status
9 declare -a music_codecs # Array for storing valid codec names (e.g. "aac" "mp3")
10
11 # Adjustable parameters
12 music_codecs=("vorbis" "aac" "mp3" "flac" "opus"); # whitelist of valid codec_names ffprobe might return
13 max_loops="1000000"; # max number of files to test whether are audio or not
14 max_filename_length="255"; # max output filename length
15 min_file_duration="10"; # minimum duration per music file
16
17 yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
18 die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
19 try() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
20 checkapp() {
21 # Desc: If arg is a command, save result in assoc array 'appRollCall'
22 # Usage: checkapp arg1 arg2 arg3 ...
23 # Version: 0.1.1
24 # Input: global assoc. array 'appRollCall'
25 # Output: adds/updates key(value) to global assoc array 'appRollCall'
26 # Depends: bash 5.0.3
27 local returnState
28
29 #===Process Args===
30 for arg in "$@"; do
31 if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
32 appRollCall[$arg]="true";
33 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
34 else
35 appRollCall[$arg]="false"; returnState="false";
36 fi;
37 done;
38
39 #===Determine function return code===
40 if [ "$returnState" = "true" ]; then
41 return 0;
42 else
43 return 1;
44 fi;
45 } # Check that app exists
46 checkfile() {
47 # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
48 # Usage: checkfile arg1 arg2 arg3 ...
49 # Version: 0.1.1
50 # Input: global assoc. array 'fileRollCall'
51 # Output: adds/updates key(value) to global assoc array 'fileRollCall';
52 # Output: returns 0 if app found, 1 otherwise
53 # Depends: bash 5.0.3
54 local returnState
55
56 #===Process Args===
57 for arg in "$@"; do
58 if [ -f "$arg" ]; then
59 fileRollCall["$arg"]="true";
60 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
61 else
62 fileRollCall["$arg"]="false"; returnState="false";
63 fi;
64 done;
65
66 #===Determine function return code===
67 if [ "$returnState" = "true" ]; then
68 return 0;
69 else
70 return 1;
71 fi;
72 } # Check that file exists
73 checkdir() {
74 # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
75 # Usage: checkdir arg1 arg2 arg3 ...
76 # Version 0.1.2
77 # Input: global assoc. array 'dirRollCall'
78 # Output: adds/updates key(value) to global assoc array 'dirRollCall';
79 # Output: returns 0 if all args are dirs; 1 otherwise
80 # Depends: Bash 5.0.3
81 local returnState
82
83 #===Process Args===
84 for arg in "$@"; do
85 if [ -z "$arg" ]; then
86 dirRollCall["(Unspecified Dirname(s))"]="false"; returnState="false";
87 elif [ -d "$arg" ]; then
88 dirRollCall["$arg"]="true";
89 if ! [ "$returnState" = "false" ]; then returnState="true"; fi
90 else
91 dirRollCall["$arg"]="false"; returnState="false";
92 fi
93 done
94
95 #===Determine function return code===
96 if [ "$returnState" = "true" ]; then
97 return 0;
98 else
99 return 1;
100 fi
101 } # Check that dir exists
102 displayMissing() {
103 # Desc: Displays missing apps, files, and dirs
104 # Usage: displayMissing
105 # Version 1.0.0
106 # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
107 # Output: stderr: messages indicating missing apps, file, or dirs
108 # Output: returns exit code 0 if nothing missing; 1 otherwise
109 # Depends: bash 5, checkAppFileDir()
110 local missingApps value appMissing missingFiles fileMissing
111 local missingDirs dirMissing
112
113 #==BEGIN Display errors==
114 #===BEGIN Display Missing Apps===
115 missingApps="Missing apps :";
116 #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
117 for key in "${!appRollCall[@]}"; do
118 value="${appRollCall[$key]}";
119 if [ "$value" = "false" ]; then
120 #echo "DEBUG:Missing apps: $key => $value";
121 missingApps="$missingApps""$key ";
122 appMissing="true";
123 fi;
124 done;
125 if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing.
126 echo "$missingApps" 1>&2;
127 fi;
128 unset value;
129 #===END Display Missing Apps===
130
131 #===BEGIN Display Missing Files===
132 missingFiles="Missing files:";
133 #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
134 for key in "${!fileRollCall[@]}"; do
135 value="${fileRollCall[$key]}";
136 if [ "$value" = "false" ]; then
137 #echo "DEBUG:Missing files: $key => $value";
138 missingFiles="$missingFiles""$key ";
139 fileMissing="true";
140 fi;
141 done;
142 if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing.
143 echo "$missingFiles" 1>&2;
144 fi;
145 unset value;
146 #===END Display Missing Files===
147
148 #===BEGIN Display Missing Directories===
149 missingDirs="Missing dirs:";
150 #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
151 for key in "${!dirRollCall[@]}"; do
152 value="${dirRollCall[$key]}";
153 if [ "$value" = "false" ]; then
154 #echo "DEBUG:Missing dirs: $key => $value";
155 missingDirs="$missingDirs""$key ";
156 dirMissing="true";
157 fi;
158 done;
159 if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing.
160 echo "$missingDirs" 1>&2;
161 fi;
162 unset value;
163 #===END Display Missing Directories===
164
165 #==END Display errors==
166 #==BEGIN Determine function return code===
167 if [ "$appMissing" == "true" ] || [ "$fileMissing" == "true" ] || [ "$dirMissing" == "true" ]; then
168 return 1;
169 else
170 return 0;
171 fi
172 #==END Determine function return code===
173 } # Display missing apps, files, dirs
174 showUsage() {
175 # Desc: Display script usage information
176 # Usage: showUsage
177 # Version 0.0.1
178 # Input: none
179 # Output: stdout
180 # Depends: GNU-coreutils 8.30 (cat)
181 cat <<'EOF'
182
183 DESCRIPTION:
184 This script may be used to copy a random selection of files containing
185 audio tracks from SOURCE to DEST.
186
187 USAGE:
188 bk-copy-rand-music [dir SOURCE] [dir DEST] [int DURATION]
189
190 EXAMPLE:
191 bk-copy-rand-music ~/Music /tmp/music-sample 3600
192
193 DEPENDENCIES:
194 ffprobe
195 GNU Coreutils 8.30
196 EOF
197 } # Display information on how to use this script.
198 check_parsable_audio_ffprobe() {
199 # Desc: Checks if ffprobe returns valid audio codec name for file
200 # Usage: check_parsable_audio_ffprobe [path FILE]
201 # Version: 0.0.1
202 # Input: arg1: file path
203 # Output: exit code 0 if returns valid codec name; 1 otherwise
204 # Depends: ffprobe, die()
205 local file_in ffprobe_out
206
207 if [[ $# -ne 1 ]]; then die "ERROR:Invalid number of args:$#"; fi;
208
209 file_in="$1";
210
211 # Check if ffprobe detects an audio stream
212 if ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in" 1>/dev/null 2>&1; then
213 return_state="true";
214 else
215 return_state="false";
216 fi;
217
218 # Fail if ffprobe returns no result
219 ffprobe_out="$(ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in")";
220 if [[ -z $ffprobe_out ]]; then
221 return_state="false";
222 fi;
223
224 # Report exit code
225 if [[ $return_state = "true" ]]; then
226 return 0;
227 else
228 return 1;
229 fi;
230 } # Checks if file has valid codec name using ffprobe
231 get_audio_format() {
232 # Desc: Gets audio format of file as string
233 # Usage: get_audio_format arg1
234 # Depends: ffprobe
235 # Version: 0.0.1
236 # Input: arg1: input file path
237 # Output: stdout (if valid audio format)
238 # exit code 0 if audio file; 1 otherwise
239 # Example: get_audio_format myvideo.mp4
240 # Note: Would return "opus" if full ffprobe report had 'Audio: opus, 48000 Hz, stereo, fltp'
241 # Note: Not tested with videos containing multiple video streams
242 # Ref/Attrib: [1] https://stackoverflow.com/questions/5618363/is-there-a-way-to-use-ffmpeg-to-determine-the-encoding-of-a-file-before-transcod
243 # [2] https://stackoverflow.com/questions/44123532/how-to-find-out-the-file-extension-for-extracting-audio-tracks-with-ffmpeg-and-p#comment88464070_50723126
244 local audio_format file_in;
245 local return_state;
246 file_in="$1";
247
248 # Return error exit code if not audio file
249 ## Return error if ffprobe itself exited on error
250 if ! ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in" 1>/dev/null 2>&1; then
251 return_state="false";
252 fi;
253
254 # Get audio format
255 audio_format="$(ffprobe -v error -select_streams a -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 "$file_in")"; # see [1]
256
257 ## Return error if audio format is incorrectly formatted (e.g. reject if contains spaces)
258 pattern="^[[:alnum:]]+$"; # alphanumeric string with no spaces
259 if [[ $audio_format =~ $pattern ]]; then
260 return_state="true";
261 # Report audio format
262 echo "$audio_format";
263 else
264 return_state="false";
265 fi;
266
267 # Report exit code
268 if [[ $return_state = "true" ]]; then
269 return 0;
270 else
271 return 1;
272 fi;
273 } # Get audio format as stdout
274 get_media_length() {
275 # Use ffprobe to get media container length in seconds (float)
276 # Usage: get_media_length arg1
277 # Input: arg1: path to file
278 # Output: stdout: seconds (float)
279 # Depends: ffprobe 4.1.8
280 # Ref/Attrib: [1] How to get video duration in seconds? https://superuser.com/a/945604
281 local file_in
282 file_in="$1";
283 if [[ ! -f $file_in ]]; then
284 die "ERROR:Not a file:$file_in";
285 fi;
286 ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$file_in";
287 } # Get media container length in seconds via stdout
288 checkInt() {
289 # Desc: Checks if arg is integer
290 # Usage: checkInt arg
291 # Input: arg: integer
292 # Output: - return code 0 (if arg is integer)
293 # - return code 1 (if arg is not integer)
294 # Example: if ! checkInt $arg; then echo "not int"; fi;
295 # Version: 0.0.1
296 local returnState
297
298 #===Process Arg===
299 if [[ $# -ne 1 ]]; then
300 die "ERROR:Invalid number of arguments:$#";
301 fi;
302
303 RETEST1='^[0-9]+$'; # Regular Expression to test
304 if [[ ! $1 =~ $RETEST1 ]] ; then
305 returnState="false";
306 else
307 returnState="true";
308 fi;
309
310 #===Determine function return code===
311 if [ "$returnState" = "true" ]; then
312 return 0;
313 else
314 return 1;
315 fi;
316 } # Checks if arg is integer
317 checkIsInArray() {
318 # Desc: Checks if input arg is element in array
319 # Usage: checkIsInArray arg1 arg2
320 # Version: 0.0.1
321 # Input: arg1: test string
322 # arg2: array
323 # Output: exit code 0 if test string is in array; 1 otherwise
324 # Example: checkIsInArray "foo" "${myArray[@]}"
325 # Ref/Attrib: [1] How do I check if variable is an array? https://stackoverflow.com/a/27254437
326 # [2] How to pass an array as function argument? https://askubuntu.com/a/674347
327 local return_state input arg1 string_test
328 declare -a arg2 array_test
329 input=("$@") # See [2]
330 arg1="${input[0]}";
331 arg2=("${input[@]:1}");
332 #yell "DEBUG:input:${input[@]}";
333 #yell "DEBUG:arg1:${arg1[@]}";
334 #yell "DEBUG:arg2:${arg2[@]}";
335
336 string_test="$arg1";
337 array_test=("${arg2[@]}");
338
339 #yell "DEBUG:string_test:$string_test";
340 #yell "DEBUG:$(declare -p array_test)";
341 for element in "${array_test[@]}"; do
342 #yell "DEBUG:element:$element";
343 if [[ "$element" =~ ^"$string_test" ]]; then
344 return_state="true";
345 continue;
346 fi;
347 done;
348
349 # Report exit code
350 if [[ $return_state == "true" ]]; then
351 return 0;
352 else
353 return 1;
354 fi;
355 } # Check if string is element in array
356 main() {
357 # Desc: Main program
358 # Input: arg1: path to source tree
359 # arg2: path to destination tree
360 # arg3: cumulative duration (seconds) of audio files in destination tree
361 # assoc arrays: appRollCall, fileRollCall, dirRollCall
362 # Output: [none]
363 # Depends: yell(), checkdir() 0.1.2, displayMissing() 1.0.0, GNU Coreutils 8.30 (shuf)
364 local arg1 arg2 arg3 dur_dest dir_source dir_dest list_all
365 declare -a list_files # array for files to be considered
366 declare -A list_copy # assoc array for files to be copied (key=path; value=duration)
367
368 # Parse args
369 arg1="$1";
370 arg2="$2";
371 arg3="$3";
372 if [[ $# -ne 3 ]]; then showUsage; die "ERROR:Invalid number of args."; fi;
373
374 ## Check duration
375 if checkInt "$arg3"; then
376 dur_dest="$arg3";
377 else
378 yell "ERROR:Duration (seconds) not an int:$arg3"
379 fi;
380
381 ## Check directories
382 if checkdir "$arg1" "$arg2"; then
383 dir_source="$arg1";
384 dir_dest="$arg2";
385 else
386 yell "ERROR:Directory error";
387 fi;
388
389 ## Check apps
390 checkapp ffprobe;
391
392 if ! displayMissing; then
393 showUsage;
394 die "ERROR:Check missing resources.";
395 fi;
396
397 yell "STATUS:Working...";
398
399 # Generate file path list
400 list_all="$(find -L "$dir_source")";
401 #yell "DEBUG:list_files_rel:$list_files_rel";
402
403 # Prune list_all of non-files and save as array list_files
404 while read -r line; do
405 #yell "DEBUG:line:$line";
406 if ! [[ -f $line ]]; then
407 #yell "DEBUG:Not a file:$line";
408 #yell ""; # debug
409 continue;
410 fi;
411 list_files+=("$line");
412 done < <(echo "$list_all");
413
414 # Randomly test and add elements of list_files array to list_copy
415 dur=0; # Initialize duration
416 n=0; # Initialize loop counter
417 ## Get element count of list_files array
418 list_files_count="${#list_files[@]}";
419 while [[ $dur -le $dur_dest ]]; do
420 #yell "DEBUG:list_copy building loop:$n";
421 ### Select random element of list_files array
422 list_files_index="$(shuf -i 1-"$list_files_count" -n1)";
423 list_files_index="$((list_files_index - 1))"; # bash arrays are zero-indexed
424 path_candfile="${list_files[$list_files_index]}"; # path of candidate file
425
426 ### Check if has valid codec
427 if ! check_parsable_audio_ffprobe "$path_candfile"; then continue; fi; # reject
428
429 ### Check if desired codec
430 file_format="$(get_audio_format "$path_candfile")";
431 if ! checkIsInArray "$file_format" "${music_codecs[@]}"; then continue; fi; # reject
432
433 ### Check and save duration
434 dur_cand="$(get_media_length "$path_candfile")";
435 dur_cand="${dur_cand%%.*}"; # convert float to int
436 if ! checkInt "$dur_cand"; then continue; fi; # reject
437 if [[ "$dur_cand" -lt "$min_file_duration" ]]; then continue; fi; # reject
438
439 ### Add/update candfile to list_copy assoc. array (key=path; value=duration)
440 #yell "DEBUG:Adding $path_candfile";
441 list_copy["$path_candfile"]="$dur_cand";
442
443 ### Update total duration $dur by summing all list_copy assoc. array values
444 dur=0;
445 for value in "${list_copy[@]}"; do
446 dur="$((dur + value))";
447 done;
448 #yell "DEBUG:dur:$dur";
449
450 ### Sanity check
451 ((n++));
452 if [[ $n -gt $max_loops ]]; then die "ERROR:Too many loops:$n"; fi;
453 done;
454
455 n=0; # Initialize loop counter
456 # Copy files in list_copy to dir_dest;
457 for key in "${!list_copy[@]}"; do
458 value="${list_copy[$key]}";
459 ## Get basename of path
460 file_basename="$(basename "$key")";
461
462 ## Get 16-character b2sum fingerprint (for different files that share basename)
463 fingerprint="$(b2sum -l64 "$key" | cut -d' ' -f1)";
464
465 ## Form output filename
466 file_name="$fingerprint".."$file_basename";
467 file_name="${file_name:0:$max_filename_length}"; # Limit filename length (e.g. Windows has max of 255 characters)
468
469 ## Form output path
470 path_output="$dir_dest"/"$file_name";
471
472 ## Copy
473 try cp "$key" "$path_output" && yell "NOTICE:Copied ($value seconds): $key ";
474 #yell "DEBUG:Copied $file_basename to $dur_dest.";
475
476 ## Append log
477 path_log_output="$dir_dest"/COPY.log;
478 if [[ $n -le 0 ]]; then
479 echo "fingerprint","duration","original_path" >> "$path_log_output";
480 else
481 echo "$fingerprint","$value","$key" >> "$path_log_output";
482 fi;
483
484 ((n++));
485 unset file_basename path_output
486 done;
487
488 # Report total duration
489 yell "NOTICE:Total duration (seconds):$dur";
490
491 } # Main program
492
493 main "$@";
494
495 # Author: Steven Baltakatei Sandoval
496 # License: GPLv3+