feat(user/ffmpeg_parallel_encode.sh):Add parallel | ffmpeg script
[BK-2020-03.git] / user / bkfeh
CommitLineData
9eaab35f 1#!/usr/bin/env bash
acaa0f3d 2# Desc: Wrapper for feh that accepts directory paths via posargs or stdin lines.
82fabd02 3# Version: 0.3.0
9eaab35f 4# Ref/Attrib: [1] Tange, Ole. GNU Parallel with Bash Array. 2019-03-24. https://unix.stackexchange.com/a/508365/411854
82fabd02 5# Depends: GNU Parallel, GNU Bash v5.1.16, feh 3.6.3, GNU Coreutils 8.32 (b2sum)
9eaab35f
SBS
6
7#===Declare local functions===
8yell() { echo "$0: $*" >&2; } # print script path and all args to stderr
9die() { yell "$*"; exit 111; } # same as yell() but non-zero exit status
10must() { "$@" || die "cannot $*"; } # runs args as command, reports args if command fails
11checkapp() {
12 # Desc: If arg is a command, save result in assoc array 'appRollCall'
13 # Usage: checkapp arg1 arg2 arg3 ...
14 # Version: 0.1.1
15 # Input: global assoc. array 'appRollCall'
16 # Output: adds/updates key(value) to global assoc array 'appRollCall'
17 # Depends: bash 5.0.3
18 local returnState
19
20 #===Process Args===
21 for arg in "$@"; do
22 if command -v "$arg" 1>/dev/null 2>&1; then # Check if arg is a valid command
23 appRollCall[$arg]="true";
24 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
25 else
26 appRollCall[$arg]="false"; returnState="false";
27 fi;
28 done;
29
30 #===Determine function return code===
31 if [ "$returnState" = "true" ]; then
32 return 0;
33 else
34 return 1;
35 fi;
36} # Check that app exists
37checkfile() {
38 # Desc: If arg is a file path, save result in assoc array 'fileRollCall'
39 # Usage: checkfile arg1 arg2 arg3 ...
40 # Version: 0.1.1
41 # Input: global assoc. array 'fileRollCall'
42 # Output: adds/updates key(value) to global assoc array 'fileRollCall';
43 # Output: returns 0 if app found, 1 otherwise
44 # Depends: bash 5.0.3
45 local returnState
46
47 #===Process Args===
48 for arg in "$@"; do
49 if [ -f "$arg" ]; then
50 fileRollCall["$arg"]="true";
51 if ! [ "$returnState" = "false" ]; then returnState="true"; fi;
52 else
53 fileRollCall["$arg"]="false"; returnState="false";
54 fi;
55 done;
56
57 #===Determine function return code===
58 if [ "$returnState" = "true" ]; then
59 return 0;
60 else
61 return 1;
62 fi;
63} # Check that file exists
64checkdir() {
65 # Desc: If arg is a dir path, save result in assoc array 'dirRollCall'
66 # Usage: checkdir arg1 arg2 arg3 ...
67 # Version 0.1.1
68 # Input: global assoc. array 'dirRollCall'
69 # Output: adds/updates key(value) to global assoc array 'dirRollCall';
70 # Output: returns 0 if app found, 1 otherwise
71 # Depends: Bash 5.0.3
72 local returnState
73
74 #===Process Args===
75 for arg in "$@"; do
76 if [ -d "$arg" ]; then
77 dirRollCall["$arg"]="true";
78 if ! [ "$returnState" = "false" ]; then returnState="true"; fi
79 else
80 dirRollCall["$arg"]="false"; returnState="false";
81 fi
82 done
83
84 #===Determine function return code===
85 if [ "$returnState" = "true" ]; then
86 return 0;
87 else
88 return 1;
89 fi
90} # Check that dir exists
91displayMissing() {
92 # Desc: Displays missing apps, files, and dirs
93 # Usage: displayMissing
94 # Version 0.1.1
95 # Input: associative arrays: appRollCall, fileRollCall, dirRollCall
96 # Output: stderr: messages indicating missing apps, file, or dirs
97 # Depends: bash 5, checkAppFileDir()
98 local missingApps value appMissing missingFiles fileMissing
99 local missingDirs dirMissing
100
101 #==BEGIN Display errors==
102 #===BEGIN Display Missing Apps===
103 missingApps="Missing apps :";
104 #for key in "${!appRollCall[@]}"; do echo "DEBUG:$key => ${appRollCall[$key]}"; done
105 for key in "${!appRollCall[@]}"; do
106 value="${appRollCall[$key]}";
107 if [ "$value" = "false" ]; then
108 #echo "DEBUG:Missing apps: $key => $value";
109 missingApps="$missingApps""$key ";
110 appMissing="true";
111 fi;
112 done;
113 if [ "$appMissing" = "true" ]; then # Only indicate if an app is missing.
114 echo "$missingApps" 1>&2;
115 fi;
116 unset value;
117 #===END Display Missing Apps===
118
119 #===BEGIN Display Missing Files===
120 missingFiles="Missing files:";
121 #for key in "${!fileRollCall[@]}"; do echo "DEBUG:$key => ${fileRollCall[$key]}"; done
122 for key in "${!fileRollCall[@]}"; do
123 value="${fileRollCall[$key]}";
124 if [ "$value" = "false" ]; then
125 #echo "DEBUG:Missing files: $key => $value";
126 missingFiles="$missingFiles""$key ";
127 fileMissing="true";
128 fi;
129 done;
130 if [ "$fileMissing" = "true" ]; then # Only indicate if an app is missing.
131 echo "$missingFiles" 1>&2;
132 fi;
133 unset value;
134 #===END Display Missing Files===
135
136 #===BEGIN Display Missing Directories===
137 missingDirs="Missing dirs:";
138 #for key in "${!dirRollCall[@]}"; do echo "DEBUG:$key => ${dirRollCall[$key]}"; done
139 for key in "${!dirRollCall[@]}"; do
140 value="${dirRollCall[$key]}";
141 if [ "$value" = "false" ]; then
142 #echo "DEBUG:Missing dirs: $key => $value";
143 missingDirs="$missingDirs""$key ";
144 dirMissing="true";
145 fi;
146 done;
147 if [ "$dirMissing" = "true" ]; then # Only indicate if an dir is missing.
148 echo "$missingDirs" 1>&2;
149 fi;
150 unset value;
151 #===END Display Missing Directories===
152
153 #==END Display errors==
154} # Display missing apps, files, dirs
155check_depends() {
82fabd02
SBS
156 if ! checkapp feh parallel bkshuf b2sum; then
157 displayMissing;
158 die "FATAL:Missing apps.";
159 fi;
9eaab35f
SBS
160 return 1;
161}; # check dependencies
acaa0f3d
SBS
162checkInt() {
163 # Desc: Checks if arg is integer
164 # Usage: checkInt arg
165 # Input: arg: integer
166 # Output: - return code 0 (if arg is integer)
167 # - return code 1 (if arg is not integer)
168 # Example: if ! checkInt $arg; then echo "not int"; fi;
169 # Version: 0.0.1
170 local returnState
171
172 #===Process Arg===
173 if [[ $# -ne 1 ]]; then
174 die "ERROR:Invalid number of arguments:$#";
175 fi;
176
177 RETEST1='^[0-9]+$'; # Regular Expression to test
178 if [[ ! $1 =~ $RETEST1 ]] ; then
179 returnState="false";
180 else
181 returnState="true";
182 fi;
183
184 #===Determine function return code===
185 if [ "$returnState" = "true" ]; then
186 return 0;
187 else
188 return 1;
189 fi;
190} # Checks if arg is integer
335b71e1
SBS
191read_stdin() {
192 # Desc: Consumes stdin; outputs as stdout lines
9eaab35f 193 # Input: stdin (consumes)
9eaab35f 194 # Output: stdout (newline delimited)
335b71e1 195 # Example: printf "foo\nbar\n" | read_stdin
9eaab35f 196 # Depends: GNU bash (version 5.1.16)
335b71e1
SBS
197 # Version: 0.0.1
198 local input_stdin output;
9eaab35f
SBS
199
200 # Store stdin
201 if [[ -p /dev/stdin ]]; then
202 input_stdin="$(cat -)";
335b71e1 203 fi;
966a8d11 204
335b71e1 205 # Store as output array elements
9eaab35f
SBS
206 ## Read in stdin
207 if [[ -n $input_stdin ]]; then
208 while read -r line; do
209 output+=("$line");
210 done < <(printf "%s\n" "$input_stdin");
211 fi;
335b71e1
SBS
212
213 # Print to stdout
214 printf "%s\n" "${output[@]}";
215}; # read stdin to stdout lines
216read_psarg() {
217 # Desc: Reads arguments; outputs as stdout lines
218 # Input: args
219 # Output: stdout (newline delimited)
220 # Example: read_psarg "$@"
221 # Depends: GNU bash (version 5.1.16)
222 # Version: 0.0.1
223 local input_psarg output;
224
225 # Store arguments
226 if [[ $# -gt 0 ]]; then
227 input_psarg="$*";
228 fi;
229
230 # Store as output array elements
9eaab35f
SBS
231 ## Read in positional arguments
232 if [[ -n $input_psarg ]]; then
233 for arg in "$@"; do
234 output+=("$arg");
235 done;
236 fi;
237
238 # Print to stdout
239 printf "%s\n" "${output[@]}";
335b71e1
SBS
240}; # read positional argument to stdout lines
241find_flist() {
9eaab35f
SBS
242 # Desc: print file list to stdout via `find` using script parameters
243 # Input: arg1: path to dir
244 # var: find_depth
245 # var: pattern_find_iregex
246 # var: find_size
247 if [[ ! -d "$1" ]]; then return 1; fi;
335b71e1 248 must find "$1" -maxdepth "$fdepth" -type f -iregex "$firegex" -size +"$fsize";
9eaab35f 249}; # print file list to stdout from dir with script parameters
4db78240
SBS
250save_sample() {
251 # Usage: save_sample arg1
252 # Input: arg1 list_paths (list of files to take samples from)
253 # envvar BKFEH_SAMPLE_DIR (environment variable set outside of this script)
acaa0f3d 254 # envvar BKFEH_SAMPLE_SIZE (space limit for sample dir files)
82fabd02
SBS
255 # Depends: GNU Parallel, GNU find, GNU Coreutils 8.32 (cut, find, du)
256 # BK-2020-03: bkshuf (0.0.1), yell()
4db78240 257 local list_paths
acaa0f3d
SBS
258 sample_count="100"; # max number of images to put in sample dir
259 sample_max_space="10000000"; # max bytes to put in sample dir
260
261 # Load environment variables if set
4db78240 262 if [[ ! -v BKFEH_SAMPLE_DIR ]]; then return 0; fi; # return early if environment var not set.
acaa0f3d
SBS
263 if [[ -v BKFEH_SAMPLE_SIZE ]] && checkInt "$BKFEH_SAMPLE_SIZE"; then
264 sample_max_space="$BKFEH_SAMPLE_SIZE";
265 fi;
266 if [[ -v BKFEH_SAMPLE_COUNT ]] && checkInt "$BKFEH_SAMPLE_COUNT"; then
267 sample_count="$BKFEH_SAMPLE_COUNT";
268 fi;
4db78240 269
82fabd02 270 if [[ -n "$1" ]]; then
4db78240
SBS
271 list_paths="$1"; # newline-delimited list of file paths to sample from
272 else
273 yell "ERROR:NO paths available to sample.";
274 fi;
275
276 if [[ -d "$BKFEH_SAMPLE_DIR" ]]; then
82fabd02 277 #sample_dir="$BKFEH_SAMPLE_DIR";
4db78240
SBS
278 yell "STATUS:Environment variable BKFEH_SAMPLE_DIR set. Clearing and saving samples...";
279
280 ## clear previous sample
82fabd02
SBS
281 count_prev_samples="$(find "$BKFEH_SAMPLE_DIR" -maxdepth 1 -type f | wc -l)";
282 yell "STATUS:Deleting $count_prev_samples previous samples...";
4db78240
SBS
283 find "$BKFEH_SAMPLE_DIR" -maxdepth 1 -type f -exec rm '{}' \; ;
284
285 ## save random sample
286 yell "STATUS:Saving random sample of size $sample_count to $BKFEH_SAMPLE_DIR...";
82fabd02
SBS
287 list_paths_sample="$(echo "$list_paths" | bkshuf "$sample_count" | head -n"$sample_count")";
288 n_samp=0; # init sample file counter
289 sample_log="$BKFEH_SAMPLE_DIR"/paths.txt;
290 printf "%s,%s,%s\n" "n_samp" "file_hash" "file_path" >> "$sample_log";
4db78240
SBS
291 while read -r line; do
292 if [[ -z "$line" ]]; then continue; fi;
293 ### check size limit
294 sample_act_space="$(du -bd1 "$BKFEH_SAMPLE_DIR" | cut -f1 )"; # actual used space
295 cand_space="$(du -bd1 "$line" | cut -f1 )"; # size of candidate file to add
296 sample_req_space="$((sample_act_space + cand_space))";
82fabd02
SBS
297
298 ### Customize file names
299 n_samp_w="$(printf "%s" "$sample_count" | wc -c)";
300 n_samp_fmt="%0""$n_samp_w""d";
301 n_samp_dd="$(printf "$n_samp_fmt" "$n_samp")"; # sample number fixed-width
302 file_path="$line";
303 #file_dir="$(dirname "$line")";
304 file_name="$(basename "$line")";
305 file_hash="$(b2sum -l32 "$line" | awk '{print $1}')"; # use file hash to avoid clobbering
306 file_ext="${file_name##*.}";
307 file_name="${file_name%.*}";
308 file_shortname="${file_name:0:32}";
309 file_name_new="$n_samp_dd"_"$file_hash".."$file_shortname"."$file_ext";
310 file_path_new="$BKFEH_SAMPLE_DIR"/"$file_name_new"
4db78240
SBS
311 if [[ "$sample_req_space" -lt "$sample_max_space" ]]; then
312 #### add file to sample dir
82fabd02
SBS
313 must cp -n "$file_path" "$file_path_new";
314 #### note path in sample dir log
315 printf "%s,%s,%s\n" "$n_samp_dd" "$file_hash" "$file_path" \
316 >> "$sample_log";
4db78240 317 fi;
82fabd02 318 ((n_samp++));
4db78240
SBS
319 done < <( echo "$list_paths_sample" );
320 else
321 yell "ERROR:Does not exist: $BKFEH_SAMPLE_DIR";
322 fi;
323}; # save sample of files
324
9eaab35f
SBS
325main() {
326 # Depends: read_stdin_psarg() v0.0.1, check_depends()
966a8d11 327 local re_dotfile;
335b71e1 328 declare -a dirs_stdin dirs_psarg;
9eaab35f
SBS
329 declare -a paths_images;
330 declare list_paths_images;
331 check_depends;
332
335b71e1
SBS
333 #Populate dirs_stdin and dirs_psarg arrays
334 ## Read stdin as lines
966a8d11 335 re_dotfile="^\."; # first char is a dot
9eaab35f 336 while read -r line; do
4db78240 337 line="$(readlink -e "$line")";
966a8d11 338 # Check if dir
9eaab35f
SBS
339 if [[ ! -d "$line" ]]; then
340 echo "ERROR:Not a dir:$line" 1>&2;
341 continue;
342 fi;
966a8d11
SBS
343 dir_name="$(basename "$line")";
344 # Exclude dotdirs
345 if [[ "$dir_name" =~ $re_dotfile ]]; then
346 echo "ERROR:Is a dotdir:$line" 1>&2;
347 continue
348 fi;
335b71e1
SBS
349 dirs_stdin+=("$line");
350 done < <(read_stdin);
351 ## Read positional arguments as lines
352 re_dotfile="^\."; # first char is a dot
353 while read -r line; do
4db78240 354 line="$(readlink -e "$line")";
335b71e1
SBS
355 # Check if dir
356 if [[ ! -d "$line" ]]; then
357 echo "ERROR:Not a dir:$line" 1>&2;
358 continue;
359 fi;
360 dir_name="$(basename "$line")";
361 # Exclude dotdirs
362 if [[ "$dir_name" =~ $re_dotfile ]]; then
363 echo "ERROR:Is a dotdir:$line" 1>&2;
364 continue
365 fi;
366 dirs_psarg+=("$line");
367 done < <(read_psarg "$@");
368
369 # Catch all arrays empty
370 if [[ "${#dirs_stdin[@]}" -le 0 ]] && [[ "${#dirs_psarg[@]}" -le 0 ]]; then
371 die "FATAL:No valid directories provided.";
372 fi;
9eaab35f
SBS
373
374 # Generate file list
335b71e1
SBS
375 # Find settings
376 firegex=".+\(jpg\|jpeg\|gif\|png\|webm\)$"; # update according to `find . -type f | grep -Eo "\.([[:alnum:]])+$" | sort -u`
377 fsize="10k"; # default: minimum "10k"
378 export firegex fsize ; # export for parallel
379 ## Call find_filelist() in parallel for positional argument input
380 if [[ "${#dirs_psarg[@]}" -gt 0 ]]; then
381 fdepth=10; export fdepth; # 10 for dirs from positional arguments
382 paths_images+=("$( parallel find_flist {} "$fdepth" "$firegex" "$fsize" ::: "${dirs_psarg[@]}" )"); # See [1]
383 fi;
384 ## Call find_filelist() in parallel for stdin input
385 if [[ "${#dirs_stdin[@]}" -gt 0 ]]; then
75e92939 386 fdepth=1; export fdepth; # 1 for dirs from stdin
335b71e1
SBS
387 paths_images+=("$( parallel find_flist {} "$fdepth" "$firegex" "$fsize" ::: "${dirs_stdin[@]}" )"); # See [1]
388 fi;
389
9eaab35f
SBS
390 # Convert paths_images array into file list
391 for i in "${!paths_images[@]}"; do
392 list_paths_images="$(printf "%s\n%s" "${paths_images[$i]}" "$list_paths_images")";
393
394 # Get stats
395 file_count="$(wc -l < <(echo -n "$list_paths_images"))";
396 echo "$DEBUG:file_count:$file_count"
397 done;
966a8d11
SBS
398
399 # Sort, remove duplicate paths
9eaab35f
SBS
400 list_paths_images="$(echo "$list_paths_images" | sort -u | tr -s '\n')";
401
335b71e1
SBS
402 # Remove paths with dotfiles
403 list_paths_images="$(echo "$list_paths_images" | grep -viE "/\." )";
404
9eaab35f
SBS
405 # Write
406 list_paths_images_tmp="/dev/shm/$(date +%Y%m%dT%H%M%S.%N%z)"..feh_paths.txt;
407 must echo -n "$list_paths_images" > "$list_paths_images_tmp";
408
409 # Print stats
410 yell "STATUS:Built file list in $SECONDS seconds.";
411
412 # Run feh with filelist
acaa0f3d
SBS
413 feh --full-screen --auto-zoom --draw-filename --filelist "$list_paths_images_tmp" && \
414 must rm "$list_paths_images_tmp" &
9eaab35f 415
acaa0f3d
SBS
416 # Save sample to path in env. var. BKFEH_SAMPLE_DIR if set
417 save_sample "$list_paths_images";
418
9eaab35f 419};
335b71e1 420export -f yell die must read_stdin read_psarg find_flist;
9eaab35f
SBS
421#==END Define local functions==
422
423main "$@";
424
75e92939 425
9eaab35f
SBS
426# Author: Steven Baltakatei Sandoval
427# License: GPLv3+