Skip to content

Commit 98b8da6

Browse files
VietND96VietND96
committed
Docker: Improve video recorder/uploader with async tasks and graceful shutdown
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent 98a4923 commit 98b8da6

File tree

5 files changed

+410
-19
lines changed

5 files changed

+410
-19
lines changed

‎Video/recorder.conf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ autostart=%(ENV_SE_RECORD_VIDEO)s
66
startsecs=0
77
autorestart=%(ENV_SE_RECORD_VIDEO)s
88
stopsignal=TERM
9-
stopwaitsecs=30
109

1110
;Logs (all activity redirected to stdout so it can be seen through "docker logs"
1211
redirect_stderr=true

‎Video/upload.sh‎

Lines changed: 188 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"}
88
UPLOAD_PIPE_FILE_NAME=${SE_UPLOAD_PIPE_FILE_NAME:-"uploadpipe"}
99
VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD}
1010
VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"}
11+
UPLOAD_RETRY_MAX_ATTEMPTS=${SE_UPLOAD_RETRY_MAX_ATTEMPTS:-3}
12+
UPLOAD_RETRY_DELAY=${SE_UPLOAD_RETRY_DELAY:-5}
13+
UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-1}
14+
UPLOAD_VERIFY_CHECKSUM=${SE_UPLOAD_VERIFY_CHECKSUM:-"true"}
1115
ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"}
1216
process_name="video.uploader"
1317

@@ -41,6 +45,57 @@ function rename_rclone_env() {
4145
}
4246

4347
list_rclone_pid=()
48+
upload_success_count=0
49+
upload_failed_count=0
50+
graceful_exit_called=false
51+
52+
function verify_file_ready() {
53+
local file=$1
54+
55+
# Check if file exists
56+
if [ ! -f "${file}" ]; then
57+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File does not exist: ${file}"
58+
return 1
59+
fi
60+
61+
# Check if file is readable
62+
if [ ! -r "${file}" ]; then
63+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is not readable: ${file}"
64+
return 1
65+
fi
66+
67+
# Check if file has non-zero size
68+
local file_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null)
69+
if [ -z "${file_size}" ] || [ "${file_size}" -eq 0 ]; then
70+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is empty: ${file}"
71+
return 1
72+
fi
73+
74+
# Wait for file to be stable (no longer being written)
75+
local initial_size=${file_size}
76+
sleep ${UPLOAD_FILE_READY_WAIT}
77+
local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null)
78+
79+
if [ "${initial_size}" != "${final_size}" ]; then
80+
echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File is still being written: ${file} (size changed from ${initial_size} to ${final_size})"
81+
return 1
82+
fi
83+
84+
echo "$(date -u +"${ts_format}") [${process_name}] - File verified ready for upload: ${file} (size: ${final_size} bytes)"
85+
return 0
86+
}
87+
88+
function calculate_checksum() {
89+
local file=$1
90+
# Use md5sum for checksum (available on most systems)
91+
if command -v md5sum >/dev/null 2>&1; then
92+
md5sum "${file}" | awk '{print $1}'
93+
elif command -v md5 >/dev/null 2>&1; then
94+
md5 -q "${file}"
95+
else
96+
echo ""
97+
fi
98+
}
4499
function check_and_clear_background() {
45100
# Wait for a batch rclone processes to finish
46101
if [ ${#list_rclone_pid[@]} -eq ${VIDEO_UPLOAD_BATCH_CHECK} ]; then
@@ -52,11 +107,89 @@ function check_and_clear_background() {
52107

53108
}
54109

110+
function rclone_upload_with_retry() {
111+
local source=$1
112+
local target=$2
113+
local attempt=1
114+
local source_checksum=""
115+
116+
# Verify file is ready before attempting upload
117+
if ! verify_file_ready "${source}"; then
118+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File verification failed, skipping upload: ${source}"
119+
upload_failed_count=$((upload_failed_count + 1))
120+
return 1
121+
fi
122+
123+
# Calculate checksum if verification is enabled
124+
if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ]; then
125+
source_checksum=$(calculate_checksum "${source}")
126+
if [ -n "${source_checksum}" ]; then
127+
echo "$(date -u +"${ts_format}") [${process_name}] - Source file checksum: ${source_checksum}"
128+
fi
129+
fi
130+
131+
# Retry loop
132+
while [ ${attempt} -le ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; do
133+
echo "$(date -u +"${ts_format}") [${process_name}] - Upload attempt ${attempt}/${UPLOAD_RETRY_MAX_ATTEMPTS}: ${source} to ${target}"
134+
135+
# Execute rclone command
136+
if rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}"; then
137+
echo "$(date -u +"${ts_format}") [${process_name}] - SUCCESS: Upload completed: ${source} to ${target}"
138+
139+
# Verify checksum if enabled and using copy command
140+
if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ] && [ -n "${source_checksum}" ] && [ "${UPLOAD_COMMAND}" = "copy" ]; then
141+
# For copy command, verify source file still has same checksum
142+
local post_upload_checksum=$(calculate_checksum "${source}")
143+
if [ "${source_checksum}" = "${post_upload_checksum}" ]; then
144+
echo "$(date -u +"${ts_format}") [${process_name}] - Checksum verification passed: ${source_checksum}"
145+
else
146+
echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: Checksum mismatch after upload (before: ${source_checksum}, after: ${post_upload_checksum})"
147+
fi
148+
fi
149+
150+
# Update statistics file (for cross-process tracking)
151+
(
152+
flock -x 200
153+
local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0)
154+
echo "upload_success_count=$((current_success + 1))" >/tmp/upload_stats.txt.tmp
155+
local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0)
156+
echo "upload_failed_count=${current_failed}" >>/tmp/upload_stats.txt.tmp
157+
mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt
158+
) 200>/tmp/upload_stats.lock
159+
160+
return 0
161+
else
162+
local exit_code=$?
163+
echo "$(date -u +"${ts_format}") [${process_name}] - FAILED: Upload attempt ${attempt} failed with exit code ${exit_code}: ${source}"
164+
165+
if [ ${attempt} -lt ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; then
166+
echo "$(date -u +"${ts_format}") [${process_name}] - Retrying in ${UPLOAD_RETRY_DELAY} seconds..."
167+
sleep ${UPLOAD_RETRY_DELAY}
168+
fi
169+
170+
attempt=$((attempt + 1))
171+
fi
172+
done
173+
174+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: All upload attempts failed for: ${source}"
175+
176+
# Update statistics file (for cross-process tracking)
177+
(
178+
flock -x 200
179+
local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0)
180+
echo "upload_success_count=${current_success}" >/tmp/upload_stats.txt.tmp
181+
local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0)
182+
echo "upload_failed_count=$((current_failed + 1))" >>/tmp/upload_stats.txt.tmp
183+
mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt
184+
) 200>/tmp/upload_stats.lock
185+
186+
return 1
187+
}
188+
55189
function rclone_upload() {
56190
local source=$1
57191
local target=$2
58-
echo "$(date -u +"${ts_format}") [${process_name}] - Uploading ${source} to ${target}"
59-
rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}" &
192+
rclone_upload_with_retry "${source}" "${target}" &
60193
list_rclone_pid+=($!)
61194
check_and_clear_background
62195
}
@@ -99,16 +232,53 @@ function wait_until_pipefile_exists() {
99232
done
100233
}
101234

235+
function wait_for_active_uploads() {
236+
if [ ${#list_rclone_pid[@]} -gt 0 ]; then
237+
echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${#list_rclone_pid[@]} active upload process(es) to complete"
238+
for pid in "${list_rclone_pid[@]}"; do
239+
if kill -0 "$pid" 2>/dev/null; then
240+
echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for upload process (PID: $pid)"
241+
wait "$pid" 2>/dev/null || true
242+
fi
243+
done
244+
echo "$(date -u +"${ts_format}") [${process_name}] - All active upload processes completed"
245+
list_rclone_pid=()
246+
fi
247+
}
248+
102249
function graceful_exit() {
250+
# Prevent duplicate execution (trap catches both SIGTERM and EXIT)
251+
if [ "$graceful_exit_called" = "true" ]; then
252+
return 0
253+
fi
254+
graceful_exit_called=true
255+
103256
echo "$(date -u +"${ts_format}") [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down uploader"
257+
258+
# Signal the pipe consumer to stop accepting new files
104259
if ! check_if_pid_alive "${UPLOAD_PID}"; then
105260
consume_pipe_file_in_background &
106261
UPLOAD_PID=$!
107262
fi
108263
echo "exit" >>"${UPLOAD_PIPE_FILE}" &
109264
wait "${UPLOAD_PID}"
110265
echo "$(date -u +"${ts_format}") [${process_name}] - Uploader consumed all files in the pipe"
266+
267+
# Wait for all active background rclone uploads to complete
268+
wait_for_active_uploads
269+
270+
# Log upload statistics from file (written by background processes)
271+
if [ -f "/tmp/upload_stats.txt" ]; then
272+
source "/tmp/upload_stats.txt" 2>/dev/null || true
273+
fi
274+
local total_uploads=$((upload_success_count + upload_failed_count))
275+
echo "$(date -u +"${ts_format}") [${process_name}] - Upload Statistics:"
276+
echo "$(date -u +"${ts_format}") [${process_name}] - Total uploads attempted: ${total_uploads}"
277+
echo "$(date -u +"${ts_format}") [${process_name}] - Successful uploads: ${upload_success_count}"
278+
echo "$(date -u +"${ts_format}") [${process_name}] - Failed uploads: ${upload_failed_count}"
279+
111280
rm -rf "${FORCE_EXIT_FILE}"
281+
rm -rf "/tmp/upload_stats.txt"
112282
echo "$(date -u +"${ts_format}") [${process_name}] - Uploader is ready to shutdown"
113283
exit 0
114284
}
@@ -117,12 +287,28 @@ rename_rclone_env
117287
trap graceful_exit SIGTERM SIGINT EXIT
118288

119289
while true; do
290+
# Exit main loop if graceful shutdown is in progress
291+
if [ "$graceful_exit_called" = "true" ]; then
292+
break
293+
fi
294+
120295
wait_until_pipefile_exists
296+
297+
# Exit main loop if graceful shutdown is in progress
298+
if [ "$graceful_exit_called" = "true" ]; then
299+
break
300+
fi
301+
121302
if ! check_if_pid_alive "${UPLOAD_PID}"; then
122303
consume_pipe_file_in_background &
123304
UPLOAD_PID=$!
124305
fi
306+
125307
while check_if_pid_alive "${UPLOAD_PID}"; do
308+
# Exit main loop if graceful shutdown is in progress
309+
if [ "$graceful_exit_called" = "true" ]; then
310+
break 2 # Break out of both while loops
311+
fi
126312
sleep 1
127313
done
128314
done

‎Video/uploader.conf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ autostart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s
66
startsecs=0
77
autorestart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s
88
stopsignal=TERM
9-
stopwaitsecs=30
109

1110
;Logs (all activity redirected to stdout so it can be seen through "docker logs"
1211
redirect_stderr=true

0 commit comments

Comments
 (0)