From 0b70940e158dced23d19884586b3a423adad3c1c Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 19 Oct 2025 16:58:17 +0200 Subject: [PATCH 01/39] Optimize file manager move operations on same filesystem --- emhttp/plugins/dynamix/nchan/file_manager | 118 ++++++++++++++++++---- 1 file changed, 99 insertions(+), 19 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 076a6b09d8..a9b9525b7f 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -12,10 +12,11 @@ */ ?> '._('Moving').'... '.($move===false ? exec("tail -1 $status") : shell_exec("tail -2 $status | awk -F\"\\r\" '{gsub(/^ +/,\"\",\$NF);print \$NF}'")); } else { @@ -173,21 +195,79 @@ while (true) { if ($target) { $move = true; $mkpath = isdir($target) ? '--mkpath' : ''; - exec("rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." 1>$status 2>$error & echo \$!", $pid); - } else { - $reply['error'] = 'Invalid target name'; - } - } - break; - case 5: // move folder (mv) - case 10: // move file (mv) - if (!empty($pid)) { - $reply['status'] = ''._('Moving').'...'; - } else { - $target = validname($target, false); - if ($target) { - $f = $exist ? 'n' : 'f'; - exec("mv -{$f}t ".escapeshellarg($target)." ".quoted($source)." 1>$status 2>$error & echo \$!", $pid); + + // determine if we can use rsync with rename(2) + // requirements: + // 1. all sources and target must be on the same mount point + // 2. all sources must have the same parent directory + // 3. either no --ignore-existing flag OR no conflicts with existing files on target + $use_rsync_rename = false; + $last_dirname = ''; + $target_mount_point = []; + + // for mount point check, find first existing parent directory + $target_for_stat = $target; + while (!file_exists($target_for_stat) && $target_for_stat != '/') { + $target_for_stat = dirname($target_for_stat); + } + $stat_cmd = "stat -c %m -- ".escapeshellarg($target_for_stat); + exec("$stat_cmd 2>&1", $target_mount_point, $rc); + + if (!empty($target_mount_point)) { + $use_rsync_rename = true; // assume we can use it, then check for disqualifying conditions + + foreach ($source as $source_path) { + + // source path must be valid + $valid_source_path = validname($source_path); + if (!$valid_source_path) { + $use_rsync_rename = false; + break; + } + + // mount points of source and target must be equal + $source_mount_point = []; + exec("stat -c %m -- ".escapeshellarg($valid_source_path)."", $source_mount_point); + if ($source_mount_point[0] != $target_mount_point[0]) { + $use_rsync_rename = false; + break; + } + + // parent directory of all source paths must be equal (not sure if this is really required, but keeping for now) + if (!empty($last_dirname) && $last_dirname != dirname($valid_source_path) ) { + $use_rsync_rename = false; + break; + } + $last_dirname = dirname($valid_source_path); + + // selected source files and directories must not exist on target when "Overwrite existing files" is not set + if (!empty($exist)) { // would add "--ignore-existing" to rsync + $target_item = rtrim($target, '/') . '/' . basename($valid_source_path); + if (file_exists($target_item)) { + $use_rsync_rename = false; + break; + } + } + + } + } + + // use rsync rename + // let rsync act like "mv" by syncing an empty directory against the source location, which moves the files to --backup-dir + // notes: + // - existing files are overwritten in --backup-dir (like not using --ignore-existing) + // - missing directories are created in --backup-dir (like using --mkpath) + if ($use_rsync_rename) { + $parent_dir = dirname(validname($source[0])); + $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." 1>$status 2>$error & echo \$!"; + exec($cmd, $pid); + + // use rsync copy-delete + } else { + $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." 1>$status 2>$error & echo \$!"; + exec($cmd, $pid); + } + } else { $reply['error'] = 'Invalid target name'; } From 42d6622dea67be4052a797d06aaa53478a55b0fd Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 19 Oct 2025 20:02:14 +0200 Subject: [PATCH 02/39] Add directory check for rsync rename optimization and suppress stderr from stat commands --- emhttp/plugins/dynamix/nchan/file_manager | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index a9b9525b7f..4b7b3e4fd5 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -210,9 +210,9 @@ while (true) { while (!file_exists($target_for_stat) && $target_for_stat != '/') { $target_for_stat = dirname($target_for_stat); } - $stat_cmd = "stat -c %m -- ".escapeshellarg($target_for_stat); - exec("$stat_cmd 2>&1", $target_mount_point, $rc); + exec("stat -c %m -- ".escapeshellarg($target_for_stat)." 2>/dev/null", $target_mount_point); + // check all source paths if (!empty($target_mount_point)) { $use_rsync_rename = true; // assume we can use it, then check for disqualifying conditions @@ -227,8 +227,8 @@ while (true) { // mount points of source and target must be equal $source_mount_point = []; - exec("stat -c %m -- ".escapeshellarg($valid_source_path)."", $source_mount_point); - if ($source_mount_point[0] != $target_mount_point[0]) { + exec("stat -c %m -- ".escapeshellarg($valid_source_path)." 2>/dev/null", $source_mount_point); + if (empty($source_mount_point) || $source_mount_point[0] != $target_mount_point[0]) { $use_rsync_rename = false; break; } @@ -240,6 +240,12 @@ while (true) { } $last_dirname = dirname($valid_source_path); + // target must be a directory + if (!is_dir(rtrim($target,'/'))) { + $use_rsync_rename = false; + break; + } + // selected source files and directories must not exist on target when "Overwrite existing files" is not set if (!empty($exist)) { // would add "--ignore-existing" to rsync $target_item = rtrim($target, '/') . '/' . basename($valid_source_path); From d37af270e56234a79cd52f3729e57b8e95d17435 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 19 Oct 2025 20:12:28 +0200 Subject: [PATCH 03/39] Use device IDs instead of mount points for more robust filesystem comparison --- emhttp/plugins/dynamix/nchan/file_manager | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 4b7b3e4fd5..3e1e102b6b 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -198,22 +198,22 @@ while (true) { // determine if we can use rsync with rename(2) // requirements: - // 1. all sources and target must be on the same mount point + // 1. all sources and target must be on the same filesystem (device) // 2. all sources must have the same parent directory // 3. either no --ignore-existing flag OR no conflicts with existing files on target $use_rsync_rename = false; $last_dirname = ''; - $target_mount_point = []; + $target_device_id = []; - // for mount point check, find first existing parent directory + // for filesystem check, find first existing parent directory $target_for_stat = $target; while (!file_exists($target_for_stat) && $target_for_stat != '/') { $target_for_stat = dirname($target_for_stat); } - exec("stat -c %m -- ".escapeshellarg($target_for_stat)." 2>/dev/null", $target_mount_point); + exec("stat -c %d -- ".escapeshellarg($target_for_stat)." 2>/dev/null", $target_device_id); // check all source paths - if (!empty($target_mount_point)) { + if (!empty($target_device_id)) { $use_rsync_rename = true; // assume we can use it, then check for disqualifying conditions foreach ($source as $source_path) { @@ -225,10 +225,10 @@ while (true) { break; } - // mount points of source and target must be equal - $source_mount_point = []; - exec("stat -c %m -- ".escapeshellarg($valid_source_path)." 2>/dev/null", $source_mount_point); - if (empty($source_mount_point) || $source_mount_point[0] != $target_mount_point[0]) { + // filesystem (device) of source and target must be equal + $source_device_id = []; + exec("stat -c %d -- ".escapeshellarg($valid_source_path)." 2>/dev/null", $source_device_id); + if (empty($source_device_id) || $source_device_id[0] != $target_device_id[0]) { $use_rsync_rename = false; break; } From 2568736259f52f8a211c3c6785346851e1a86f84 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 19 Oct 2025 20:22:29 +0200 Subject: [PATCH 04/39] Escape rsync filter meta-characters and handle symlinked directories properly --- emhttp/plugins/dynamix/nchan/file_manager | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 3e1e102b6b..bd73d78511 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -112,15 +112,22 @@ function escape($name) {return escapeshellarg(validname($name));} function quoted($name) {return is_array($name) ? implode(' ',array_map('escape', $name)) : escape($name);} function source($name) {return is_array($name) ? implode(' ',array_map('escapeshellarg', $name)) : escapeshellarg($name);} +// Escape rsync filter meta characters in a single path component +function rsync_escape_component($s) { + // escape: *, ?, [, ], \ + return preg_replace('/([*?\[\]\\\\])/', '\\\\$1', $s); +} + function quoted_rsync_include($paths) { // note: this function is never called with invalid names because of "if (!$valid_source_path)" $result = []; foreach ($paths as $path) { $valid_path = validname($path); - if (is_dir($valid_path)) { - $result[] = "--include=".escapeshellarg("/".basename($valid_path)."/***"); + $base = rsync_escape_component(basename($valid_path)); + if (is_dir($valid_path) && !is_link($valid_path)) { + $result[] = "--include=" . escapeshellarg("/{$base}/***"); } else { - $result[] = "--include=".escapeshellarg("/".basename($valid_path)); + $result[] = "--include=" . escapeshellarg("/{$base}"); } } return implode(' ', $result); From fbcc3684c64f8e447c0f945b0fb7d95430bed0c6 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Fri, 24 Oct 2025 04:06:01 +0200 Subject: [PATCH 05/39] file_manager: remove redundant PID validation and persist PID to survive restarts --- emhttp/plugins/dynamix/nchan/file_manager | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index bd73d78511..aeae5dbf62 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -316,6 +316,22 @@ while (true) { continue 2; } $pid = pgrep($pid??0); + + // Save PID to active file to survive file_manager restarts + if ($pid !== false && file_exists($active)) { + $ini = parse_ini_file($active); + if (!isset($ini['pid']) || $ini['pid'] != $pid) { + $ini['pid'] = $pid; + $content = ''; + foreach ($ini as $key => $value) { + // Escape double quotes and backslashes for INI format + $escaped_value = str_replace(['\\', '"'], ['\\\\', '\\"'], $value); + $content .= "$key=\"$escaped_value\"\n"; + } + file_put_contents($active, $content); + } + } + if ($pid === false) { if (!empty($move)) { exec("find ".quoted($source)." -type d -empty -print -delete 1>$status 2>$null & echo \$!", $pid); From 65110291726672f4e5809beffeac4109eb1bba8e Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Fri, 24 Oct 2025 04:21:46 +0200 Subject: [PATCH 06/39] file_manager: use separate PID file instead of INI serialization Replace complex INI serialization with simple separate .pid file: - Avoids escaping issues (semicolons, hashes, newlines, etc.) - No risk of INI file corruption - Consistent with existing pattern (.active, .status, .error, .pid) - Simpler code with fewer error sources - Atomic writes (single number to file) --- emhttp/plugins/dynamix/nchan/file_manager | 28 +++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index aeae5dbf62..508c488ab1 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -14,6 +14,7 @@ $value) { - // Escape double quotes and backslashes for INI format - $escaped_value = str_replace(['\\', '"'], ['\\\\', '\\"'], $value); - $content .= "$key=\"$escaped_value\"\n"; - } - file_put_contents($active, $content); - } + // Save PID to separate file to survive file_manager restarts + if ($pid !== false) { + file_put_contents($pid_file, $pid); } if ($pid === false) { @@ -354,7 +352,7 @@ while (true) { } } if (file_exists($error)) $reply['error'] = str_replace("\n","
", trim(file_get_contents($error))); - delete_file($active, $status, $error); + delete_file($active, $pid_file, $status, $error); unset($pid, $move); } } From 69a21de7a4dd0ab8bb5cd399c97449501ac3f44f Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:26:10 +0200 Subject: [PATCH 07/39] file_manager: persist move state and handle restarts - Initialize outside the main loop to ensure correct state handling for move/cleanup phases - If file_manager restarts and PID is loaded from file and move operation is still active, enable to cleanup empty directories afterwards - Clarify comment for INI job parameters --- emhttp/plugins/dynamix/nchan/file_manager | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 508c488ab1..4dc0419fab 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -139,16 +139,20 @@ if (!file_exists($empty_dir)) { mkdir($empty_dir); } +// initialize $move state: null = not a move operation (yet), true = rsync phase, false = cleanup phase +$move = null; + while (true) { unset($action, $source, $target, $H, $sparse, $exist, $zfs); - + + // read job parameters from ini file: $action, $title, $source, $target, $H, $sparse, $exist, $zfs (set by emhttp/plugins/dynamix/include/Control.php) if (file_exists($active)) extract(parse_ini_file($active)); - - // Read PID from separate file if exists - if (file_exists($pid_file)) { + + // read PID from file (file_manager may have been restarted) + if (!$pid && file_exists($pid_file)) { $pid = trim(file_get_contents($pid_file)); } - + $reply = []; if (isset($action)) { // check for language changes @@ -204,6 +208,8 @@ while (true) { case 9: // move file (rsync) case 10: // move file (mv) - this case needs to be removed from Browse.page if (!empty($pid)) { + // Set move state for resume: true=rsync phase, false=cleanup phase + if ($move === null) $move = true; $reply['status'] = ''._('Moving').'... '.($move===false ? exec("tail -1 $status") : shell_exec("tail -2 $status | awk -F\"\\r\" '{gsub(/^ +/,\"\",\$NF);print \$NF}'")); } else { $target = validname($target, false); @@ -324,12 +330,12 @@ while (true) { continue 2; } $pid = pgrep($pid??0); - - // Save PID to separate file to survive file_manager restarts + + // Store PID to survive file_manager restarts if ($pid !== false) { file_put_contents($pid_file, $pid); } - + if ($pid === false) { if (!empty($move)) { exec("find ".quoted($source)." -type d -empty -print -delete 1>$status 2>$null & echo \$!", $pid); From 7c38306d8276ba71e0d0d85aade539de7caf90a3 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:27:19 +0200 Subject: [PATCH 08/39] file_manager: suppress outdated status messages --- etc/rc.d/rc.nginx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/rc.d/rc.nginx b/etc/rc.d/rc.nginx index 54237695c9..3f59d39d99 100755 --- a/etc/rc.d/rc.nginx +++ b/etc/rc.d/rc.nginx @@ -164,6 +164,13 @@ build_servers(){ # server { listen unix:/var/run/nginx.socket default_server; + # filemanager should not display outdated messages + location /pub/filemanager { + nchan_publisher; + nchan_channel_id "filemanager"; + nchan_message_buffer_length $arg_buffer_length; + nchan_message_timeout 120s; + } location ~ /pub/(.*)$ { nchan_publisher; nchan_channel_id "$1"; From d26390c6df0da57fcbdf0ba8fa332e17b17dd993 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:32:19 +0200 Subject: [PATCH 09/39] file_manager: fixed syntax bug in nginx config to force using filemanager location block --- etc/rc.d/rc.nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/rc.d/rc.nginx b/etc/rc.d/rc.nginx index 3f59d39d99..f26020947d 100755 --- a/etc/rc.d/rc.nginx +++ b/etc/rc.d/rc.nginx @@ -165,7 +165,7 @@ build_servers(){ server { listen unix:/var/run/nginx.socket default_server; # filemanager should not display outdated messages - location /pub/filemanager { + location = /pub/filemanager { nchan_publisher; nchan_channel_id "filemanager"; nchan_message_buffer_length $arg_buffer_length; From 37e59b68e57f8bccc54f5534e81508312b568b0c Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 25 Oct 2025 02:27:00 +0200 Subject: [PATCH 10/39] Fix file_manager bugs and migrate to JSON status format Multiple improvements to file_manager reliability and status handling: 1. Fix rsync output parsing bug - rsync returns filename and progress lines in unpredictable order - Old JS parsing: text[1].split() assumed fixed line order - New approach: Simplified shell parsing (tail + tr) in PHP - parse_rsync_progress() helper function for consistent handling 2. Migrate status updates to JSON format - Structure: {action: int, text: [file_text, progress_text?]} - PHP: mb_strimhalf() for UTF-8-safe middle truncation - PHP: htmlspecialchars() escaping, JS adds icons client-side - Minimizes WebSocket overhead (no HTML in payload) - Universal handler in BrowseButton.page (6 lines) 3. Fix stale status messages (Nchan caching issue) - rc.nginx: nchan_message_timeout 30s for /pub/filemanager - Status expires automatically after 30s - Prevents displaying outdated progress status in footer 4. Remove obsolete move operation variants (case 5/10) - file_manager: Remove case 5/10 comments (mv-based move) - Browse.page: Remove action++ logic from doAction/doActions - BrowseButton.page: Remove case 5/10 from dfm_makeDialog() - All moves now use case 4/9 with automatic rsync-rename optimization --- emhttp/plugins/dynamix/Browse.page | 36 ----------- emhttp/plugins/dynamix/BrowseButton.page | 28 +++++++-- emhttp/plugins/dynamix/nchan/file_manager | 76 ++++++++++++++++++++--- etc/rc.d/rc.nginx | 2 +- 4 files changed, 92 insertions(+), 50 deletions(-) diff --git a/emhttp/plugins/dynamix/Browse.page b/emhttp/plugins/dynamix/Browse.page index e67d655b26..cb1f94be8d 100644 --- a/emhttp/plugins/dynamix/Browse.page +++ b/emhttp/plugins/dynamix/Browse.page @@ -498,15 +498,6 @@ function doAction(action, title, id) { case 9: // move file // disallow mixing of disk and user shares var valid = ud ? /^\/mnt\/.+/ : (user ? /^\/mnt\/(user0?|disks|remotes)\/.+/ : /^\/mnt\/(?!.*(user0?|rootshare)\/).+$|^\/boot\/.+/); - // check if 'mv' can be used - if (path.length > 2) { - if (user) { - if (home(source,target) == 1) action++; - } else { - var mv = '/'+path[0]+'/'+path[1]+'/'; - if (target.substr(0,mv.length) == mv) action++; - } - } break; case 11: // change owner var valid = /.+/; @@ -568,15 +559,6 @@ function doAction(action, title, id) { case 9: // move file // disallow mixing of disk and user shares var valid = ud ? /^\/mnt\/.+/ : (user ? /^\/mnt\/(user0?|disks|remotes)\/.+/ : /^\/mnt\/(?!.*(user0?|rootshare)\/).+$|^\/boot\/.+/); - // check if 'mv' can be used - if (path.length > 2) { - if (user) { - if (home(source,target) == 1) action++; - } else { - var mv = '/'+path[0]+'/'+path[1]+'/'; - if (target.substr(0,mv.length) == mv) action++; - } - } break; case 11: // change owner var valid = /.+/; @@ -782,15 +764,6 @@ function doActions(action, title) { case 4: // move object // disallow mixing of disk and user shares var valid = ud ? /^\/mnt\/.+/ : (user ? /^\/mnt\/(user0?|disks|remotes)\/.+/ : /^\/mnt\/(?!.*(user0?|rootshare)\/).+$|^\/boot\/.+/); - // check if 'mv' can be used - if (path.length > 2) { - if (user) { - if (home(source.join('\n'),target) == 1) action++; - } else { - var mv = '/'+path[0]+'/'+path[1]+'/'; - if (target.substr(0,mv.length) == mv) action++; - } - } break; case 11: // change owner var valid = /.+/; @@ -854,15 +827,6 @@ function doActions(action, title) { case 4: // move object // disallow mixing of disk and user shares var valid = ud ? /^\/mnt\/.+/ : (user ? /^\/mnt\/(user0?|disks|remotes)\/.+/ : /^\/mnt\/(?!.*(user0?|rootshare)\/).+$|^\/boot\/.+/); - // check if 'mv' can be used - if (path.length > 2) { - if (user) { - if (home(source.join('\n'),target) == 1) action++; - } else { - var mv = '/'+path[0]+'/'+path[1]+'/'; - if (target.substr(0,mv.length) == mv) action++; - } - } break; case 11: // change owner var valid = /.+/; diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index 71804a5642..ca2b42e7ef 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -113,6 +113,28 @@ function dfm_createSource(source) { function dfm_showProgress(data) { if (!data) return 0; + + // Try to parse as JSON first + try { + let parsed = JSON.parse(data); + + // Universal JSON format: {action: int, text: [text0, text1]} + // text[0] = file/main text, text[1] = progress info (optional) + if (parsed.action !== undefined && Array.isArray(parsed.text)) { + let icon = ""; + let footer = icon + (parsed.text[1] || parsed.text[0] || ''); + let dialog = parsed.text[0] ? (icon + parsed.text[0] + (parsed.text[1] ? '
' + footer : '')) : footer; + + dfm.window.find('.dfm_text').html(dialog); + dfm_footer('write', footer); + dfm.previous = footer; + return 0; + } + } catch(e) { + // Not JSON, fallback to old string parsing + } + + // Legacy string parsing for non-migrated operations let file = null; let text = data.split('\n'); let line = text[0].split('... '); @@ -206,8 +228,7 @@ function dfm_makeDialog(open) { dfm.window.find('#dfm_exist').prop('checked',dfm_read.exist ? false : true); dfm.height = 630; break; - case 4: // move folder/object (rsync) - case 5: // move folder/object (mv) + case 4: // move folder/object dfm.window.html($('#dfm_templateMoveFolder').html()); dfm.window.find('#dfm_target').val(dfm_read.target).prop('disabled',true); dfm.window.find('#dfm_sparse').prop('checked',dfm_read.sparse ? true : false); @@ -225,8 +246,7 @@ function dfm_makeDialog(open) { dfm.window.find('#dfm_exist').prop('checked',dfm_read.exist ? false : true); dfm.height = 630; break; - case 9: // move file (rsync) - case 10: // move file (mv) + case 9: // move file dfm.window.html($('#dfm_templateMoveFile').html()); dfm.window.find('#dfm_target').val(dfm_read.target).prop('disabled',true); dfm.window.find('#dfm_sparse').prop('checked',dfm_read.sparse ? true :false); diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 4dc0419fab..6619170113 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -54,6 +54,34 @@ function isdir($name) { return mb_substr($name, -1) == '/'; } +function mb_strimhalf($text, $width, $trim_marker = "") { + if (mb_strlen($text) <= $width) return $text; + $half = (int)($width / 2) - 2; + return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -1 - $half); +} + +function parse_rsync_progress($status, $action_label) { + // obtain last status + $last_status = shell_exec("tail -n 2 $status | tr '\r' '\n' | tr -s '[:space:]'"); + $lines = array_filter(explode("\n", trim($last_status))); + + // parse status and generate text array + $text = []; + foreach ($lines as $line) { + // progress line like "6.51G 56% 58.66MB/s 0:01:45 (xfr#2458, ir-chk=3439/5912)" + if (preg_match('/\d+%.*\d+:\d+:\d+/', $line)) { + $parts = explode(' ', $line); + if (count($parts) >= 4) { + $text[1] = _('Completed') . ": " . $parts[1] . ",  " . _('Speed') . ": " . $parts[2] . ",  " . _('ETA') . ": " . $parts[3]; + } + // file line like "mnt/disk3/sharename/file.bin" + } else { + $text[0] = $action_label . "... " . htmlspecialchars(mb_strimhalf($line, 70, '...'), ENT_QUOTES, 'UTF-8'); + } + } + return $text; +} + function truepath($name) { $bits = array_filter(explode('/', $name), 'mb_strlen'); $path = []; @@ -113,7 +141,6 @@ function escape($name) {return escapeshellarg(validname($name));} function quoted($name) {return is_array($name) ? implode(' ',array_map('escape', $name)) : escape($name);} function source($name) {return is_array($name) ? implode(' ',array_map('escapeshellarg', $name)) : escapeshellarg($name);} -// Escape rsync filter meta characters in a single path component function rsync_escape_component($s) { // escape: *, ?, [, ], \ return preg_replace('/([*?\[\]\\\\])/', '\\\\$1', $s); @@ -174,8 +201,15 @@ while (true) { break; case 1: // delete folder case 6: // delete file + + // return status of running action if (!empty($pid)) { - $reply['status'] = ''._('Removing').'... '.exec("tail -1 $status"); + $reply['status'] = json_encode([ + 'action' => $action, + 'text' => [htmlspecialchars(mb_strimhalf(exec("tail -1 $status"), 70, '...'), ENT_QUOTES, 'UTF-8')] + ]); + + // start action } else { exec("find ".quoted($source)." -name \"*\" -print -delete 1>$status 2>$null & echo \$!", $pid); } @@ -191,8 +225,15 @@ while (true) { break; case 3: // copy folder case 8: // copy file + + // return status of running action if (!empty($pid)) { - $reply['status'] = ''._('Copying').'... '.shell_exec("tail -2 $status | awk -F\"\\r\" '{gsub(/^ +/,\"\",\$NF);print \$NF}'"); + $reply['status'] = json_encode([ + 'action' => $action, + 'text' => parse_rsync_progress($status, _('Copying')) + ]); + + // start action } else { $target = validname($target, false); if ($target) { @@ -203,14 +244,31 @@ while (true) { } } break; - case 4: // move folder (rsync) - case 5: // move folder (mv) - this case needs to be removed from Browse.page - case 9: // move file (rsync) - case 10: // move file (mv) - this case needs to be removed from Browse.page + case 4: // move folder + case 9: // move file + + // return status of running action if (!empty($pid)) { - // Set move state for resume: true=rsync phase, false=cleanup phase + + // set move state for resume: true=rsync phase, false=cleanup phase if ($move === null) $move = true; - $reply['status'] = ''._('Moving').'... '.($move===false ? exec("tail -1 $status") : shell_exec("tail -2 $status | awk -F\"\\r\" '{gsub(/^ +/,\"\",\$NF);print \$NF}'")); + + // cleanup empty directories: simple status + if ($move === false) { + $reply['status'] = json_encode([ + 'action' => $action, + 'text' => [htmlspecialchars(mb_strimhalf(exec("tail -1 $status"), 70, '...'), ENT_QUOTES, 'UTF-8')] + ]); + + // moving: progress + } else { + $reply['status'] = json_encode([ + 'action' => $action, + 'text' => parse_rsync_progress($status, _('Moving')) + ]); + } + + // start action } else { $target = validname($target, false); if ($target) { diff --git a/etc/rc.d/rc.nginx b/etc/rc.d/rc.nginx index f26020947d..c8e71ba764 100755 --- a/etc/rc.d/rc.nginx +++ b/etc/rc.d/rc.nginx @@ -169,7 +169,7 @@ build_servers(){ nchan_publisher; nchan_channel_id "filemanager"; nchan_message_buffer_length $arg_buffer_length; - nchan_message_timeout 120s; + nchan_message_timeout 30s; } location ~ /pub/(.*)$ { nchan_publisher; From be210fc701217d7296a4b0c53d3f32cdddbcd35a Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 25 Oct 2025 02:41:01 +0200 Subject: [PATCH 11/39] Add minimum width guard to mb_strimhalf() Prevent negative or zero values in $half calculation when $width < 8. For small widths, simply truncate text without middle ellipsis. --- emhttp/plugins/dynamix/nchan/file_manager | 1 + 1 file changed, 1 insertion(+) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 6619170113..a40d4c616d 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -56,6 +56,7 @@ function isdir($name) { function mb_strimhalf($text, $width, $trim_marker = "") { if (mb_strlen($text) <= $width) return $text; + if ($width < 8) return mb_substr($text, 0, $width); $half = (int)($width / 2) - 2; return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -1 - $half); } From 031b4bae9acf381cb33599419f7dae11a3aefbd3 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 25 Oct 2025 13:52:41 +0200 Subject: [PATCH 12/39] Fix JSON array index bug in parse_rsync_progress() Initialize text array as indexed array to prevent associative array when only progress line is parsed. Ensures text[0] is always set, avoiding JSON output like {"1":"..."} in WebGUI. --- emhttp/plugins/dynamix/nchan/file_manager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index a40d4c616d..43bc4ea185 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -67,7 +67,7 @@ function parse_rsync_progress($status, $action_label) { $lines = array_filter(explode("\n", trim($last_status))); // parse status and generate text array - $text = []; + $text = [$action_label . "..."]; // ensure text[0] is always set foreach ($lines as $line) { // progress line like "6.51G 56% 58.66MB/s 0:01:45 (xfr#2458, ir-chk=3439/5912)" if (preg_match('/\d+%.*\d+:\d+:\d+/', $line)) { From ef0be9c2b7d2ea1aaa14e3280d1e61b90e04d1ec Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 25 Oct 2025 15:50:53 +0200 Subject: [PATCH 13/39] Rename $move to $delete_empty_dirs and add commented parent-to-subfolder check - Renamed $move variable to $delete_empty_dirs for clearer intent - Variable now only set to true for rsync copy-delete (not for rsync-rename) - rsync-rename doesn't need cleanup since source is atomically moved - Added commented code to prevent moving directory into its own subdirectory --- emhttp/plugins/dynamix/nchan/file_manager | 30 +++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 43bc4ea185..76bdd8fe40 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -167,8 +167,8 @@ if (!file_exists($empty_dir)) { mkdir($empty_dir); } -// initialize $move state: null = not a move operation (yet), true = rsync phase, false = cleanup phase -$move = null; +// initialize $delete_empty_dirs state: null = not a move operation (yet), true = rsync copy-delete phase, false = cleanup phase (done) +$delete_empty_dirs = null; while (true) { unset($action, $source, $target, $H, $sparse, $exist, $zfs); @@ -251,11 +251,11 @@ while (true) { // return status of running action if (!empty($pid)) { - // set move state for resume: true=rsync phase, false=cleanup phase - if ($move === null) $move = true; + // set delete_empty_dirs state for resume after file_manager restart (only relevant for rsync copy-delete) + if ($delete_empty_dirs === null) $delete_empty_dirs = true; // cleanup empty directories: simple status - if ($move === false) { + if ($delete_empty_dirs === false) { $reply['status'] = json_encode([ 'action' => $action, 'text' => [htmlspecialchars(mb_strimhalf(exec("tail -1 $status"), 70, '...'), ENT_QUOTES, 'UTF-8')] @@ -273,7 +273,6 @@ while (true) { } else { $target = validname($target, false); if ($target) { - $move = true; $mkpath = isdir($target) ? '--mkpath' : ''; // determine if we can use rsync with rename(2) @@ -335,6 +334,14 @@ while (true) { } } + // // target must not be a subdirectory of source parent (backup-dir should be outside source tree) + // $source_parent_dir = dirname($valid_source_path); + // if (strpos(rtrim($target,'/'), rtrim($source_parent_dir,'/') . '/') === 0) { + // $reply['error'] = 'Cannot move directory into its own subdirectory'; + // $use_rsync_rename = false; + // break 2; // break out of both foreach and if + // } + } } @@ -350,6 +357,7 @@ while (true) { // use rsync copy-delete } else { + $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." 1>$status 2>$error & echo \$!"; exec($cmd, $pid); } @@ -383,7 +391,8 @@ while (true) { case 99: // kill running background process if (!empty($pid)) exec("kill $pid"); delete_file($active, $pid_file, $status, $error); - unset($pid, $move); + unset($pid); + $delete_empty_dirs = null; break; default: continue 2; @@ -396,9 +405,9 @@ while (true) { } if ($pid === false) { - if (!empty($move)) { + if (!empty($delete_empty_dirs)) { exec("find ".quoted($source)." -type d -empty -print -delete 1>$status 2>$null & echo \$!", $pid); - $move = false; + $delete_empty_dirs = false; $pid = pgrep($pid); } else { if ($action != 15) { @@ -418,7 +427,8 @@ while (true) { } if (file_exists($error)) $reply['error'] = str_replace("\n","
", trim(file_get_contents($error))); delete_file($active, $pid_file, $status, $error); - unset($pid, $move); + unset($pid); + $delete_empty_dirs = null; } } } From f105f35369a489ea5c7faaf262aa4099b32cede8 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 25 Oct 2025 15:58:13 +0200 Subject: [PATCH 14/39] Enable parent-to-subdirectory check for rsync-rename moves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevents moving a directory into its own subdirectory, which would cause rsync --backup-dir to create nested structures and fail to delete source. Example prevented: /mnt/disk1/parent → /mnt/disk1/parent/subfolder The check uses trailing slashes to ensure exact directory boundary matching and prevent false positives (e.g., /parent vs /parent2). --- emhttp/plugins/dynamix/nchan/file_manager | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 76bdd8fe40..0dae28f396 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -334,13 +334,13 @@ while (true) { } } - // // target must not be a subdirectory of source parent (backup-dir should be outside source tree) - // $source_parent_dir = dirname($valid_source_path); - // if (strpos(rtrim($target,'/'), rtrim($source_parent_dir,'/') . '/') === 0) { - // $reply['error'] = 'Cannot move directory into its own subdirectory'; - // $use_rsync_rename = false; - // break 2; // break out of both foreach and if - // } + // target must not be a subdirectory of any source (backup-dir should be outside source tree) + $parent_dir = dirname($valid_source_path); + if (strpos(rtrim($target,'/') . '/', rtrim($parent_dir,'/') . '/') === 0) { + $reply['error'] = 'Cannot move directory into its own subdirectory'; + $use_rsync_rename = false; + break 2; // break out of both foreach and if + } } } From a33f8c44d3602a7123e42b579487463e996b58c4 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 01:55:15 +0200 Subject: [PATCH 15/39] Fix file_manager rsync issues: status parsing and directory permissions 1. Fixed status parsing bug where 'Completed: 118.88G, Speed: 2%, ETA: 58.91MB/s' showed wrong value assignments due to tr -s '[:space:]' compressing newlines. - Changed to tr -s ' \t' to preserve newlines - Optimized with shell-based tac/grep approach for better performance - Handles large rsync status files with thousands of progress lines efficiently 2. Added --no-inc-recursive flag to fix rsync creating directories with 0700 root:root permissions during copy-delete operations, causing SMB access issues. - Disables incremental recursion to avoid generator.c:1430 do_mkdir(...& 0700) - Should preserve 1:1 source permissions and improve progress calculation - NOTE: This is experimental and needs testing on dev server Both fixes target the copy-delete operation (move files) that was most problematic. --- emhttp/plugins/dynamix/nchan/file_manager | 41 ++++++++++++----------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 0dae28f396..ccb604a82f 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -62,25 +62,28 @@ function mb_strimhalf($text, $width, $trim_marker = "") { } function parse_rsync_progress($status, $action_label) { - // obtain last status - $last_status = shell_exec("tail -n 2 $status | tr '\r' '\n' | tr -s '[:space:]'"); - $lines = array_filter(explode("\n", trim($last_status))); - - // parse status and generate text array - $text = [$action_label . "..."]; // ensure text[0] is always set - foreach ($lines as $line) { - // progress line like "6.51G 56% 58.66MB/s 0:01:45 (xfr#2458, ir-chk=3439/5912)" - if (preg_match('/\d+%.*\d+:\d+:\d+/', $line)) { - $parts = explode(' ', $line); - if (count($parts) >= 4) { - $text[1] = _('Completed') . ": " . $parts[1] . ",  " . _('Speed') . ": " . $parts[2] . ",  " . _('ETA') . ": " . $parts[3]; - } - // file line like "mnt/disk3/sharename/file.bin" - } else { - $text[0] = $action_label . "... " . htmlspecialchars(mb_strimhalf($line, 70, '...'), ENT_QUOTES, 'UTF-8'); + + // obtain filename line like "/mnt/disk6/images/image.jpg" + $file_line = exec("tac $status | grep -m1 -v -E -C1 '\\d+%.*\\d+:\\d+:\\d+' | tr -s ' \\t' ' ' || echo ''"); + $text[0] = $action_label . "... "; + if ($file_line) { + $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); + } + + // obtain progress line like: + // currently running: " 7.77G 66% 28.61MB/s 0:02:11" + // finished transfer: " 7.78G 66% 23.39MB/s 0:05:17 (xfr#2907, ir-chk=2990/5913)" + // note: we do not loop with PHP as there could be a huge amount of progress lines b + $progress_line = exec("tac $status | grep -m1 -E -C1 '\\d+%.*\\d+:\\d+:\\d+' | tr -s ' \\t' ' ' || echo ''"); + if ($progress_line) { + $parts = explode(' ', $progress_line); + if (count($parts) >= 4) { + $text[1] = _('Completed') . ": " . $parts[1] . ",  " . _('Speed') . ": " . $parts[2] . ",  " . _('ETA') . ": " . $parts[3]; } } + return $text; + } function truepath($name) { @@ -239,7 +242,7 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - exec("rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." 1>$status 2>$error & echo \$!", $pid); + exec("rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!", $pid); } else { $reply['error'] = 'Invalid target name'; } @@ -352,13 +355,13 @@ while (true) { // - missing directories are created in --backup-dir (like using --mkpath) if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); - $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." 1>$status 2>$error & echo \$!"; + $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; exec($cmd, $pid); // use rsync copy-delete } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories - $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." 1>$status 2>$error & echo \$!"; + $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; exec($cmd, $pid); } From 58ec118854cb84d9334f7fcc691d085c7fdd5fcd Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 02:13:45 +0100 Subject: [PATCH 16/39] Optimize file_manager buffering and regex patterns - Add stdbuf -oL to rsync commands for immediate line-buffered output Fixes choppy/delayed progress updates in status file during transfers - Switch from grep -P to -E with [0-9]+ patterns for POSIX compatibility Eliminates 'stray \ before d' warnings and improves portability More explicit and readable than \d+ shortcuts - Update comments about rsync progress format differences: Running transfers show ETA, completed files show actual transfer time --- emhttp/plugins/dynamix/nchan/file_manager | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index ccb604a82f..68bf6c2ce6 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -64,17 +64,19 @@ function mb_strimhalf($text, $width, $trim_marker = "") { function parse_rsync_progress($status, $action_label) { // obtain filename line like "/mnt/disk6/images/image.jpg" - $file_line = exec("tac $status | grep -m1 -v -E -C1 '\\d+%.*\\d+:\\d+:\\d+' | tr -s ' \\t' ' ' || echo ''"); + $file_line = exec("tac $status | grep -m1 -v -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' \\t' ' ' || echo ''"); $text[0] = $action_label . "... "; if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); } // obtain progress line like: - // currently running: " 7.77G 66% 28.61MB/s 0:02:11" - // finished transfer: " 7.78G 66% 23.39MB/s 0:05:17 (xfr#2907, ir-chk=2990/5913)" + // currently running (showing total ETA): + // " 37.91G 4% 58.59MB/s 3:47:20" + // transfer of single file finished (ETA changes temporarily to estimated time) + // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines b - $progress_line = exec("tac $status | grep -m1 -E -C1 '\\d+%.*\\d+:\\d+:\\d+' | tr -s ' \\t' ' ' || echo ''"); + $progress_line = exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' \\t' ' ' || echo ''"); if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { @@ -242,7 +244,7 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - exec("rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!", $pid); + exec("stdbuf -oL rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!", $pid); } else { $reply['error'] = 'Invalid target name'; } @@ -355,13 +357,13 @@ while (true) { // - missing directories are created in --backup-dir (like using --mkpath) if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); - $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; + $cmd = "stdbuf -oL rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; exec($cmd, $pid); // use rsync copy-delete } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories - $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; + $cmd = "stdbuf -oL rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; exec($cmd, $pid); } From 38c6d843f0b79872f0cdff4b9fbc2e9e536ef0ca Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 03:23:35 +0100 Subject: [PATCH 17/39] Test stdbuf -o0 for unbuffered rsync output - Add stdbuf -o0 to enable real-time progress updates in status file - Capture rsync PID using subshell: { echo $!; tr ... } Testing if this resolves buffering and PID tracking issues. --- Another progress without newline | 0 Done. | 0 Final line with newline | 0 Progress 1 | 0 Progress 10 | 0 Progress 2 | 0 Progress 3 | 0 Progress 4 | 0 Progress 5 | 0 Progress 6 | 0 Progress 7 | 0 Progress 8 | 0 Progress 9 | 0 Starting fake rsync... | 0 emhttp/plugins/dynamix/nchan/file_manager | 6 +++--- 15 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 Another progress without newline create mode 100644 Done. create mode 100644 Final line with newline create mode 100644 Progress 1 create mode 100644 Progress 10 create mode 100644 Progress 2 create mode 100644 Progress 3 create mode 100644 Progress 4 create mode 100644 Progress 5 create mode 100644 Progress 6 create mode 100644 Progress 7 create mode 100644 Progress 8 create mode 100644 Progress 9 create mode 100644 Starting fake rsync... diff --git a/Another progress without newline b/Another progress without newline new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Done. b/Done. new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Final line with newline b/Final line with newline new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 1 b/Progress 1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 10 b/Progress 10 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 2 b/Progress 2 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 3 b/Progress 3 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 4 b/Progress 4 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 5 b/Progress 5 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 6 b/Progress 6 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 7 b/Progress 7 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 8 b/Progress 8 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Progress 9 b/Progress 9 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Starting fake rsync... b/Starting fake rsync... new file mode 100644 index 0000000000..e69de29bb2 diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 68bf6c2ce6..8ea34f409f 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -244,7 +244,7 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - exec("stdbuf -oL rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!", $pid); + exec("stdbuf -o0 rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error", $pid); } else { $reply['error'] = 'Invalid target name'; } @@ -357,13 +357,13 @@ while (true) { // - missing directories are created in --backup-dir (like using --mkpath) if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); - $cmd = "stdbuf -oL rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; + $cmd = "stdbuf -o0 rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error"; exec($cmd, $pid); // use rsync copy-delete } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories - $cmd = "stdbuf -oL rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | tr '\\r' '\\n' | tee -a $status >/dev/null 2>$error & echo \$!"; + $cmd = "stdbuf -o0 rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error"; exec($cmd, $pid); } From 8103cbad4d226c8370d9768efb1cc7f46696bce9 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 12:31:19 +0100 Subject: [PATCH 18/39] Fix rsync output handling with process substitution Use process substitution with stdbuf to capture rsync progress output line-by-line in real-time. The '> >(stdbuf -o0 tr '\r' '\n' >status)' pattern ensures unbuffered conversion of carriage returns to newlines, allowing parse_rsync_progress() to read the latest updates via tac. --- emhttp/plugins/dynamix/nchan/file_manager | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 8ea34f409f..a3527d8f9b 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -63,8 +63,8 @@ function mb_strimhalf($text, $width, $trim_marker = "") { function parse_rsync_progress($status, $action_label) { - // obtain filename line like "/mnt/disk6/images/image.jpg" - $file_line = exec("tac $status | grep -m1 -v -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' \\t' ' ' || echo ''"); + // obtain filename line like "/mnt/disk6/images/image.jpg" (-v is used to obtain the opposite of the progress line regex) + $file_line = exec("tac $status | grep -m1 -v -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); $text[0] = $action_label . "... "; if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); @@ -76,7 +76,7 @@ function parse_rsync_progress($status, $action_label) { // transfer of single file finished (ETA changes temporarily to estimated time) // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines b - $progress_line = exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' \\t' ' ' || echo ''"); + $progress_line = exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { @@ -244,7 +244,8 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - exec("stdbuf -o0 rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error", $pid); + $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + exec($cmd, $pid); } else { $reply['error'] = 'Invalid target name'; } @@ -357,13 +358,13 @@ while (true) { // - missing directories are created in --backup-dir (like using --mkpath) if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); - $cmd = "stdbuf -o0 rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error"; + $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; exec($cmd, $pid); // use rsync copy-delete } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories - $cmd = "stdbuf -o0 rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." | { echo \$!; tr '\\r' '\\n' } | tee -a $status >/dev/null 2>$error"; + $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; exec($cmd, $pid); } From 63328e944a9c78487713d53c5fa4990b45986699 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 12:48:32 +0100 Subject: [PATCH 19/39] Add explanatory comments and fix progress parsing - Add comment explaining grep -v usage for filename extraction - Document delete_empty_dirs state handling for both rsync methods - Add trim() to progress line parsing to fix array indexing issue caused by leading whitespace in rsync output --- emhttp/plugins/dynamix/nchan/file_manager | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index a3527d8f9b..0382d625e2 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -76,7 +76,7 @@ function parse_rsync_progress($status, $action_label) { // transfer of single file finished (ETA changes temporarily to estimated time) // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines b - $progress_line = exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); + $progress_line = trim(exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { @@ -167,7 +167,8 @@ function quoted_rsync_include($paths) { return implode(' ', $result); } -// create empty directory for optimized move if not exists +// create empty directory for "rsync rename" if not exists +// note: as only file_manager creates and uses this directory no "is empty" check and rebuild mechanism is required if (!file_exists($empty_dir)) { mkdir($empty_dir); } @@ -257,7 +258,11 @@ while (true) { // return status of running action if (!empty($pid)) { - // set delete_empty_dirs state for resume after file_manager restart (only relevant for rsync copy-delete) + // set delete_empty_dirs state for resume after file_manager restart (only relevant for rsync copy-delete)) + // note: would be theoretically enabled for "rsync rename", too, so find would even then try to delete + // source empty directories which do not exist, but as "rsync-rename" is very fast and the file_manager + // runs for several seconds in the background even if the user directly closes the WebGUI it should be + // nearly impossible that the user is able to see any errors if ($delete_empty_dirs === null) $delete_empty_dirs = true; // cleanup empty directories: simple status From f693c4966b2e9569e90d44abc2ca3772423fbf85 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 13:06:59 +0100 Subject: [PATCH 20/39] Add comment explaining bash requirement for process substitution Document that exec() uses /bin/sh which is symlinked to bash on Unraid, making process substitution syntax >(...) available for rsync commands. --- emhttp/plugins/dynamix/nchan/file_manager | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 0382d625e2..ed5b494a12 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -176,6 +176,9 @@ if (!file_exists($empty_dir)) { // initialize $delete_empty_dirs state: null = not a move operation (yet), true = rsync copy-delete phase, false = cleanup phase (done) $delete_empty_dirs = null; + +// infinite loop to monitor and execute file operations +// Note: exec() uses /bin/sh which is symlinked to bash in unraid and a requirement for process substitution syntax >(...) while (true) { unset($action, $source, $target, $H, $sparse, $exist, $zfs); From da0bfcafccc4b58d7fcd426512962db28d93436b Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 13:56:46 +0100 Subject: [PATCH 21/39] Fix grep regex: Remove -C1 context flag to prevent progress line in file line output --- Another progress without newline | 0 Done. | 0 Final line with newline | 0 Progress 1 | 0 Progress 10 | 0 Progress 2 | 0 Progress 3 | 0 Progress 4 | 0 Progress 5 | 0 Progress 6 | 0 Progress 7 | 0 Progress 8 | 0 Progress 9 | 0 Starting fake rsync... | 0 emhttp/plugins/dynamix/nchan/file_manager | 4 ++-- 15 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 Another progress without newline delete mode 100644 Done. delete mode 100644 Final line with newline delete mode 100644 Progress 1 delete mode 100644 Progress 10 delete mode 100644 Progress 2 delete mode 100644 Progress 3 delete mode 100644 Progress 4 delete mode 100644 Progress 5 delete mode 100644 Progress 6 delete mode 100644 Progress 7 delete mode 100644 Progress 8 delete mode 100644 Progress 9 delete mode 100644 Starting fake rsync... diff --git a/Another progress without newline b/Another progress without newline deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Done. b/Done. deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Final line with newline b/Final line with newline deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 1 b/Progress 1 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 10 b/Progress 10 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 2 b/Progress 2 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 3 b/Progress 3 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 4 b/Progress 4 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 5 b/Progress 5 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 6 b/Progress 6 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 7 b/Progress 7 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 8 b/Progress 8 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Progress 9 b/Progress 9 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Starting fake rsync... b/Starting fake rsync... deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index ed5b494a12..82658c2de1 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -64,7 +64,7 @@ function mb_strimhalf($text, $width, $trim_marker = "") { function parse_rsync_progress($status, $action_label) { // obtain filename line like "/mnt/disk6/images/image.jpg" (-v is used to obtain the opposite of the progress line regex) - $file_line = exec("tac $status | grep -m1 -v -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); + $file_line = exec("tac $status | grep -m1 -v -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); $text[0] = $action_label . "... "; if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); @@ -76,7 +76,7 @@ function parse_rsync_progress($status, $action_label) { // transfer of single file finished (ETA changes temporarily to estimated time) // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines b - $progress_line = trim(exec("tac $status | grep -m1 -E -C1 '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); + $progress_line = trim(exec("tac $status | grep -m1 -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { From 8c0ff09401178a59df28f128cccd8458ed66afb5 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 14:00:57 +0100 Subject: [PATCH 22/39] Improve code readability: Reorder progress parsing logic and fix comment (ETA vs elapsed time) --- emhttp/plugins/dynamix/nchan/file_manager | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 82658c2de1..e1fd9bc0a2 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -63,17 +63,10 @@ function mb_strimhalf($text, $width, $trim_marker = "") { function parse_rsync_progress($status, $action_label) { - // obtain filename line like "/mnt/disk6/images/image.jpg" (-v is used to obtain the opposite of the progress line regex) - $file_line = exec("tac $status | grep -m1 -v -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); - $text[0] = $action_label . "... "; - if ($file_line) { - $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); - } - // obtain progress line like: // currently running (showing total ETA): // " 37.91G 4% 58.59MB/s 3:47:20" - // transfer of single file finished (ETA changes temporarily to estimated time) + // transfer of single file finished (ETA changes temporarily to elapsed time) // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines b $progress_line = trim(exec("tac $status | grep -m1 -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); @@ -84,6 +77,13 @@ function parse_rsync_progress($status, $action_label) { } } + // obtain filename line like "/mnt/disk6/images/image.jpg" (-v is used to obtain the opposite of the progress line regex) + $file_line = exec("tac $status | grep -m1 -v -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); + $text[0] = $action_label . "... "; + if ($file_line) { + $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); + } + return $text; } From 67c5626e3f1cda59f3854bc4a32f1c584dc3102f Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 14:15:47 +0100 Subject: [PATCH 23/39] Fix rsync progress parsing: Initialize text array and filter empty lines --- emhttp/plugins/dynamix/nchan/file_manager | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index e1fd9bc0a2..72c9739f94 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -63,12 +63,15 @@ function mb_strimhalf($text, $width, $trim_marker = "") { function parse_rsync_progress($status, $action_label) { + // initialize text array with action label + $text[0] = $action_label . "... "; + // obtain progress line like: // currently running (showing total ETA): // " 37.91G 4% 58.59MB/s 3:47:20" - // transfer of single file finished (ETA changes temporarily to elapsed time) + // transfer of single file finished (ETA changes temporarily to elapsed time): // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" - // note: we do not loop with PHP as there could be a huge amount of progress lines b + // note: we do not loop with PHP as there could be a huge amount of progress lines $progress_line = trim(exec("tac $status | grep -m1 -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); if ($progress_line) { $parts = explode(' ', $progress_line); @@ -77,9 +80,10 @@ function parse_rsync_progress($status, $action_label) { } } - // obtain filename line like "/mnt/disk6/images/image.jpg" (-v is used to obtain the opposite of the progress line regex) - $file_line = exec("tac $status | grep -m1 -v -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''"); - $text[0] = $action_label . "... "; + // obtain filename line like: + // "/mnt/disk6/images/image.jpg" + // note: -v is used to obtain the opposite of the progress line regex and to filter out empty lines + $file_line = exec("tac $status | grep -m1 -v -E '(^\$|[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tr -s ' ' || echo ''"); if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); } From 617ce85f37dfc79517f4c50f2fb826aac1303c4e Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 16:59:00 +0100 Subject: [PATCH 24/39] Optimize spinning icon animation in File Manager - Preserve existing icon DOM elements to prevent animation restart - Extract icon selector dynamically from jQuery object classes - Use .empty() + .append() instead of .html() for better performance - Reuse existing icons when updating progress text (250ms interval) - Simplify dfm_footer signature by removing redundant iconSelector param This ensures smooth spinning animation during file operations without constant restarts, improving visual experience for users. --- emhttp/plugins/dynamix/BrowseButton.page | 85 +++++++++++++++++++++--- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index ca2b42e7ef..2da2523da9 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -52,7 +52,7 @@ const dfm = { }; var dfm_read = {}; -function dfm_footer(action, text) { +function dfm_footer(action, text, $icon) { switch (action) { case 'show': $('#user-notice').show(); @@ -61,8 +61,35 @@ function dfm_footer(action, text) { $('#user-notice').hide(); break; case 'write': - let icon = ''; - $('#user-notice').html(icon + text); + let fileManagerIcon = ''; + let $notice = $('#user-notice'); + + // Ensure text is a string + text = text || ''; + + // Add icon and text + if ($icon && text) { + + // Extract selector from icon classes + let iconSelector = '.' + $icon.attr('class').split(' ').join('.'); + + // Check if icon already exists in DOM to preserve animation + let $existingIcon = $notice.find(iconSelector); + + // Reuse existing icon element, update only text + if ($existingIcon.length > 0) { + $notice.empty().append(fileManagerIcon).append($existingIcon).append(document.createTextNode(text)); + + // First time - insert icon + text + } else { + $notice.empty().append(fileManagerIcon).append($icon.clone()).append(document.createTextNode(text)); + } + + // No icon or no text - insert as-is + } else { + $notice.html(fileManagerIcon + text); + } + break; case 'clear': $('#user-notice').html(''); @@ -121,14 +148,54 @@ function dfm_showProgress(data) { // Universal JSON format: {action: int, text: [text0, text1]} // text[0] = file/main text, text[1] = progress info (optional) if (parsed.action !== undefined && Array.isArray(parsed.text)) { - let icon = ""; - let footer = icon + (parsed.text[1] || parsed.text[0] || ''); - let dialog = parsed.text[0] ? (icon + parsed.text[0] + (parsed.text[1] ? '
' + footer : '')) : footer; - dfm.window.find('.dfm_text').html(dialog); - dfm_footer('write', footer); - dfm.previous = footer; + // Define spinning icon as jQuery object + let $icon = $('').addClass('fa fa-circle-o-notch fa-spin dfm'); + + // Extract selector from icon classes + let iconSelector = '.' + $icon.attr('class').split(' ').join('.'); + + // Build dialog text - preserve existing icon if present to avoid animation restart + let $dialogContainer = dfm.window.find('.dfm_text'); + let dialogText = parsed.text[0] || ''; + if (dialogText) { + + // Find existing icon elements (there might be two: one for file, one for progress) + let $existingIcons = $dialogContainer.find(iconSelector); + if ($existingIcons.length > 0) { + + // Update content while preserving icon elements + let $firstIcon = $existingIcons.first(); + $dialogContainer.empty().append($firstIcon).append(document.createTextNode(dialogText)); + + // Add progress line with its own icon (reuse second or clone first) + if (parsed.text[1]) { + let $secondIcon = $existingIcons.length > 1 ? $existingIcons.eq(1) : $icon.clone(); + $dialogContainer.append('
').append($secondIcon).append(document.createTextNode(parsed.text[1])); + } + + // First time - insert icon + text + } else { + $dialogContainer.empty().append($icon.clone()).append(document.createTextNode(dialogText)); + + // Optionally add second line of icon + text + if (parsed.text[1]) { + $dialogContainer.append('
').append($icon.clone()).append(document.createTextNode(parsed.text[1])); + } + + } + + // No text - clear everything including icon + } else { + $dialogContainer.empty(); + } + + // Build footer text (progress info only) + let footerText = parsed.text[1] || parsed.text[0] || ''; + dfm_footer('write', footerText, $icon); + dfm.previous = footerText; return 0; + } } catch(e) { // Not JSON, fallback to old string parsing From 07d4e067db7000c23603017a2d56ef0e73331477 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 17:18:31 +0100 Subject: [PATCH 25/39] Replace   with whitespace-nowrap utility class - Remove   HTML entities from file_manager progress text - Use Unraid's built-in .whitespace-nowrap utility class instead - Wrap progress info in span elements with whitespace-nowrap - Add jQuery object validation in dfm_footer for robustness - Preserve smooth icon animation for new JSON-based operations - Legacy operations still work but with flickering (migration incentive) --- emhttp/plugins/dynamix/BrowseButton.page | 52 +++++++++++++++++++---- emhttp/plugins/dynamix/nchan/file_manager | 2 +- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index 2da2523da9..ef33d20371 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -70,6 +70,12 @@ function dfm_footer(action, text, $icon) { // Add icon and text if ($icon && text) { + // Ensure $icon is a jQuery object + if (!($icon instanceof jQuery)) { + $notice.html(fileManagerIcon + text); + break; + } + // Extract selector from icon classes let iconSelector = '.' + $icon.attr('class').split(' ').join('.'); @@ -78,11 +84,22 @@ function dfm_footer(action, text, $icon) { // Reuse existing icon element, update only text if ($existingIcon.length > 0) { - $notice.empty().append(fileManagerIcon).append($existingIcon).append(document.createTextNode(text)); + // Preserve icon in DOM, only update text nodes + $notice.contents().filter(function() { + return this.nodeType === 3; // Text nodes only + }).remove(); + + // Remove existing progress span if present + $notice.find('.whitespace-nowrap').remove(); + + // Update text after icon (which stays in place) + let $progressSpan = $('').addClass('whitespace-nowrap').text(text); + $existingIcon.after($progressSpan); // First time - insert icon + text } else { - $notice.empty().append(fileManagerIcon).append($icon.clone()).append(document.createTextNode(text)); + let $progressSpan = $('').addClass('whitespace-nowrap').text(text); + $notice.empty().append(fileManagerIcon).append($icon.clone()).append($progressSpan); } // No icon or no text - insert as-is @@ -164,14 +181,30 @@ function dfm_showProgress(data) { let $existingIcons = $dialogContainer.find(iconSelector); if ($existingIcons.length > 0) { - // Update content while preserving icon elements - let $firstIcon = $existingIcons.first(); - $dialogContainer.empty().append($firstIcon).append(document.createTextNode(dialogText)); + // Preserve icons in DOM, only update text nodes + $dialogContainer.contents().filter(function() { + return this.nodeType === 3; // Text nodes only + }).remove(); + + // Remove
if present + $dialogContainer.find('br').remove(); + + // Insert updated text after first icon (which stays in place) + $existingIcons.first().after(document.createTextNode(dialogText)); - // Add progress line with its own icon (reuse second or clone first) + // Add progress line with its own icon (reuse or create second icon) if (parsed.text[1]) { - let $secondIcon = $existingIcons.length > 1 ? $existingIcons.eq(1) : $icon.clone(); - $dialogContainer.append('
').append($secondIcon).append(document.createTextNode(parsed.text[1])); + let $secondIcon = $existingIcons.eq(1); + if ($secondIcon.length === 0) { + // Need to add second icon + let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); + $dialogContainer.append('
').append($icon.clone()).append($progressSpan); + } else { + // Second icon exists, just update text + $secondIcon.before('
'); + let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); + $secondIcon.after($progressSpan); + } } // First time - insert icon + text @@ -180,7 +213,8 @@ function dfm_showProgress(data) { // Optionally add second line of icon + text if (parsed.text[1]) { - $dialogContainer.append('
').append($icon.clone()).append(document.createTextNode(parsed.text[1])); + let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); + $dialogContainer.append('
').append($icon.clone()).append($progressSpan); } } diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 72c9739f94..cfae622cc6 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -76,7 +76,7 @@ function parse_rsync_progress($status, $action_label) { if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { - $text[1] = _('Completed') . ": " . $parts[1] . ",  " . _('Speed') . ": " . $parts[2] . ",  " . _('ETA') . ": " . $parts[3]; + $text[1] = _('Completed') . ": " . $parts[1] . ", " . _('Speed') . ": " . $parts[2] . ", " . _('ETA') . ": " . $parts[3]; } } From 46b3d5d951ed0ac78307417e78cdbbc6d645b8e7 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 17:34:39 +0100 Subject: [PATCH 26/39] Remove unnecessary nowrap spans, simplify DOM manipulation - Remove whitespace-nowrap class usage from progress text - Simplify text insertion using direct TextNodes without span wrappers - Remove span cleanup code - Keep icon preservation logic for smooth animations - Text wrapping left to browser defaults (not an issue in practice) --- emhttp/plugins/dynamix/BrowseButton.page | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index ef33d20371..4857926fab 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -89,17 +89,12 @@ function dfm_footer(action, text, $icon) { return this.nodeType === 3; // Text nodes only }).remove(); - // Remove existing progress span if present - $notice.find('.whitespace-nowrap').remove(); - // Update text after icon (which stays in place) - let $progressSpan = $('').addClass('whitespace-nowrap').text(text); - $existingIcon.after($progressSpan); + $existingIcon.after(document.createTextNode(text)); // First time - insert icon + text } else { - let $progressSpan = $('').addClass('whitespace-nowrap').text(text); - $notice.empty().append(fileManagerIcon).append($icon.clone()).append($progressSpan); + $notice.empty().append(fileManagerIcon).append($icon.clone()).append(document.createTextNode(text)); } // No icon or no text - insert as-is @@ -186,7 +181,7 @@ function dfm_showProgress(data) { return this.nodeType === 3; // Text nodes only }).remove(); - // Remove
if present + // Remove
elements $dialogContainer.find('br').remove(); // Insert updated text after first icon (which stays in place) @@ -197,13 +192,11 @@ function dfm_showProgress(data) { let $secondIcon = $existingIcons.eq(1); if ($secondIcon.length === 0) { // Need to add second icon - let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); - $dialogContainer.append('
').append($icon.clone()).append($progressSpan); + $dialogContainer.append('
').append($icon.clone()).append(document.createTextNode(parsed.text[1])); } else { // Second icon exists, just update text $secondIcon.before('
'); - let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); - $secondIcon.after($progressSpan); + $secondIcon.after(document.createTextNode(parsed.text[1])); } } @@ -213,8 +206,7 @@ function dfm_showProgress(data) { // Optionally add second line of icon + text if (parsed.text[1]) { - let $progressSpan = $('').addClass('whitespace-nowrap').text(parsed.text[1]); - $dialogContainer.append('
').append($icon.clone()).append($progressSpan); + $dialogContainer.append('
').append($icon.clone()).append(document.createTextNode(parsed.text[1])); } } From 737a51515198997f75785afb4047a6ed9d5a29b6 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 17:45:52 +0100 Subject: [PATCH 27/39] Add validation for empty jQuery objects in icon handling - Check .length === 0 to prevent TypeError on .attr('class') - Prevents issues when empty jQuery object is passed (e.g., $('.nonexistent')) --- emhttp/plugins/dynamix/BrowseButton.page | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index 4857926fab..ec151e761c 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -70,8 +70,8 @@ function dfm_footer(action, text, $icon) { // Add icon and text if ($icon && text) { - // Ensure $icon is a jQuery object - if (!($icon instanceof jQuery)) { + // Ensure $icon is a jQuery object with elements + if (!($icon instanceof jQuery) || $icon.length === 0) { $notice.html(fileManagerIcon + text); break; } @@ -180,7 +180,7 @@ function dfm_showProgress(data) { $dialogContainer.contents().filter(function() { return this.nodeType === 3; // Text nodes only }).remove(); - + // Remove
elements $dialogContainer.find('br').remove(); From 4b90b9a31ab31bcca564a4f6b54776c99827e77b Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 18:04:52 +0100 Subject: [PATCH 28/39] Remove unnecessary jQuery validation checks - Trust developers to call dfm_footer() with correct parameters - Cleaner code without defensive over-engineering - Clear error messages if function is called incorrectly --- emhttp/plugins/dynamix/BrowseButton.page | 6 ------ 1 file changed, 6 deletions(-) diff --git a/emhttp/plugins/dynamix/BrowseButton.page b/emhttp/plugins/dynamix/BrowseButton.page index ec151e761c..f1a88f4d54 100644 --- a/emhttp/plugins/dynamix/BrowseButton.page +++ b/emhttp/plugins/dynamix/BrowseButton.page @@ -70,12 +70,6 @@ function dfm_footer(action, text, $icon) { // Add icon and text if ($icon && text) { - // Ensure $icon is a jQuery object with elements - if (!($icon instanceof jQuery) || $icon.length === 0) { - $notice.html(fileManagerIcon + text); - break; - } - // Extract selector from icon classes let iconSelector = '.' + $icon.attr('class').split(' ').join('.'); From ef01699083bcc9bd5d5a20e3ff8b0dec8543a5d2 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 18:21:04 +0100 Subject: [PATCH 29/39] Add comment explaining --no-inc-recursive flag usage - Explain why rsync's default inc_recurse mode is problematic - Document temporary root:root 0700 permissions issue - Clarify impact on /mnt/user access for Unraid users --- emhttp/plugins/dynamix/nchan/file_manager | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index cfae622cc6..7d9103cac8 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -377,6 +377,9 @@ while (true) { } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + // note: rsync defaults to "inc_recurse mode" which causes creating the whole directory structure at target before transfering files. + // This is a problem because those target directories get temporary permissions of "root:root" and "chmod 0700", which + // breaks access for all unraid users through /mnt/user/sharename. We avoid this behavior by using "--no-inc-recursive". exec($cmd, $pid); } From d764b1e823f682e1edafa57ea927048442037040 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 20:39:35 +0100 Subject: [PATCH 30/39] Calculate ETA when rsync only shows elapsed time - Added calculate_eta() function to compute ETA from transferred size, percent, and speed - Modified parse_rsync_progress() to detect elapsed time lines (with xfr# info) - When rsync shows elapsed time instead of ETA, calculate our own ETA - Ensures consistent ETA display instead of jumping between ETA and elapsed time - ETA calculation accuracy: ~20-30 seconds difference vs rsync (due to percent rounding) --- emhttp/plugins/dynamix/nchan/file_manager | 52 ++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 7d9103cac8..034d6753e2 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -61,6 +61,44 @@ function mb_strimhalf($text, $width, $trim_marker = "") { return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -1 - $half); } +function calculate_eta($transferred, $percent, $speed) { + // Convert transferred size to bytes + $multipliers = ['K' => 1024, 'M' => 1024*1024, 'G' => 1024*1024*1024, 'T' => 1024*1024*1024*1024]; + $transferred_bytes = floatval($transferred); + foreach ($multipliers as $unit => $mult) { + if (stripos($transferred, $unit) !== false) { + $transferred_bytes *= $mult; + break; + } + } + + // Convert speed to bytes/sec + $speed_bytes = floatval($speed); + if (stripos($speed, 'kB/s') !== false) { + $speed_bytes *= 1024; + } elseif (stripos($speed, 'MB/s') !== false) { + $speed_bytes *= 1024 * 1024; + } elseif (stripos($speed, 'GB/s') !== false) { + $speed_bytes *= 1024 * 1024 * 1024; + } + + // Calculate total size from percent + $percent_val = intval(str_replace('%', '', $percent)); + if ($percent_val > 0 && $percent_val < 100 && $speed_bytes > 0) { + $total_bytes = $transferred_bytes * 100 / $percent_val; + $remaining_bytes = $total_bytes - $transferred_bytes; + $eta_seconds = intval($remaining_bytes / $speed_bytes); + + // Format as HH:MM:SS + $hours = intval($eta_seconds / 3600); + $minutes = intval(($eta_seconds % 3600) / 60); + $seconds = $eta_seconds % 60; + return sprintf("%d:%02d:%02d", $hours, $minutes, $seconds); + } + + return "N/A"; +} + function parse_rsync_progress($status, $action_label) { // initialize text array with action label @@ -76,7 +114,19 @@ function parse_rsync_progress($status, $action_label) { if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { - $text[1] = _('Completed') . ": " . $parts[1] . ", " . _('Speed') . ": " . $parts[2] . ", " . _('ETA') . ": " . $parts[3]; + $transferred = $parts[0]; + $percent = $parts[1]; + $speed = $parts[2]; + $time = $parts[3]; + + // Check if this is an ETA line or elapsed time line + // ETA lines have only 4 parts, elapsed time lines have additional (xfr#...) info + if (isset($parts[4])) { + // Calculate our own ETA from transferred size, percent, and speed + $time = calculate_eta($transferred, $percent, $speed); + } + + $text[1] = _('Completed') . ": " . $percent . ", " . _('Speed') . ": " . $speed . ", " . _('ETA') . ": " . $time; } } From 6a4ec6ee3e236f4c1379e8b2d5658ad9249d1fba Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 26 Oct 2025 21:33:23 +0100 Subject: [PATCH 31/39] Add ETA hysteresis to smooth fluctuations - Added time_to_seconds() and seconds_to_time() helper functions - Modified calculate_eta() to use hysteresis (70% last rsync ETA, 30% calculated) - Static variable in parse_rsync_progress() stores last known rsync ETA - At 0% progress: use last known ETA if available, otherwise N/A - Prevents ETA from jumping wildly when rsync alternates between ETA and elapsed time - Smooths out rounding errors at low percentages --- emhttp/plugins/dynamix/nchan/file_manager | 60 +++++++++++++++++------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 034d6753e2..64f64947f1 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -61,7 +61,21 @@ function mb_strimhalf($text, $width, $trim_marker = "") { return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -1 - $half); } -function calculate_eta($transferred, $percent, $speed) { +function time_to_seconds($time_str) { + if (!preg_match('/^(\d+):(\d+):(\d+)$/', $time_str, $matches)) { + return null; + } + return intval($matches[1]) * 3600 + intval($matches[2]) * 60 + intval($matches[3]); +} + +function seconds_to_time($seconds) { + $hours = intval($seconds / 3600); + $minutes = intval(($seconds % 3600) / 60); + $secs = $seconds % 60; + return sprintf("%d:%02d:%02d", $hours, $minutes, $secs); +} + +function calculate_eta($transferred, $percent, $speed, $last_rsync_eta_seconds = null) { // Convert transferred size to bytes $multipliers = ['K' => 1024, 'M' => 1024*1024, 'G' => 1024*1024*1024, 'T' => 1024*1024*1024*1024]; $transferred_bytes = floatval($transferred); @@ -71,7 +85,7 @@ function calculate_eta($transferred, $percent, $speed) { break; } } - + // Convert speed to bytes/sec $speed_bytes = floatval($speed); if (stripos($speed, 'kB/s') !== false) { @@ -81,25 +95,40 @@ function calculate_eta($transferred, $percent, $speed) { } elseif (stripos($speed, 'GB/s') !== false) { $speed_bytes *= 1024 * 1024 * 1024; } - + // Calculate total size from percent $percent_val = intval(str_replace('%', '', $percent)); + + // At 0%, we cannot calculate ETA reliably from transferred/percent + // but we can use the last known rsync ETA if available + if ($percent_val == 0) { + if ($last_rsync_eta_seconds !== null && $last_rsync_eta_seconds > 0) { + return seconds_to_time($last_rsync_eta_seconds); + } + return "N/A"; + } + if ($percent_val > 0 && $percent_val < 100 && $speed_bytes > 0) { $total_bytes = $transferred_bytes * 100 / $percent_val; $remaining_bytes = $total_bytes - $transferred_bytes; - $eta_seconds = intval($remaining_bytes / $speed_bytes); + $calculated_eta_seconds = intval($remaining_bytes / $speed_bytes); - // Format as HH:MM:SS - $hours = intval($eta_seconds / 3600); - $minutes = intval(($eta_seconds % 3600) / 60); - $seconds = $eta_seconds % 60; - return sprintf("%d:%02d:%02d", $hours, $minutes, $seconds); + // Apply hysteresis: blend with last known rsync ETA to smooth out fluctuations + if ($last_rsync_eta_seconds !== null && $last_rsync_eta_seconds > 0) { + // Weight: 70% last rsync ETA, 30% calculated ETA + $eta_seconds = intval($last_rsync_eta_seconds * 0.7 + $calculated_eta_seconds * 0.3); + } else { + $eta_seconds = $calculated_eta_seconds; + } + + return seconds_to_time($eta_seconds); } - + return "N/A"; } function parse_rsync_progress($status, $action_label) { + static $last_rsync_eta_seconds = null; // initialize text array with action label $text[0] = $action_label . "... "; @@ -118,14 +147,17 @@ function parse_rsync_progress($status, $action_label) { $percent = $parts[1]; $speed = $parts[2]; $time = $parts[3]; - + // Check if this is an ETA line or elapsed time line // ETA lines have only 4 parts, elapsed time lines have additional (xfr#...) info if (isset($parts[4])) { - // Calculate our own ETA from transferred size, percent, and speed - $time = calculate_eta($transferred, $percent, $speed); + // Elapsed time line - calculate our own ETA with hysteresis + $time = calculate_eta($transferred, $percent, $speed, $last_rsync_eta_seconds); + } else { + // ETA line from rsync - store it for hysteresis + $last_rsync_eta_seconds = time_to_seconds($time); } - + $text[1] = _('Completed') . ": " . $percent . ", " . _('Speed') . ": " . $speed . ", " . _('ETA') . ": " . $time; } } From d8e1034202bf45e138cf1311074ac019d1814f71 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Tue, 28 Oct 2025 00:14:43 +0100 Subject: [PATCH 32/39] Small rsync ETA fix and improved regex - tac --before prevents line concatenation, when status file randomly lacks trailing \n - Add leading space (^ ) to regex which guarantess matching the progress line --- emhttp/plugins/dynamix/nchan/file_manager | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 64f64947f1..813de33808 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -139,7 +139,9 @@ function parse_rsync_progress($status, $action_label) { // transfer of single file finished (ETA changes temporarily to elapsed time): // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines - $progress_line = trim(exec("tac $status | grep -m1 -E '[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); + // note: leading space in regex ensures we only match progress lines, not filenames + // note: tac --before prevents line concatenation when $status lacks trailing \n + $progress_line = trim(exec("tac --before $status | grep -m1 -E '^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); if ($progress_line) { $parts = explode(' ', $progress_line); if (count($parts) >= 4) { @@ -165,7 +167,7 @@ function parse_rsync_progress($status, $action_label) { // obtain filename line like: // "/mnt/disk6/images/image.jpg" // note: -v is used to obtain the opposite of the progress line regex and to filter out empty lines - $file_line = exec("tac $status | grep -m1 -v -E '(^\$|[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tr -s ' ' || echo ''"); + $file_line = exec("tail -n5 $status | grep -v -E '(^\$|^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tail -1 | tr -s ' ' || echo ''"); if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); } From dc180a86b39835befe1d0cc1a9201d4981c72e0e Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Tue, 28 Oct 2025 01:37:22 +0100 Subject: [PATCH 33/39] Revert wrong test to extract file line of status file --- emhttp/plugins/dynamix/nchan/file_manager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 813de33808..3d488b3420 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -167,7 +167,7 @@ function parse_rsync_progress($status, $action_label) { // obtain filename line like: // "/mnt/disk6/images/image.jpg" // note: -v is used to obtain the opposite of the progress line regex and to filter out empty lines - $file_line = exec("tail -n5 $status | grep -v -E '(^\$|^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tail -1 | tr -s ' ' || echo ''"); + $file_line = exec("tac $status | grep -v -E '(^\$|^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tr -s ' ' || echo ''"); if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); } From bd271c73f496250ea146fd87fda5b3f76f34e5b1 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Tue, 28 Oct 2025 01:53:57 +0100 Subject: [PATCH 34/39] Add -m1 to file_line grep for performance Stop after first match to avoid processing entire file --- emhttp/plugins/dynamix/nchan/file_manager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 3d488b3420..66698431b7 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -167,7 +167,7 @@ function parse_rsync_progress($status, $action_label) { // obtain filename line like: // "/mnt/disk6/images/image.jpg" // note: -v is used to obtain the opposite of the progress line regex and to filter out empty lines - $file_line = exec("tac $status | grep -v -E '(^\$|^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tr -s ' ' || echo ''"); + $file_line = exec("tac $status | grep -m1 -v -E '(^\$|^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+)' | tr -s ' ' || echo ''"); if ($file_line) { $text[0] .= htmlspecialchars(mb_strimhalf($file_line, 70, '...'), ENT_QUOTES, 'UTF-8'); } From dada8c4da03488a8114c8af13e4823ed72897325 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:51:14 +0100 Subject: [PATCH 35/39] Fix subdirectory check for move operations The subdirectory check was incorrectly using dirname() on source directories, causing false positives. For example, moving /mnt/disk7/Backups/Smartphone to /mnt/disk7/Backups/Marc/ would fail because dirname() reduced the source to /mnt/disk7/Backups, making it appear that Marc was a subdirectory of the source's parent. Now correctly checks: - For directories: use the directory path itself - For files: use the containing directory (dirname) This ensures the check properly prevents moving a directory into its own subdirectories while allowing valid moves to sibling directories. --- emhttp/plugins/dynamix/nchan/file_manager | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 66698431b7..3b45b2fdd9 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -397,7 +397,7 @@ while (true) { if (!empty($target_device_id)) { $use_rsync_rename = true; // assume we can use it, then check for disqualifying conditions - foreach ($source as $source_path) { + foreach ($source as $source_path) { // this can be a file or directory // source path must be valid $valid_source_path = validname($source_path); @@ -437,11 +437,11 @@ while (true) { } // target must not be a subdirectory of any source (backup-dir should be outside source tree) - $parent_dir = dirname($valid_source_path); - if (strpos(rtrim($target,'/') . '/', rtrim($parent_dir,'/') . '/') === 0) { + $source_dirname = is_dir($valid_source_path) ? $valid_source_path : dirname($valid_source_path); + if (strpos(rtrim($target,'/') . '/', rtrim($source_dirname,'/') . '/') === 0) { $reply['error'] = 'Cannot move directory into its own subdirectory'; $use_rsync_rename = false; - break 2; // break out of both foreach and if + break 2; // break out of both: foreach and case } } From e2f53a664d96893a20034b415bc73fa11146d330 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:59:41 +0100 Subject: [PATCH 36/39] Improve mb_strimhalf and calculate_eta functions mb_strimhalf: - Fixed substring calculation for right side (was -1-$half, now -$half) - Properly account for marker length in width calculation to ensure output never exceeds specified width calculate_eta: - Add minimum threshold check (> 1.0 instead of > 0) to avoid unrealistically large ETAs from very small speed values --- emhttp/plugins/dynamix/nchan/file_manager | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 3b45b2fdd9..03bbcfc1c6 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -57,8 +57,10 @@ function isdir($name) { function mb_strimhalf($text, $width, $trim_marker = "") { if (mb_strlen($text) <= $width) return $text; if ($width < 8) return mb_substr($text, 0, $width); - $half = (int)($width / 2) - 2; - return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -1 - $half); + $marker_len = mb_strlen($trim_marker); + $available = $width - $marker_len; + $half = (int)($available / 2); + return mb_substr($text, 0, $half) . $trim_marker . mb_substr($text, -($available - $half)); } function time_to_seconds($time_str) { @@ -108,11 +110,11 @@ function calculate_eta($transferred, $percent, $speed, $last_rsync_eta_seconds = return "N/A"; } - if ($percent_val > 0 && $percent_val < 100 && $speed_bytes > 0) { + if ($percent_val > 0 && $percent_val < 100 && $speed_bytes > 1.0) { $total_bytes = $transferred_bytes * 100 / $percent_val; $remaining_bytes = $total_bytes - $transferred_bytes; $calculated_eta_seconds = intval($remaining_bytes / $speed_bytes); - + // Apply hysteresis: blend with last known rsync ETA to smooth out fluctuations if ($last_rsync_eta_seconds !== null && $last_rsync_eta_seconds > 0) { // Weight: 70% last rsync ETA, 30% calculated ETA From f0d0c288df8a0a78b8ab0446efd6870a29714757 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Wed, 29 Oct 2025 22:46:11 +0100 Subject: [PATCH 37/39] Remove 'deleting' prefix from rsync rename status output The rsync rename operation uses --delete flag which outputs 'deleting filename/' in the status. This could cause user confusion as it looks like files are being deleted rather than moved. Filter out the 'deleting ' prefix with sed before writing to status file to show a cleaner 'Moving... filename/' message. --- emhttp/plugins/dynamix/nchan/file_manager | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 03bbcfc1c6..93a782c5be 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -454,9 +454,10 @@ while (true) { // notes: // - existing files are overwritten in --backup-dir (like not using --ignore-existing) // - missing directories are created in --backup-dir (like using --mkpath) + // - rsync prefixes the moved files with "deleting " in the output, which we strip with sed, to not confuse the user if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); - $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + $cmd = "rsync -r --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --delete --backup --backup-dir=".escapeshellarg($target)." ".quoted_rsync_include($source)." --exclude='*' ".escapeshellarg($empty_dir)." ".escapeshellarg($parent_dir)." > >(stdbuf -o0 tr '\\r' '\\n' | sed 's/^deleting //' >$status) 2>$error & echo \$!"; exec($cmd, $pid); // use rsync copy-delete From 59c1615fa84b8ead3e7381c1e47a61279c9c60b9 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sat, 1 Nov 2025 12:12:36 +0100 Subject: [PATCH 38/39] Add --no-inc-recursive flag to rsync copy operations - Prevents rsync from creating directory structure with temporary root:root permissions - Ensures proper access for unraid users through /mnt/user/sharename - Applied to both copy and move operations for consistency --- emhttp/plugins/dynamix/nchan/file_manager | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 93a782c5be..d5dac2a9b5 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -338,7 +338,10 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + // note: rsync defaults to "inc_recurse mode" which causes creating the whole directory structure at target before transfering files. + // This is a problem because those target directories get temporary permissions of "root:root" and "chmod 0700", which + // breaks access for all unraid users through /mnt/user/sharename. We avoid this behavior by using "--no-inc-recursive". exec($cmd, $pid); } else { $reply['error'] = 'Invalid target name'; From 38563695d26c94857ab05dcb3a6d50c7f0a4f0b8 Mon Sep 17 00:00:00 2001 From: mgutt <10757176+mgutt@users.noreply.github.com> Date: Sun, 2 Nov 2025 14:32:53 +0100 Subject: [PATCH 39/39] improve comments in parse_rsync_progress function - clarify that timestamps represent ETA vs elapsed time - improve filename vs progress line distinction in regex comment - remove unnecessary empty line --- emhttp/plugins/dynamix/nchan/file_manager | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index d5dac2a9b5..810df8c785 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -136,12 +136,12 @@ function parse_rsync_progress($status, $action_label) { $text[0] = $action_label . "... "; // obtain progress line like: - // currently running (showing total ETA): + // currently running (timestamp represents total ETA): // " 37.91G 4% 58.59MB/s 3:47:20" - // transfer of single file finished (ETA changes temporarily to elapsed time): + // transfer of single file finished (note: timestamp now represents total elapsed time): // " 37.93G 4% 53.80MB/s 0:11:12 (xfr#32, to-chk=554/596)" // note: we do not loop with PHP as there could be a huge amount of progress lines - // note: leading space in regex ensures we only match progress lines, not filenames + // note: leading space in regex ensures we only match progress lines, not filename lines // note: tac --before prevents line concatenation when $status lacks trailing \n $progress_line = trim(exec("tac --before $status | grep -m1 -E '^ .*[0-9]+%.*[0-9]+:[0-9]+:[0-9]+' | tr -s ' ' || echo ''")); if ($progress_line) { @@ -266,7 +266,6 @@ if (!file_exists($empty_dir)) { // initialize $delete_empty_dirs state: null = not a move operation (yet), true = rsync copy-delete phase, false = cleanup phase (done) $delete_empty_dirs = null; - // infinite loop to monitor and execute file operations // Note: exec() uses /bin/sh which is symlinked to bash in unraid and a requirement for process substitution syntax >(...) while (true) {