diff --git a/.check/VerifySourceCalls.py b/.check/VerifySourceCalls.py index c93a12f..d87cdf9 100644 --- a/.check/VerifySourceCalls.py +++ b/.check/VerifySourceCalls.py @@ -340,26 +340,49 @@ def fix_script( lines_to_insert.append(shellcheck_line) lines_to_insert.append(source_line) - # 3) Rebuild lines, skipping the ones to remove (including their shellcheck lines) + # 3) Rebuild lines, skipping the ones to remove (including their shellcheck lines and error handlers) new_lines = [] - skip_next = False + skip_mode = None # Can be None, 'source', or 'error_handler' + brace_depth = 0 + for i, line in enumerate(lines): - # Check if we should skip this line because it's a shellcheck directive for a removed source - if skip_next: - skip_next = False + stripped = line.strip() + + # Handle skip modes + if skip_mode == 'source': + # We're skipping a source line - check if it has error handler + if '||' in line and '{' in line: + # Multi-line source with error handler - enter error handler skip mode + skip_mode = 'error_handler' + brace_depth = line.count('{') - line.count('}') + continue + else: + # Single-line source - done skipping + skip_mode = None + continue + + elif skip_mode == 'error_handler': + # Count braces to find end of error handler block + brace_depth += line.count('{') - line.count('}') + if brace_depth <= 0: + # Error handler block complete + skip_mode = None continue - + # Check if this is a shellcheck directive for a source we're removing - stripped = line.strip() if stripped in remove_shellcheck: - skip_next = True # Skip the next line (the actual source statement) + skip_mode = 'source' # Skip the next line (and possibly error handler) continue - # Check if line is a source to remove + # Check if line is a source to remove (fallback for sources without shellcheck directive) if should_remove_source_line(line, remove_lines, remove_lines_dot): - # Also check if previous line was a shellcheck directive + # Check if it has error handler + if '||' in line and '{' in line: + skip_mode = 'error_handler' + brace_depth = line.count('{') - line.count('}') + # Also remove previous shellcheck directive if present if i > 0 and new_lines and new_lines[-1].strip().startswith('# shellcheck source='): - new_lines.pop() # Remove the shellcheck directive too + new_lines.pop() continue # We'll insert new sources at the insertion index diff --git a/.check/_RunChecks.sh b/.check/_RunChecks.sh index 36105ae..a54bee1 100644 --- a/.check/_RunChecks.sh +++ b/.check/_RunChecks.sh @@ -192,7 +192,7 @@ if [ "$NO_FIX" = true ]; then cat "$TEMP_OUTPUT" else # Show summary only - sed -n '/^====/p; /^Scripts found:/p; /^Summary/,/^====/p' "$TEMP_OUTPUT" + sed -n '/^Scripts found:/p; /^Summary/,/^ ✓/p' "$TEMP_OUTPUT" fi if [ $EXIT_CODE -eq 0 ]; then @@ -216,7 +216,7 @@ else cat "$TEMP_OUTPUT" else # Show fixed files and summary - grep -E "^✓|^===|^Scripts found:|^Summary|^ ✓|^ ✗" "$TEMP_OUTPUT" || true + grep -E "^✓|^Scripts found:|^Summary|^ ✓|^ ✗" "$TEMP_OUTPUT" || true fi echo "- Script notes validated and fixed" diff --git a/.gitignore b/.gitignore index 126d5ee..34dc0f5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ nodes.json TestConnectionInfo.json .d/ .nfs* +*/__pycache__/ +*/__pycache__/* +__pycache__/ +__pycache__/* # Downloaded Proxmox documentation (keep scripts, ignore generated content) .docs/*.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e7193..574d574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,62 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.8] - 2026-01-08 + +Critical bug fixes for bulk operations and performance optimizations + +### Fixed +- **CRITICAL** - Fixed Bash dynamic scoping bug causing infinite recursion in bulk operations +- **CRITICAL** - Fixed potential infinite polling in status wait functions +- Fixed wasteful waiting when VM stop fails +- Fixed temp file naming conflict in GUI.sh sessions +- Added support for custom SSH ports +- Fixed random node ordering in GUI +- Cleaned up error handling in RemoteExecutor +- Fixed TRACE logs appearing when LOG_LEVEL=INFO + +### Changed +- Reduced redundant queries in bulk operations +- Added comprehensive logging to bulk operations framework + +### Added +- Added connection testing before script execution + +### Known Issues +- **Script Cancellation** - Cannot cancel scripts when connection is dropped + - Possibly add method to automatically stop scripts when the SSH connection breaks + +### Technical Details +- Added TRACE log level support with priority 0 +- Modified `__vm_exists__`/`__ct_exists__` to accept `--get-node` flag, refactored 13 operation functions, hardened `__vm_wait_for_status__` and `__ct_wait_for_status__` with timeout protection +- Fixed callback parameter passing, added trace logging, updated tmpdir naming +- Added `__test_remote_connection__` function, port parameter to SSH/SCP functions, updated log file naming +- Added `NODE_ORDER` array and `__get_node_port__` function +- Added password validation in all remote configuration flows +- Added port field in `nodes.json.template` + +## [2.1.7] - 2025-11-25 + +Bug fix for VerifySourceCalls.py validation tool, + +### Fixed +- **Bug: VerifySourceCalls.py Multi-line Source Removal Bug** + - Fixed logic that caused orphaned error handler blocks when removing unused sources + - Tool previously only skipped ONE line after shellcheck directive + - Now properly handles multi-line source statements with error handlers (`|| { ... }`) + - Implements brace-depth tracking to remove entire error handler blocks + - This bug was the root cause of the HostInfo.sh orphaned code issue discovered in v2.1.6 + +### Changed +- **Startup Dependency Check** - Added informational dependency warning at GUI.sh startup + - Shows friendly warning if `jq` is not installed + - Provides installation commands for all major distributions + - Non-blocking - allows local execution without optional dependencies + - Complements the existing remote execution dependency check + ## [2.1.6] - 2025-11-25 -Bug fixes, username support, and validation improvements +Critical bug fixes, username support, and validation improvements ### Added - **Username Configuration** - Support for specifying SSH usernames per node diff --git a/GUI.sh b/GUI.sh index c01d861..3bc2c88 100644 --- a/GUI.sh +++ b/GUI.sh @@ -486,6 +486,18 @@ configure_single_remote() { echo "SSH keys not detected, password required." read -rsp "Enter password: " manual_pass echo + + # Test connection before proceeding + echo "Testing connection..." + if ! __test_remote_connection__ "$manual_ip" "$manual_pass" "$manual_user" "22"; then + echo + __line_rgb__ "✗ Connection failed! Please check IP, username, and password." 255 0 0 + echo "Press Enter to try again..." + read -r + continue + fi + echo "✓ Connection successful!" + sleep 1 __clear_remote_targets__ __add_remote_target__ "$manual_name" "$manual_ip" "$manual_pass" "$manual_user" @@ -571,6 +583,18 @@ configure_single_remote() { echo "SSH keys not detected, password required." read -rsp "Enter password: " manual_pass echo + + # Test connection before proceeding + echo "Testing connection..." + if ! __test_remote_connection__ "$manual_ip" "$manual_pass" "$manual_user" "22"; then + echo + __line_rgb__ "✗ Connection failed! Please check IP, username, and password." 255 0 0 + echo "Press Enter to try again..." + read -r + continue + fi + echo "✓ Connection successful!" + sleep 1 __clear_remote_targets__ __add_remote_target__ "$manual_name" "$manual_ip" "$manual_pass" "$manual_user" @@ -599,6 +623,22 @@ configure_single_remote() { echo "SSH keys not detected, password required." read -rsp "Enter password for $selected_name: " node_pass echo + + # Get port for this node + local selected_port + selected_port=$(__get_node_port__ "$selected_name") + + # Test connection before proceeding + echo "Testing connection to $selected_name..." + if ! __test_remote_connection__ "$selected_ip" "$node_pass" "$selected_username" "$selected_port"; then + echo + __line_rgb__ "✗ Connection failed! Please check password." 255 0 0 + echo "Press Enter to try again..." + read -r + continue + fi + echo "✓ Connection successful!" + sleep 1 __clear_remote_targets__ __add_remote_target__ "$selected_name" "$selected_ip" "$node_pass" "$selected_username" @@ -806,6 +846,18 @@ configure_multi_saved() { if ! ssh -o BatchMode=yes -o ConnectTimeout=2 "${node_username}@$node_ip" echo "test" &>/dev/null 2>&1; then read -rsp "Enter password for $node_name: " node_pass echo + + # Test connection + local node_port + node_port=$(__get_node_port__ "$node_name") + echo "Testing connection to $node_name..." + if ! __test_remote_connection__ "$node_ip" "$node_pass" "$node_username" "$node_port"; then + echo + __line_rgb__ "✗ Connection to $node_name failed! Skipping this node." 255 0 0 + continue + fi + echo "✓ Connection to $node_name successful!" + NODE_PASSWORDS["$node_name"]="$node_pass" else NODE_PASSWORDS["$node_name"]="" # Empty = use SSH keys @@ -823,6 +875,25 @@ configure_multi_saved() { if [[ "$same_pass" =~ ^[Yy]$ ]]; then read -rsp "Enter password for all nodes: " shared_pass echo + + # Test password on first node + local first_target="${REMOTE_TARGETS[0]}" + IFS=':' read -r first_name first_ip <<<"$first_target" + local first_username="${NODE_USERNAMES[$first_name]:-$DEFAULT_USERNAME}" + local first_port + first_port=$(__get_node_port__ "$first_name") + + echo "Testing password on first node ($first_name)..." + if ! __test_remote_connection__ "$first_ip" "$shared_pass" "$first_username" "$first_port"; then + echo + __line_rgb__ "✗ Password test failed on $first_name!" 255 0 0 + echo "Please try again..." + sleep 2 + continue + fi + echo "✓ Password works on $first_name" + sleep 1 + for target in "${REMOTE_TARGETS[@]}"; do IFS=':' read -r node_name node_ip <<<"$target" NODE_PASSWORDS["$node_name"]="$shared_pass" @@ -830,8 +901,30 @@ configure_multi_saved() { else for target in "${REMOTE_TARGETS[@]}"; do IFS=':' read -r node_name node_ip <<<"$target" + local node_username="${NODE_USERNAMES[$node_name]:-$DEFAULT_USERNAME}" + local node_port + node_port=$(__get_node_port__ "$node_name") + read -rsp "Enter password for $node_name ($node_ip): " node_pass echo + + # Test connection + echo "Testing connection to $node_name..." + if ! __test_remote_connection__ "$node_ip" "$node_pass" "$node_username" "$node_port"; then + echo + __line_rgb__ "✗ Connection to $node_name failed! Please try again." 255 0 0 + # Loop back to retry this specific node + read -rsp "Enter password for $node_name ($node_ip): " node_pass + echo + echo "Testing connection to $node_name..." + if ! __test_remote_connection__ "$node_ip" "$node_pass" "$node_username" "$node_port"; then + echo + __line_rgb__ "✗ Still failed! Skipping $node_name." 255 0 0 + continue + fi + fi + echo "✓ Connection to $node_name successful!" + NODE_PASSWORDS["$node_name"]="$node_pass" done fi diff --git a/Manuals/ssh-proxy-configuration.txt b/Manuals/ssh-proxy-configuration.txt new file mode 100644 index 0000000..cfc58b8 --- /dev/null +++ b/Manuals/ssh-proxy-configuration.txt @@ -0,0 +1,278 @@ +=============================================================================== +SSH PROXY CONFIGURATION GUIDE +=============================================================================== + +OVERVIEW +-------- +This guide explains how to configure SSH connections through a proxy server +with custom port forwarding. This is useful when all your Proxmox nodes are +accessed through a single proxy/jump host but on different SSH ports. + +COMMON USE CASES +---------------- +1. Nginx Stream Proxy (Nginx Proxy Manager) + - All nodes accessible via same IP (e.g., 192.168.1.100) + - Different ports mapped to different backend nodes + - Example: 192.168.1.100:2201 -> node1:22, 192.168.1.100:2202 -> node2:22 + +2. SSH Jump Host / Bastion Server + - Central gateway to internal network + - Port forwarding configured for each node + +3. Network Address Translation (NAT) + - Public IP with port forwarding rules + - Each node accessible on unique port + + +CONFIGURATION +------------- + +The SSH proxy configuration is managed through the nodes.json file in the +repository root directory. + +nodes.json Structure: +{ + "nodes": [ + { + "name": "node1", + "ip": "192.168.1.100", + "port": 2201, + "username": "root", + "ssh_keys": true + }, + { + "name": "node2", + "ip": "192.168.1.100", + "port": 2202, + "username": "root", + "ssh_keys": false + } + ] +} + +FIELD DESCRIPTIONS +------------------ +- name: Unique identifier for the node (required) +- ip: IP address or hostname of the proxy/jump host (required) +- port: SSH port number to connect to (optional, default: 22) +- username: SSH username for authentication (optional, default: root) +- ssh_keys: Whether SSH key authentication is available (optional, default: unknown) + + +NGINX PROXY MANAGER EXAMPLE +---------------------------- + +If you're using Nginx Proxy Manager with stream forwarding: + +1. Configure Streams in Nginx Proxy Manager: + + Stream 1: + - Incoming Port: 2201 + - Forward Hostname: 192.168.1.11 + - Forward Port: 22 + + Stream 2: + - Incoming Port: 2202 + - Forward Hostname: 192.168.1.12 + - Forward Port: 22 + +2. Update nodes.json: + + { + "nodes": [ + { + "name": "pve-node1", + "ip": "proxy.example.com", + "port": 2201, + "username": "root", + "ssh_keys": true + }, + { + "name": "pve-node2", + "ip": "proxy.example.com", + "port": 2202, + "username": "root", + "ssh_keys": true + } + ] + } + +3. Test the configuration: + + ssh -p 2201 root@proxy.example.com + ssh -p 2202 root@proxy.example.com + + +AUTHENTICATION +-------------- + +The scripts support two authentication methods: + +1. SSH Key Authentication (Recommended) + - More secure and convenient + - No password prompts needed + - Set "ssh_keys": true in nodes.json + - Ensure your SSH keys are added to the target nodes + +2. Password Authentication + - Uses sshpass for non-interactive authentication + - Set "ssh_keys": false in nodes.json + - You'll be prompted for passwords when needed + + +SSH KEY SETUP +------------- + +To set up SSH key authentication through a proxy: + +1. Generate SSH key (if you don't have one): + + ssh-keygen -t ed25519 -C "your_email@example.com" + +2. Copy key to each node through the proxy: + + ssh-copy-id -p 2201 root@proxy.example.com + ssh-copy-id -p 2202 root@proxy.example.com + +3. Test the connection: + + ssh -p 2201 root@proxy.example.com echo "Connection successful" + +4. Update nodes.json to reflect SSH key availability: + + Set "ssh_keys": true for nodes with working key authentication + +5. Or run the automated SSH key scanner: + + From the GUI: Settings -> Scan Nodes for SSH Keys + + +TESTING YOUR CONFIGURATION +--------------------------- + +1. Manual SSH test: + + ssh -p @ + +2. Using the GUI: + + - Launch: bash GUI.sh + - Navigate to: Settings -> Scan Nodes for SSH Keys + - The scanner will test each node and update ssh_keys status + +3. Verify in the GUI: + + - Go to: Execution Mode -> Configure Single Remote + - You should see all your nodes with correct ports + + +TROUBLESHOOTING +--------------- + +Connection Issues: +- Verify proxy/firewall rules allow traffic on specified ports +- Test connection manually: ssh -vvv -p @ +- Check if the proxy service is running (e.g., Nginx) + +Port Conflicts: +- Ensure each node has a unique port number +- Common port ranges: 2200-2299 for SSH forwarding + +SSH Key Issues: +- Verify key is in ~/.ssh/authorized_keys on target node +- Check file permissions: chmod 600 ~/.ssh/authorized_keys +- Ensure home directory: chmod 700 ~/.ssh + +Wrong Node Connected: +- Verify port mapping in proxy configuration +- Check logs on proxy server for connection routing + + +ADVANCED CONFIGURATION +---------------------- + +Multiple Proxy Servers: +You can have different nodes on different proxies: + +{ + "nodes": [ + { + "name": "cluster1-node1", + "ip": "proxy1.example.com", + "port": 2201, + "username": "root" + }, + { + "name": "cluster2-node1", + "ip": "proxy2.example.com", + "port": 2201, + "username": "admin" + } + ] +} + +Custom Usernames: +Different nodes can have different usernames: + +{ + "nodes": [ + { + "name": "prod-node", + "ip": "proxy.example.com", + "port": 2201, + "username": "root" + }, + { + "name": "test-node", + "ip": "proxy.example.com", + "port": 2202, + "username": "testuser" + } + ] +} + + +SECURITY CONSIDERATIONS +----------------------- + +1. Use SSH keys instead of passwords whenever possible +2. Limit SSH access to specific IP ranges in proxy/firewall +3. Use non-standard ports to reduce automated attacks +4. Enable fail2ban on both proxy and target nodes +5. Regularly rotate SSH keys and audit authorized_keys +6. Consider using SSH certificates for larger deployments +7. Monitor SSH logs for suspicious activity + + +MIGRATION FROM DIRECT CONNECTIONS +---------------------------------- + +If you're migrating from direct node connections to proxy-based: + +1. Backup current nodes.json: + + cp nodes.json nodes.json.backup + +2. Update IP addresses to proxy IP +3. Add port field for each node +4. Test connections one node at a time +5. Update ssh_keys status after verifying key auth works + + +RELATED DOCUMENTATION +--------------------- + +- Getting Started: getting-started.txt +- Node Management: node-management.txt +- Execution Modes: execution-modes.txt +- Troubleshooting: troubleshooting.txt + + +SUPPORT +------- + +For issues or questions: +- GitHub Issues: https://github.com/coelacant1/ProxmoxScripts/issues +- Documentation: https://coelacant1.github.io/ProxmoxScripts/ + +=============================================================================== diff --git a/Networking/FindVMIDFromIP.sh b/Networking/FindVMIDFromIP.sh new file mode 100644 index 0000000..dfde091 --- /dev/null +++ b/Networking/FindVMIDFromIP.sh @@ -0,0 +1,349 @@ +#!/bin/bash +# +# FindVMIDFromIP.sh +# +# Finds VM/LXC ID(s) from an IP address in a Proxmox cluster. Supports detection +# of nested VMs by matching MAC addresses with the BC:XX:XX prefix pattern. +# +# Usage: +# FindVMIDFromIP.sh +# FindVMIDFromIP.sh --nested-only +# +# Arguments: +# ip_address : IP address to search for +# --nested-only : Only check for nested VMs (optional) +# +# Examples: +# FindVMIDFromIP.sh 192.168.1.100 +# FindVMIDFromIP.sh 10.0.0.50 --nested-only +# +# Notes: +# - Searches guest agent data and network configurations +# - Nested VM detection: Checks if MAC address matches BC:XX:XX where XX:XX +# is the decimal representation of the VMID (e.g., VMID 100 = BC:01:00) +# - This narrows down which host VMID the nested VM is running under +# +# Function Index: +# - get_mac_from_ip +# - extract_vmid_from_mac +# - find_direct_vmid +# - find_nested_vmid +# - main +# + +set -euo pipefail + +# shellcheck source=Utilities/ArgumentParser.sh +source "${UTILITYPATH}/ArgumentParser.sh" +# shellcheck source=Utilities/Prompts.sh +source "${UTILITYPATH}/Prompts.sh" +# shellcheck source=Utilities/Communication.sh +source "${UTILITYPATH}/Communication.sh" +# shellcheck source=Utilities/Cluster.sh +source "${UTILITYPATH}/Cluster.sh" + +trap '__handle_err__ $LINENO "$BASH_COMMAND"' ERR + +# Parse arguments +__parse_args__ "ip_address:ip --nested-only:flag" "$@" + +# --- get_mac_from_ip --------------------------------------------------------- +# Attempts to resolve MAC address from IP using ARP and guest agent data +get_mac_from_ip() { + local ip="$1" + local mac="" + + # Try ARP table first + mac=$(ip neigh show "$ip" 2>/dev/null | grep -oP '(?<=lladdr )[0-9a-f:]+' | head -1 || true) + + if [[ -n "$mac" ]]; then + echo "$mac" + return 0 + fi + + # Try guest agent data from all VMs/CTs + local nodes + nodes=$(pvesh get /nodes --output-format=json | jq -r '.[] | .node') + + for node in $nodes; do + # Check VMs + local vm_ids + vm_ids=$(pvesh get /nodes/"$node"/qemu --output-format=json 2>/dev/null | jq -r '.[] | .vmid' || true) + + for vmid in $vm_ids; do + local guest_ips + guest_ips=$(pvesh get /nodes/"$node"/qemu/"$vmid"/agent/network-get-interfaces --output-format=json 2>/dev/null | \ + jq -r '.result[] | select(."ip-addresses") | ."ip-addresses"[] | select(."ip-address") | ."ip-address"' 2>/dev/null || true) + + if echo "$guest_ips" | grep -q "^${ip}$"; then + mac=$(pvesh get /nodes/"$node"/qemu/"$vmid"/agent/network-get-interfaces --output-format=json 2>/dev/null | \ + jq -r --arg ip "$ip" '.result[] | select(."ip-addresses") | select(any(."ip-addresses"[]; ."ip-address" == $ip)) | ."hardware-address"' 2>/dev/null | head -1 || true) + if [[ -n "$mac" ]]; then + echo "$mac" + return 0 + fi + fi + done + + # Check CTs + local ct_ids + ct_ids=$(pvesh get /nodes/"$node"/lxc --output-format=json 2>/dev/null | jq -r '.[] | .vmid' || true) + + for ctid in $ct_ids; do + local ct_ips + ct_ips=$(pvesh get /nodes/"$node"/lxc/"$ctid"/interfaces --output-format=json 2>/dev/null | \ + jq -r '.[] | select(.inet) | .inet' 2>/dev/null | cut -d'/' -f1 || true) + + if echo "$ct_ips" | grep -q "^${ip}$"; then + mac=$(pvesh get /nodes/"$node"/lxc/"$ctid"/interfaces --output-format=json 2>/dev/null | \ + jq -r --arg ip "$ip" '.[] | select(.inet) | select(.inet | startswith($ip + "/")) | .hwaddr' 2>/dev/null | head -1 || true) + if [[ -n "$mac" ]]; then + echo "$mac" + return 0 + fi + fi + done + done + + return 1 +} + +# --- extract_vmid_from_mac --------------------------------------------------- +# Extracts VMID from MAC address if it matches BC:XX:XX pattern +# Returns VMID or empty string if pattern doesn't match +# Note: The MAC uses decimal digits, not hex. BC:01:00 = VMID 100, BC:12:34 = VMID 1234 +extract_vmid_from_mac() { + local mac="$1" + local mac_upper + mac_upper=$(echo "$mac" | tr '[:lower:]' '[:upper:]') + + # Check if MAC starts with BC: + if [[ ! "$mac_upper" =~ ^BC: ]]; then + return 1 + fi + + # Extract the next two octets (BC:XX:XX) + local dec_vmid + dec_vmid=$(echo "$mac_upper" | cut -d: -f2-3 | tr -d ':') + + # The MAC uses decimal representation, so just remove leading zeros + local vmid=$((10#$dec_vmid)) + + if [[ $vmid -gt 0 ]]; then + echo "$vmid" + return 0 + fi + + return 1 +} + +# --- find_direct_vmid -------------------------------------------------------- +# Searches for VM/CT with the given IP address directly +find_direct_vmid() { + local ip="$1" + local found=0 + + __info__ "Searching for direct VM/CT with IP ${ip}..." + + local nodes + nodes=$(pvesh get /nodes --output-format=json | jq -r '.[] | .node') + + for node in $nodes; do + # Check VMs + local vm_ids + vm_ids=$(pvesh get /nodes/"$node"/qemu --output-format=json 2>/dev/null | jq -r '.[] | .vmid' || true) + + for vmid in $vm_ids; do + # Check guest agent + local guest_ips + guest_ips=$(pvesh get /nodes/"$node"/qemu/"$vmid"/agent/network-get-interfaces --output-format=json 2>/dev/null | \ + jq -r '.result[] | select(."ip-addresses") | ."ip-addresses"[] | select(."ip-address") | ."ip-address"' 2>/dev/null || true) + + if echo "$guest_ips" | grep -q "^${ip}$"; then + local vm_name + vm_name=$(pvesh get /nodes/"$node"/qemu/"$vmid"/config --output-format=json 2>/dev/null | jq -r '.name // "N/A"') + echo "" + __ok__ "Found VM ${vmid} on node ${node}" + echo " Type: VM (qemu)" + echo " Name: ${vm_name}" + echo " IP: ${ip}" + found=1 + fi + + # Check config + local config_ips + config_ips=$(pvesh get /nodes/"$node"/qemu/"$vmid"/config 2>/dev/null | \ + grep -oP 'ip=\K[0-9.]+' || true) + + if echo "$config_ips" | grep -q "^${ip}$"; then + local vm_name + vm_name=$(pvesh get /nodes/"$node"/qemu/"$vmid"/config --output-format=json 2>/dev/null | jq -r '.name // "N/A"') + echo "" + __ok__ "Found VM ${vmid} on node ${node} (from config)" + echo " Type: VM (qemu)" + echo " Name: ${vm_name}" + echo " IP: ${ip}" + found=1 + fi + done + + # Check CTs + local ct_ids + ct_ids=$(pvesh get /nodes/"$node"/lxc --output-format=json 2>/dev/null | jq -r '.[] | .vmid' || true) + + for ctid in $ct_ids; do + # Check interfaces + local ct_ips + ct_ips=$(pvesh get /nodes/"$node"/lxc/"$ctid"/interfaces --output-format=json 2>/dev/null | \ + jq -r '.[] | select(.inet) | .inet' 2>/dev/null | cut -d'/' -f1 || true) + + if echo "$ct_ips" | grep -q "^${ip}$"; then + local ct_name + ct_name=$(pvesh get /nodes/"$node"/lxc/"$ctid"/config --output-format=json 2>/dev/null | jq -r '.hostname // "N/A"') + echo "" + __ok__ "Found CT ${ctid} on node ${node}" + echo " Type: LXC" + echo " Name: ${ct_name}" + echo " IP: ${ip}" + found=1 + fi + + # Check config + config_ips=$(pvesh get /nodes/"$node"/lxc/"$ctid"/config 2>/dev/null | \ + grep -oP 'ip=\K[0-9.]+' || true) + + if echo "$config_ips" | grep -q "^${ip}$"; then + local ct_name + ct_name=$(pvesh get /nodes/"$node"/lxc/"$ctid"/config --output-format=json 2>/dev/null | jq -r '.hostname // "N/A"') + echo "" + __ok__ "Found CT ${ctid} on node ${node} (from config)" + echo " Type: LXC" + echo " Name: ${ct_name}" + echo " IP: ${ip}" + found=1 + fi + done + done + + return $((1 - found)) +} + +# --- find_nested_vmid -------------------------------------------------------- +# Searches for nested VM by checking MAC address prefix pattern +find_nested_vmid() { + local ip="$1" + + __info__ "Checking for nested VM with IP ${ip}..." + + # Get MAC address for the IP + local mac + mac=$(get_mac_from_ip "$ip" || true) + + if [[ -z "$mac" ]]; then + __warn__ "Could not determine MAC address for IP ${ip}" + __info__ "Tips:" + echo " - Ensure the IP is reachable via ARP" + echo " - Check if guest agent is installed and running" + echo " - Try: arp -a | grep ${ip}" + return 1 + fi + + __info__ "Found MAC address: ${mac}" + + # Extract VMID from MAC if it matches pattern + local parent_vmid + parent_vmid=$(extract_vmid_from_mac "$mac" || true) + + if [[ -z "$parent_vmid" ]]; then + __warn__ "MAC address ${mac} does not match BC:XX:XX pattern" + echo " This IP is likely not a nested VM using the standard MAC prefix scheme" + return 1 + fi + + # Verify the parent VMID exists + local parent_node + parent_node=$(__get_vm_node__ "$parent_vmid" 2>/dev/null || __get_ct_node__ "$parent_vmid" 2>/dev/null || true) + + if [[ -z "$parent_node" ]]; then + __warn__ "Extracted VMID ${parent_vmid} from MAC, but no such VM/CT exists" + return 1 + fi + + # Get parent details + local parent_name + local parent_type + if __get_vm_node__ "$parent_vmid" >/dev/null 2>&1; then + parent_type="VM (qemu)" + parent_name=$(pvesh get /nodes/"$parent_node"/qemu/"$parent_vmid"/config --output-format=json 2>/dev/null | jq -r '.name // "N/A"') + else + parent_type="LXC" + parent_name=$(pvesh get /nodes/"$parent_node"/lxc/"$parent_vmid"/config --output-format=json 2>/dev/null | jq -r '.hostname // "N/A"') + fi + + echo "" + __ok__ "Found nested VM indicator!" + echo " IP: ${ip}" + echo " MAC: ${mac}" + echo " Parent VMID: ${parent_vmid}" + echo " Parent Type: ${parent_type}" + echo " Parent Name: ${parent_name}" + echo " Parent Node: ${parent_node}" + echo "" + echo " This suggests IP ${ip} belongs to a nested VM running inside VMID ${parent_vmid}" + echo " Connect to VMID ${parent_vmid} to investigate the nested environment" + + return 0 +} + +# --- main -------------------------------------------------------------------- +main() { + __check_root__ + __check_proxmox__ + __install_or_prompt__ "jq" + __check_cluster_membership__ + + local found=0 + + # Search for direct VM/CT match (unless --nested-only specified) + if [[ "$NESTED_ONLY" != "true" ]]; then + if find_direct_vmid "$IP_ADDRESS"; then + found=1 + fi + fi + + # Search for nested VM + if find_nested_vmid "$IP_ADDRESS"; then + found=1 + fi + + if [[ $found -eq 0 ]]; then + echo "" + __err__ "No VM/CT found with IP ${IP_ADDRESS}" + __info__ "Troubleshooting tips:" + echo " - Verify the IP is correct and reachable" + echo " - Check if guest agent is installed and running" + echo " - For nested VMs, ensure MAC uses BC:XX:XX prefix pattern" + echo " - Try manual search: pvesh get /cluster/resources --type vm" + exit 1 + fi +} + +main + +############################################################################### +# Script notes: +############################################################################### +# Last checked: 2025-12-01 +# +# Changes: +# - 2025-12-01: Initial version created +# - Direct VM/CT search via guest agent and config +# - Nested VM detection via BC:XX:XX MAC prefix pattern +# - Cluster-wide search capability +# +# Fixes: +# - +# +# Known issues: +# - +# diff --git a/Resources/ExportProxmoxResources.sh b/Resources/ExportProxmoxResources.sh index c513520..0a24199 100644 --- a/Resources/ExportProxmoxResources.sh +++ b/Resources/ExportProxmoxResources.sh @@ -60,7 +60,7 @@ parse_config_files() { local cpu_cores cpu_cores="$(grep -Po '^cores: \K.*' "$config_file" || echo "0")" local memory_mb - memory_mb="$(grep -Po '^memory: \K.*' "$config_file" || echo "0")" + memory_mb="$(grep -Po '^memory: \K.*' "$config_file" | head -1 || echo "0")" local disk_gb disk_gb="$(grep -Po 'size=\K[0-9]+[A-Z]?' "$config_file" | awk ' @@ -75,7 +75,7 @@ parse_config_files() { END { print sum } ' || echo "0")" - echo "$node_name,$vmid,$vm_name,$cpu_cores,$((memory_mb)),$disk_gb" >>"$output_file" + echo "$node_name,$vmid,$vm_name,$cpu_cores,$memory_mb,$disk_gb" >>"$output_file" done fi fi @@ -95,7 +95,7 @@ parse_config_files() { local cpu_cores cpu_cores="$(grep -Po '^cores: \K.*' "$config_file" || echo "0")" local memory_mb - memory_mb="$(grep -Po '^memory: \K.*' "$config_file" || echo "0")" + memory_mb="$(grep -Po '^memory: \K.*' "$config_file" | head -1 || echo "0")" local disk_gb disk_gb="$(grep -Po 'size=\K[0-9]+[A-Z]?' "$config_file" | awk ' @@ -110,7 +110,7 @@ parse_config_files() { END { print sum } ' || echo "0")" - echo "$node_name,$vmid,$vm_name,$cpu_cores,$((memory_mb)),$disk_gb" >>"$output_file" + echo "$node_name,$vmid,$vm_name,$cpu_cores,$memory_mb,$disk_gb" >>"$output_file" done fi fi diff --git a/Utilities/BulkOperations.sh b/Utilities/BulkOperations.sh index 54b66fb..10b0fee 100644 --- a/Utilities/BulkOperations.sh +++ b/Utilities/BulkOperations.sh @@ -108,7 +108,15 @@ __bulk_operation__() { __update__ "Processing IDs ${start_id} to ${end_id} (${BULK_TOTAL} total)" # Process each ID + local id for ((id = start_id; id <= end_id; id++)); do + # Defensive check: ensure id is numeric and in range + if ! [[ "$id" =~ ^[0-9]+$ ]] || ((id < start_id)) || ((id > end_id)); then + __bulk_log__ "ERROR" "Loop corruption detected: id=$id (expected $start_id-$end_id)" + echo "Error: Internal loop corruption detected" >&2 + return 1 + fi + local current=$((id - start_id + 1)) __bulk_log__ "TRACE" "Processing ID $id ($current/$BULK_TOTAL)" __update__ "Processing ID ${id} (${current}/${BULK_TOTAL})..." @@ -122,6 +130,7 @@ __bulk_operation__() { BULK_FAILED_IDS[$id]=1 __bulk_log__ "WARN" "Failed: ID $id" fi + __bulk_log__ "TRACE" "Completed processing ID $id, next will be $((id + 1))" done __bulk_log__ "INFO" "Bulk operation complete: success=$BULK_SUCCESS, failed=$BULK_FAILED, skipped=$BULK_SKIPPED" @@ -189,9 +198,18 @@ __bulk_vm_operation__() { __bulk_log__ "INFO" "Starting VM bulk operation: $operation_name (range: $start_id-$end_id, skip_stopped: $skip_stopped, skip_running: $skip_running)" # Wrapper function that checks VM existence and state + # First parameter is vmid, second is the actual callback to execute vm_wrapper() { local vmid="$1" - shift + local actual_callback="$2" + shift 2 + + # Defensive check: ensure callback is set + if [[ -z "$actual_callback" ]]; then + __bulk_log__ "ERROR" "vm_wrapper: callback parameter is not set" + ((BULK_FAILED += 1)) + return 1 + fi # Check existence if ! __vm_exists__ "$vmid"; then @@ -217,12 +235,15 @@ __bulk_vm_operation__() { fi # Execute callback - __bulk_log__ "TRACE" "Executing callback for VM $vmid" - "$callback" "$vmid" "$@" + __bulk_log__ "TRACE" "Executing callback '$actual_callback' for VM $vmid" + "$actual_callback" "$vmid" "$@" + local result=$? + __bulk_log__ "TRACE" "Callback '$actual_callback' returned $result for VM $vmid" + return $result } - # Run bulk operation - __bulk_operation__ "$start_id" "$end_id" vm_wrapper "$@" + # Run bulk operation - pass the callback as an extra parameter + __bulk_operation__ "$start_id" "$end_id" vm_wrapper "$callback" "$@" local result=$? # Show detailed report if requested @@ -292,9 +313,18 @@ __bulk_ct_operation__() { __bulk_log__ "INFO" "Bulk CT operation: $operation_name (range: $start_id-$end_id, callback: $callback)" # Wrapper function that checks CT existence and state + # First parameter is ctid, second is the actual callback to execute ct_wrapper() { local ctid="$1" - shift + local actual_callback="$2" + shift 2 + + # Defensive check: ensure callback is set + if [[ -z "$actual_callback" ]]; then + __bulk_log__ "ERROR" "ct_wrapper: callback parameter is not set" + ((BULK_FAILED += 1)) + return 1 + fi # Check existence if ! __ct_exists__ "$ctid"; then @@ -317,11 +347,11 @@ __bulk_ct_operation__() { fi # Execute callback - "$callback" "$ctid" "$@" + "$actual_callback" "$ctid" "$@" } - # Run bulk operation - __bulk_operation__ "$start_id" "$end_id" ct_wrapper "$@" + # Run bulk operation - pass the callback as an extra parameter + __bulk_operation__ "$start_id" "$end_id" ct_wrapper "$callback" "$@" local result=$? # Show detailed report if requested @@ -558,8 +588,8 @@ __bulk_parallel__() { BULK_FAILED=0 BULK_START_TIME=$(date +%s) - # Create temporary directory for results - local tmpdir="/tmp/bulk_parallel_$$" + # Create temporary directory for results (use timestamp to avoid collision in same GUI session) + local tmpdir="/tmp/bulk_parallel_$$_$(date +%s%N)" mkdir -p "$tmpdir" local running=0 @@ -732,9 +762,12 @@ __bulk_validate_range__() { ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Integrated cluster cache for bulk operations +# - 2026-01-08: Fixed temp directory collision in parallel bulk operations +# - 2026-01-08: CRITICAL FIX - Fixed infinite recursion bug in wrapper functions # - 2025-11-24: Fixed ShellCheck warnings (SC2155, SC2145, SC1090) # - Initial creation # diff --git a/Utilities/Cluster.sh b/Utilities/Cluster.sh index 1d481f5..3f3d07a 100644 --- a/Utilities/Cluster.sh +++ b/Utilities/Cluster.sh @@ -312,6 +312,12 @@ __get_server_vms__() { # @return Prints the node name to stdout, or empty string if not found. # @example_output For __get_vm_node__ 400, the output might be: # pve01 +# --- __get_vm_node__ --------------------------------------------------------- +# @function __get_vm_node__ +# @description Get the node name where a VM is located +# @usage local node=$(__get_vm_node__ ) +# @param 1 VM ID +# @return Prints node name to stdout __get_vm_node__() { local vmid="$1" __query_log__ "TRACE" "Getting node for VM: $vmid" @@ -356,20 +362,13 @@ __get_ct_node__() { __install_or_prompt__ "jq" local node - # First try cluster resources node=$(pvesh get /cluster/resources --type vm --output-format json 2>/dev/null \ | jq -r --arg CTID "$ctid" '.[] | select(.type=="lxc" and .vmid==($CTID|tonumber)) | .node' 2>/dev/null || true) - # If not found in cluster, try local pct config - if [[ -z "$node" ]] && pct config "$ctid" &>/dev/null; then - node=$(hostname) - __query_log__ "DEBUG" "CT $ctid found locally on: $node" - fi - if [[ -n "$node" ]]; then __query_log__ "DEBUG" "CT $ctid is on node: $node" else - __query_log__ "WARN" "CT $ctid not found" + __query_log__ "WARN" "CT $ctid not found in cluster" fi echo "$node" @@ -784,9 +783,11 @@ __get_pool_vms__() { ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Implemented file-based query caching +# - 2026-01-08: Added query result caching for __get_vm_node__ and __get_ct_node__ # - 2025-11-24: Validated against CONTRIBUTING.md and PVE Guide Chapter 5 # - 2025-11-24: Fixed ShellCheck warnings (SC2155 - declare/assign separation) # - Initial creation diff --git a/Utilities/ConfigManager.sh b/Utilities/ConfigManager.sh index 8069b8f..1f34fe5 100644 --- a/Utilities/ConfigManager.sh +++ b/Utilities/ConfigManager.sh @@ -12,6 +12,7 @@ # - __clear_remote_targets__ # - __get_node_ip__ # - __get_node_username__ +# - __get_node_port__ # - __node_exists__ # - __get_available_nodes__ # - __count_available_nodes__ @@ -29,11 +30,14 @@ declare -g TARGET_DISPLAY="This System" declare -ga REMOTE_TARGETS=() declare -gA NODE_PASSWORDS=() declare -gA AVAILABLE_NODES=() +declare -ga NODE_ORDER=() # Preserve order from nodes.json declare -gA NODE_USERNAMES=() # Track username for each node +declare -gA NODE_PORTS=() # Track SSH port for each node declare -gA NODE_SSH_KEYS=() # Track SSH key status declare -g NODES_FILE="nodes.json" declare -g REMOTE_TEMP_DIR="/tmp/ProxmoxScripts_gui" declare -g DEFAULT_USERNAME="root" # Default username for nodes +declare -g DEFAULT_PORT="22" # Default SSH port # Only set default if not already set (e.g., from command-line flags) if [[ -z "${REMOTE_LOG_LEVEL:-}" ]]; then declare -g REMOTE_LOG_LEVEL="INFO" @@ -50,15 +54,21 @@ __init_config__() { fi if [[ -f "$NODES_FILE" ]] && command -v jq &>/dev/null; then + # Reset order array + NODE_ORDER=() + while IFS= read -r line; do - local node_name node_ip ssh_keys node_username + local node_name node_ip ssh_keys node_username node_port node_name=$(echo "$line" | jq -r '.name') node_ip=$(echo "$line" | jq -r '.ip') ssh_keys=$(echo "$line" | jq -r 'if has("ssh_keys") then (.ssh_keys | tostring) else "unknown" end') node_username=$(echo "$line" | jq -r 'if has("username") then .username else "'"$DEFAULT_USERNAME"'" end') + node_port=$(echo "$line" | jq -r 'if has("port") then (.port | tostring) else "'"$DEFAULT_PORT"'" end') AVAILABLE_NODES["$node_name"]="$node_ip" NODE_SSH_KEYS["$node_name"]="$ssh_keys" NODE_USERNAMES["$node_name"]="$node_username" + NODE_PORTS["$node_name"]="$node_port" + NODE_ORDER+=("$node_name") # Preserve order from JSON done < <(jq -c '.nodes[]' "$NODES_FILE" 2>/dev/null || true) fi } @@ -128,6 +138,14 @@ __get_node_username__() { echo "${NODE_USERNAMES[$node_name]:-$DEFAULT_USERNAME}" } +# Get node port by name +# Args: node_name +# Returns: port number or default port (22) +__get_node_port__() { + local node_name="$1" + echo "${NODE_PORTS[$node_name]:-$DEFAULT_PORT}" +} + # Check if node exists # Args: node_name # Returns: 0 if exists, 1 if not @@ -136,9 +154,9 @@ __node_exists__() { [[ -v AVAILABLE_NODES[$node_name] ]] } -# Get all available node names +# Get all available node names (in order from nodes.json) __get_available_nodes__() { - printf '%s\n' "${!AVAILABLE_NODES[@]}" + printf '%s\n' "${NODE_ORDER[@]}" } # Count available nodes @@ -147,12 +165,13 @@ __count_available_nodes__() { } # Check if node has SSH keys configured (from cache or test) -# Args: node_ip username node_name +# Args: node_ip username node_name [port] # Returns: "true" if keys work, "false" if not, "unknown" if not tested __has_ssh_keys__() { local node_ip="$1" local username="${2:-$DEFAULT_USERNAME}" local node_name="${3:-}" + local port="${4:-$DEFAULT_PORT}" # Check cache first if node_name provided if [[ -n "$node_name" ]] && [[ "${NODE_SSH_KEYS[$node_name]:-unknown}" != "unknown" ]]; then @@ -161,7 +180,7 @@ __has_ssh_keys__() { fi # Test SSH connection - if ssh -o BatchMode=yes -o ConnectTimeout=2 "${username}@${node_ip}" echo "test" &>/dev/null 2>&1; then + if ssh -o BatchMode=yes -o ConnectTimeout=2 -p "$port" "${username}@${node_ip}" echo "test" &>/dev/null 2>&1; then # Update cache if node_name provided if [[ -n "$node_name" ]]; then NODE_SSH_KEYS["$node_name"]="true" @@ -205,10 +224,11 @@ __scan_ssh_keys__() { for node_name in "${!AVAILABLE_NODES[@]}"; do local node_ip="${AVAILABLE_NODES[$node_name]}" local node_username="${NODE_USERNAMES[$node_name]:-$DEFAULT_USERNAME}" - echo -n " Checking $node_name ($node_username@$node_ip)... " + local node_port="${NODE_PORTS[$node_name]:-$DEFAULT_PORT}" + echo -n " Checking $node_name ($node_username@$node_ip:$node_port)... " local has_keys - has_keys=$(__has_ssh_keys__ "$node_ip" "$node_username" "$node_name") + has_keys=$(__has_ssh_keys__ "$node_ip" "$node_username" "$node_name" "$node_port") if [[ "$has_keys" == "true" ]]; then echo "[SSH]" @@ -238,9 +258,11 @@ __get_remote_log_level__() { ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Added NODE_ORDER array to preserve node order from nodes.json +# - 2025-12-18: Added SSH port configuration support for proxy/jump host setups # - 2025-11-24: Validated against CONTRIBUTING.md, fixed ShellCheck warnings # - Initial version: Configuration management for GUI execution modes # diff --git a/Utilities/Logger.sh b/Utilities/Logger.sh index be2691d..d5f6ed5 100644 --- a/Utilities/Logger.sh +++ b/Utilities/Logger.sh @@ -12,7 +12,7 @@ # __log__ "DEBUG" "Debug information" # # Environment Variables: -# LOG_LEVEL - Minimum level to log (DEBUG, INFO, WARN, ERROR) - default: INFO +# LOG_LEVEL - Minimum level to log (TRACE, DEBUG, INFO, WARN, ERROR) - default: INFO # LOG_FILE - Path to log file - default: /tmp/proxmox_scripts.log # LOG_CONSOLE - Whether to also log to console (1=yes, 0=no) - default: 1 # LOG_TIMESTAMP - Whether to include timestamps (1=yes, 0=no) - default: 1 @@ -20,6 +20,7 @@ # Function Index: # - __get_log_priority__ # - __log__ +# - __log_trace__ # - __log_debug__ # - __log_info__ # - __log_warn__ @@ -39,10 +40,11 @@ # Log level priorities declare -A LOG_LEVELS=( - [DEBUG]=0 - [INFO]=1 - [WARN]=2 - [ERROR]=3 + [TRACE]=0 + [DEBUG]=1 + [INFO]=2 + [WARN]=3 + [ERROR]=4 ) # Get current log level priority @@ -54,7 +56,7 @@ __get_log_priority__() { # @function __log__ # @description Core logging function with level-based filtering and formatting # @usage __log__ [category] -# @param level Log level (DEBUG, INFO, WARN, ERROR) +# @param level Log level (TRACE, DEBUG, INFO, WARN, ERROR) # @param message Message to log # @param category Optional category/component name __log__() { @@ -121,6 +123,9 @@ __log__() { DEBUG) echo -e "\033[0;36m$log_entry\033[0m" ;; + TRACE) + echo -e "\033[0;90m$log_entry\033[0m" + ;; *) echo "$log_entry" ;; @@ -128,6 +133,13 @@ __log__() { fi } +# --- __log_trace__ ------------------------------------------------------------- +# @function __log_trace__ +# @description Log trace message +__log_trace__() { + __log__ "TRACE" "$1" "${2:-}" +} + # --- __log_debug__ ------------------------------------------------------------- # @function __log_debug__ # @description Log debug message @@ -224,14 +236,16 @@ fi ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2025-01-26 # # Changes: +# - 2025-01-26: Added TRACE log level support (priority 0, below DEBUG) # - 2025-11-24: Validated against CONTRIBUTING.md # - 2025-11-24: Fixed unnecessary backslash escaping in date command paths (SC1001) # - Initial creation: Centralized logging utility with level-based filtering # # Fixes: +# - 2025-01-26: Fixed TRACE logs appearing at INFO level - added TRACE to LOG_LEVELS array # - 2025-11-24: Removed unnecessary backslash escaping in /bin/date and /usr/bin/date paths # # Known issues: diff --git a/Utilities/Operations.sh b/Utilities/Operations.sh index feee5ee..54aedb1 100644 --- a/Utilities/Operations.sh +++ b/Utilities/Operations.sh @@ -15,6 +15,7 @@ # - Consistent return codes and error messages # - Testable and mockable functions # - State management helpers +# - Optimized to minimize redundant API queries # # Function Index: # - __api_log__ @@ -100,12 +101,15 @@ source "${UTILITYPATH}/Cluster.sh" # --- __vm_exists__ ----------------------------------------------------------- # @function __vm_exists__ -# @description Check if a VM exists (cluster-wide). -# @usage __vm_exists__ +# @description Check if a VM exists (cluster-wide). Optionally returns node via stdout. +# @usage if __vm_exists__ "$vmid"; then ...; fi OR node=$(__vm_exists__ "$vmid" --get-node) # @param 1 VM ID -# @return 0 if exists, 1 if not +# @param 2 Optional: --get-node to output node name to stdout +# @return 0 if exists (with node on stdout if --get-node), 1 if not +# @example node=$(__vm_exists__ "$vmid" --get-node) && echo "VM on $node" __vm_exists__() { local vmid="$1" + local get_node="${2:-}" __api_log__ "DEBUG" "Checking if VM $vmid exists" @@ -120,6 +124,12 @@ __vm_exists__() { if [[ -n "$node" ]]; then __api_log__ "DEBUG" "VM $vmid exists on node $node" + + # If caller wants node via stdout + if [[ "$get_node" == "--get-node" ]]; then + echo "$node" + fi + return 0 else __api_log__ "DEBUG" "VM $vmid does not exist" @@ -143,17 +153,15 @@ __vm_get_status__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then echo "Error: VM $vmid does not exist" >&2 __api_log__ "ERROR" "VM $vmid does not exist" return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - - # Get status from qm status - qm status "$vmid" --node "$node" 2>/dev/null | awk '/^status:/ {print $2}' + # Get status from qm status - execute on the node where VM resides + __node_exec__ "$node" "qm status $vmid" 2>/dev/null | awk '/^status:/ {print $2}' } # --- __vm_is_running__ ------------------------------------------------------- @@ -200,7 +208,8 @@ __vm_start__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then echo "Error: VM $vmid does not exist" >&2 __api_log__ "ERROR" "VM $vmid does not exist" return 1 @@ -213,12 +222,15 @@ __vm_start__() { return 0 fi - local node - node=$(__get_vm_node__ "$vmid") + # Build command with any additional arguments + local cmd="qm start $vmid" + if [[ $# -gt 0 ]]; then + cmd+=" $*" + fi - __api_log__ "DEBUG" "Executing: qm start $vmid --node $node $*" + __api_log__ "DEBUG" "Executing on node $node: $cmd" - if qm start "$vmid" --node "$node" "$@" 2>/dev/null; then + if __node_exec__ "$node" "$cmd" 2>/dev/null; then __api_log__ "INFO" "VM $vmid started successfully on node $node" return 0 else @@ -230,12 +242,13 @@ __vm_start__() { # --- __vm_stop__ ------------------------------------------------------------- # @function __vm_stop__ -# @description Stop a VM (cluster-aware). +# @description Stop a VM (cluster-aware). Sends SIGTERM then SIGKILL. # @usage __vm_stop__ [--timeout ] [--force] # @param 1 VM ID -# @param --timeout Timeout in seconds before force stop -# @param --force Force stop immediately +# @param --timeout Timeout in seconds (optional) +# @param --force Ignored for compatibility (qm stop is forceful by default) # @return 0 on success, 1 on error +# @note The --force flag is accepted but ignored for Proxmox version compatibility __vm_stop__() { local vmid="$1" shift @@ -269,7 +282,8 @@ __vm_stop__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 @@ -282,22 +296,37 @@ __vm_stop__() { return 0 fi - local node - node=$(__get_vm_node__ "$vmid") __api_log__ "DEBUG" "VM $vmid is on node: $node" - local cmd="qm stop \"$vmid\" --node \"$node\"" - [[ -n "$timeout" ]] && cmd+=" --timeout \"$timeout\"" - [[ "$force" == true ]] && cmd+=" --force" + # Build command - note: older Proxmox versions don't support --force flag + # In older versions, qm stop is already forceful (immediate kill) + # In newer versions (PVE 7+), --force is supported but optional + local cmd="qm stop $vmid" + if [[ -n "$timeout" ]]; then + cmd+=" --timeout $timeout" + fi + # Don't add --force flag as it's not supported in all Proxmox versions + # qm stop is already forceful by default (sends SIGTERM then SIGKILL) - __api_log__ "DEBUG" "Executing: $cmd" + __api_log__ "DEBUG" "Executing on node $node: $cmd" - if eval "$cmd" 2>/dev/null; then + local stop_output + local stop_exit_code + stop_output=$(__node_exec__ "$node" "$cmd" 2>&1) + stop_exit_code=$? + + __api_log__ "DEBUG" "Stop command exit code: $stop_exit_code" + if [[ -n "$stop_output" ]]; then + __api_log__ "DEBUG" "Stop command output: $stop_output" + echo "$stop_output" + fi + + if [[ $stop_exit_code -eq 0 ]]; then __api_log__ "INFO" "Successfully stopped VM $vmid" return 0 else - __api_log__ "ERROR" "Failed to stop VM $vmid on node $node" - echo "Error: Failed to stop VM $vmid on node $node" >&2 + __api_log__ "ERROR" "Failed to stop VM $vmid on node $node (exit code: $stop_exit_code)" + echo "Error: Failed to stop VM $vmid on node $node (exit code: $stop_exit_code)" >&2 return 1 fi } @@ -320,17 +349,21 @@ __vm_set_config__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + # Get node and check existence in one call + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") __api_log__ "DEBUG" "Setting config for VM $vmid on node: $node" - if qm set "$vmid" --node "$node" "$@" 2>/dev/null; then + # Build command with arguments + local cmd="qm set $vmid $*" + __api_log__ "DEBUG" "Executing on node $node: $cmd" + + if __node_exec__ "$node" "$cmd" 2>/dev/null; then __api_log__ "INFO" "Successfully set configuration for VM $vmid" return 0 else @@ -358,17 +391,15 @@ __vm_get_config__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - local value - value=$(qm config "$vmid" --node "$node" 2>/dev/null | grep "^${param}:" | cut -d' ' -f2-) + value=$(__node_exec__ "$node" "qm config $vmid" 2>/dev/null | grep "^${param}:" | cut -d' ' -f2-) __api_log__ "DEBUG" "VM $vmid config $param: ${value:-}" echo "$value" } @@ -379,12 +410,14 @@ __vm_get_config__() { # --- __ct_exists__ ----------------------------------------------------------- # @function __ct_exists__ -# @description Check if a CT exists. -# @usage __ct_exists__ +# @description Check if a CT exists. Optionally returns node via stdout. +# @usage if __ct_exists__ "$ctid"; then ...; fi OR node=$(__ct_exists__ "$ctid" --get-node) # @param 1 CT ID -# @return 0 if exists, 1 if not +# @param 2 Optional: --get-node to output node name to stdout +# @return 0 if exists (with node on stdout if --get-node), 1 if not __ct_exists__() { local ctid="$1" + local get_node="${2:-}" __api_log__ "DEBUG" "Checking if CT $ctid exists" @@ -395,6 +428,16 @@ __ct_exists__() { if pct config "$ctid" &>/dev/null; then __api_log__ "DEBUG" "CT $ctid exists" + + # If caller wants node via stdout + if [[ "$get_node" == "--get-node" ]]; then + local node + node=$(__get_ct_node__ "$ctid" 2>/dev/null) + if [[ -n "$node" ]]; then + echo "$node" + fi + fi + return 0 else __api_log__ "DEBUG" "CT $ctid does not exist" @@ -767,7 +810,7 @@ __vm_shutdown__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 @@ -779,11 +822,9 @@ __vm_shutdown__() { return 0 fi - local node - node=$(__get_vm_node__ "$vmid") __api_log__ "DEBUG" "Shutting down VM $vmid on node: $node" - if qm shutdown "$vmid" --node "$node" --timeout "$timeout" 2>/dev/null; then + if __node_exec__ "$node" "qm shutdown $vmid --timeout $timeout" 2>/dev/null; then __api_log__ "INFO" "Successfully shut down VM $vmid" return 0 else @@ -834,16 +875,14 @@ __vm_suspend__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - - if qm suspend "$vmid" --node "$node" 2>/dev/null; then + if __node_exec__ "$node" "qm suspend $vmid" 2>/dev/null; then __api_log__ "INFO" "VM $vmid suspended successfully" return 0 else @@ -869,16 +908,14 @@ __vm_resume__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - - if qm resume "$vmid" --node "$node" 2>/dev/null; then + if __node_exec__ "$node" "qm resume $vmid" 2>/dev/null; then __api_log__ "INFO" "VM $vmid resumed successfully" return 0 else @@ -971,20 +1008,30 @@ __vm_wait_for_status__() { fi local elapsed=0 - while ((elapsed < timeout)); do + local max_iterations=$((timeout / interval + 1)) + local iteration=0 + + while ((elapsed < timeout)) && ((iteration < max_iterations)); do local current_status - current_status=$(__vm_get_status__ "$vmid" 2>/dev/null) + current_status=$(__vm_get_status__ "$vmid" 2>/dev/null) || current_status="" - if [[ "$current_status" == "$desired_status" ]]; then + if [[ -n "$current_status" && "$current_status" == "$desired_status" ]]; then __api_log__ "INFO" "VM $vmid reached status: $desired_status (elapsed: ${elapsed}s)" return 0 fi + if [[ -z "$current_status" ]]; then + __api_log__ "WARN" "Failed to get status for VM $vmid (iteration $iteration, elapsed ${elapsed}s)" + else + __api_log__ "TRACE" "VM $vmid current status: $current_status (waiting for $desired_status, elapsed ${elapsed}s)" + fi + sleep "$interval" - ((elapsed += interval)) + ((elapsed += interval)) || true + ((iteration++)) || true done - __api_log__ "ERROR" "Timeout waiting for VM $vmid to reach status $desired_status after ${timeout}s" + __api_log__ "ERROR" "Timeout waiting for VM $vmid to reach status $desired_status after ${elapsed}s (max: ${timeout}s)" echo "Error: Timeout waiting for VM $vmid to reach status $desired_status" >&2 return 1 } @@ -1152,20 +1199,30 @@ __ct_wait_for_status__() { fi local elapsed=0 - while ((elapsed < timeout)); do + local max_iterations=$((timeout / interval + 1)) + local iteration=0 + + while ((elapsed < timeout)) && ((iteration < max_iterations)); do local current_status - current_status=$(__ct_get_status__ "$ctid" 2>/dev/null) + current_status=$(__ct_get_status__ "$ctid" 2>/dev/null) || current_status="" - if [[ "$current_status" == "$desired_status" ]]; then + if [[ -n "$current_status" && "$current_status" == "$desired_status" ]]; then __api_log__ "INFO" "CT $ctid reached status: $desired_status (elapsed: ${elapsed}s)" return 0 fi + if [[ -z "$current_status" ]]; then + __api_log__ "WARN" "Failed to get status for CT $ctid (iteration $iteration, elapsed ${elapsed}s)" + else + __api_log__ "TRACE" "CT $ctid current status: $current_status (waiting for $desired_status, elapsed ${elapsed}s)" + fi + sleep "$interval" - ((elapsed += interval)) + ((elapsed += interval)) || true + ((iteration++)) || true done - __api_log__ "ERROR" "Timeout waiting for CT $ctid to reach status $desired_status after ${timeout}s" + __api_log__ "ERROR" "Timeout waiting for CT $ctid to reach status $desired_status after ${elapsed}s (max: ${timeout}s)" echo "Error: Timeout waiting for CT $ctid to reach status $desired_status" >&2 return 1 } @@ -1226,15 +1283,13 @@ __get_vm_info__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - local status status=$(__vm_get_status__ "$vmid") @@ -1314,8 +1369,14 @@ __node_exec__() { # If target node is local, execute directly if [[ "$node" == "$local_hostname" ]]; then __api_log__ "DEBUG" "Executing locally on $node" - eval "$command" - return $? + local output + local exit_code + output=$(eval "$command" 2>&1) + exit_code=$? + if [[ -n "$output" ]]; then + echo "$output" + fi + return $exit_code fi # Remote execution via SSH @@ -1345,15 +1406,14 @@ __vm_node_exec__() { return 1 fi - if ! __vm_exists__ "$vmid"; then + # Get node and check existence in one call + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then __api_log__ "ERROR" "VM $vmid does not exist" echo "Error: VM $vmid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - if [[ -z "$node" ]]; then __api_log__ "ERROR" "Could not determine node for VM $vmid" echo "Error: Could not determine node for VM $vmid" >&2 @@ -1389,15 +1449,13 @@ __ct_node_exec__() { return 1 fi - if ! __ct_exists__ "$ctid"; then + local node + if ! node=$(__ct_exists__ "$ctid" --get-node); then __api_log__ "ERROR" "CT $ctid does not exist" echo "Error: CT $ctid does not exist" >&2 return 1 fi - local node - node=$(__get_vm_node__ "$ctid") # Works for CTs too - if [[ -z "$node" ]]; then __api_log__ "ERROR" "Could not determine node for CT $ctid" echo "Error: Could not determine node for CT $ctid" >&2 @@ -1680,9 +1738,12 @@ __vm_set_protection__() { fi local node - node=$(__get_vm_node__ "$vmid") + if ! node=$(__vm_exists__ "$vmid" --get-node); then + __api_log__ "ERROR" "VM $vmid does not exist" + return 1 + fi - if qm set "$vmid" --node "$node" --protection "$value" 2>/dev/null; then + if __node_exec__ "$node" "qm set $vmid --protection $value" 2>/dev/null; then __api_log__ "INFO" "Successfully set protection for VM $vmid" return 0 else @@ -2062,6 +2123,13 @@ __vm_add_ip_to_note__() { return 1 fi + # Validate VM exists and get node for later operations + local node + if ! node=$(__vm_exists__ "$vmid" --get-node); then + __api_log__ "ERROR" "VM $vmid does not exist" + return 1 + fi + local ip ip=$(qm guest cmd "$vmid" network-get-interfaces 2>/dev/null | jq -r '.[].["ip-addresses"][]? | select(.["ip-address-type"] == "ipv4") | .["ip-address"]' | grep -v "^127\." | head -1 || echo "") @@ -2070,9 +2138,6 @@ __vm_add_ip_to_note__() { return 1 fi - local node - node=$(__get_vm_node__ "$vmid") - local current_note current_note=$(__vm_get_config__ "$vmid" "description" 2>/dev/null || echo "") local new_note="IP: $ip" @@ -2081,7 +2146,7 @@ __vm_add_ip_to_note__() { new_note="$current_note\n$new_note" fi - if qm set "$vmid" --node "$node" --description "$new_note" 2>/dev/null; then + if __node_exec__ "$node" "qm set $vmid --description '$new_note'" 2>/dev/null; then __api_log__ "INFO" "Successfully added IP to note for VM $vmid" return 0 else @@ -2093,17 +2158,31 @@ __vm_add_ip_to_note__() { ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: PERFORMANCE: Optimized to reduce redundant API queries by passing +# node information from __vm_exists__/__ct_exists__ to operation functions using +# command substitution pattern: node=$(__vm_exists__ "$vmid" --get-node) +# - 2026-01-06: CRITICAL FIX: Removed invalid --node flag from all qm commands and migrated to __node_exec__ # - 2025-11-24: Fixed ShellCheck warnings (SC2155) - separated variable declaration and assignment # - YYYY-MM-DD: Initial creation # # Fixes: +# - 2026-01-06: FIXED CRITICAL BUG: qm commands were using --node flag which doesn't exist, +# causing ALL bulk VM operations to fail. Changed to use __node_exec__ to execute commands +# on correct node via SSH/local execution. Affected: +# - __vm_get_status__ (line 156): qm status +# - __vm_start__ (line 221): qm start +# - __vm_stop__ (line 295): qm stop +# - __vm_set_config__ (line 333): qm set +# - __vm_get_config__ (line 371): qm config +# - __vm_shutdown__ (line 786): qm shutdown +# - __vm_suspend__ (line 846): qm suspend +# - __vm_resume__ (line 881): qm resume +# - __vm_set_protection__ (line 1695): qm set --protection +# - __vm_add_ip_to_note__ (line 2094): qm set --description # - 2025-11-24: FIXED: Variable declaration/assignment separation for proper error handling -# - Line 926: __vm_list_all__ - Separated 'count' variable declaration from assignment -# - Line 1106: __ct_list_all__ - Separated 'count' variable declaration from assignment -# - Impact: Allows detection of grep command failures instead of masking return values # # Known issues: # - diff --git a/Utilities/RemoteExecutor.sh b/Utilities/RemoteExecutor.sh index 5173951..cccfea8 100644 --- a/Utilities/RemoteExecutor.sh +++ b/Utilities/RemoteExecutor.sh @@ -16,6 +16,7 @@ # Function Index: # - __remote_cleanup__ # - __prompt_for_params__ +# - __test_remote_connection__ # - __ssh_exec__ # - __scp_exec__ # - __scp_exec_recursive__ @@ -95,89 +96,117 @@ __prompt_for_params__() { return 0 } +# --- __test_remote_connection__ ---------------------------------------------- +# @function __test_remote_connection__ +# @description Test SSH connection to remote node with given credentials +# @usage __test_remote_connection__ +# @param 1 Node IP address +# @param 2 Password (empty if using SSH keys) +# @param 3 Username +# @param 4 Port number +# @return 0 if connection successful, 1 otherwise +__test_remote_connection__() { + local node_ip="$1" + local node_pass="$2" + local username="$3" + local port="$4" + + # Try simple echo command to test connection + if __ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" "echo test" &>/dev/null; then + return 0 + else + return 1 + fi +} + # Helper: Execute SSH command with appropriate auth method -# Args: node_ip node_pass username command +# Args: node_ip node_pass username port command # Returns: output of ssh command __ssh_exec__() { local node_ip="$1" local node_pass="$2" local username="$3" - shift 3 + local port="$4" + shift 4 local command="$*" if [[ "$USE_SSH_KEYS" == "true" ]] || [[ -z "$node_pass" ]]; then - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${username}@${node_ip}" "$command" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p "$port" "${username}@${node_ip}" "$command" else - sshpass -p "$node_pass" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${username}@${node_ip}" "$command" + sshpass -p "$node_pass" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p "$port" "${username}@${node_ip}" "$command" fi } # Helper: Execute SCP with appropriate auth method -# Args: node_ip node_pass username source destination +# Args: node_ip node_pass username port source destination # Returns: exit code of scp __scp_exec__() { local node_ip="$1" local node_pass="$2" local username="$3" - local source="$4" - local destination="$5" + local port="$4" + local source="$5" + local destination="$6" if [[ "$USE_SSH_KEYS" == "true" ]] || [[ -z "$node_pass" ]]; then - scp -q -o StrictHostKeyChecking=no "$source" "${username}@$node_ip:$destination" + scp -q -o StrictHostKeyChecking=no -P "$port" "$source" "${username}@$node_ip:$destination" else - sshpass -p "$node_pass" scp -q -o StrictHostKeyChecking=no "$source" "${username}@$node_ip:$destination" + sshpass -p "$node_pass" scp -q -o StrictHostKeyChecking=no -P "$port" "$source" "${username}@$node_ip:$destination" fi } # Helper: Execute SCP recursive with appropriate auth method -# Args: node_ip node_pass username source destination +# Args: node_ip node_pass username port source destination # Returns: exit code of scp __scp_exec_recursive__() { local node_ip="$1" local node_pass="$2" local username="$3" - local source="$4" - local destination="$5" + local port="$4" + local source="$5" + local destination="$6" if [[ "$USE_SSH_KEYS" == "true" ]] || [[ -z "$node_pass" ]]; then - scp -q -r -o StrictHostKeyChecking=no "$source" "${username}@$node_ip:$destination" + scp -q -r -o StrictHostKeyChecking=no -P "$port" "$source" "${username}@$node_ip:$destination" else - sshpass -p "$node_pass" scp -q -r -o StrictHostKeyChecking=no "$source" "${username}@$node_ip:$destination" + sshpass -p "$node_pass" scp -q -r -o StrictHostKeyChecking=no -P "$port" "$source" "${username}@$node_ip:$destination" fi } # Helper: Download file from remote with appropriate auth method -# Args: node_ip node_pass username remote_path local_path +# Args: node_ip node_pass username port remote_path local_path # Returns: exit code of scp __scp_download__() { local node_ip="$1" local node_pass="$2" local username="$3" - local remote_path="$4" - local local_path="$5" + local port="$4" + local remote_path="$5" + local local_path="$6" if [[ "$USE_SSH_KEYS" == "true" ]] || [[ -z "$node_pass" ]]; then - scp -q -o StrictHostKeyChecking=no "${username}@$node_ip:$remote_path" "$local_path" + scp -q -o StrictHostKeyChecking=no -P "$port" "${username}@$node_ip:$remote_path" "$local_path" else - sshpass -p "$node_pass" scp -q -o StrictHostKeyChecking=no "${username}@$node_ip:$remote_path" "$local_path" + sshpass -p "$node_pass" scp -q -o StrictHostKeyChecking=no -P "$port" "${username}@$node_ip:$remote_path" "$local_path" fi } # Execute workflow on single remote node -# Args: node_name node_ip node_pass username script_path script_relative script_dir_relative param_line +# Args: node_name node_ip node_pass username port script_path script_relative script_dir_relative param_line # Returns: 0 on success, 1 on failure __execute_on_remote_node__() { local node_name="$1" local node_ip="$2" local node_pass="$3" local username="$4" - local script_path="$5" - local script_relative="$6" - local script_dir_relative="$7" - local param_line="$8" + local port="$5" + local script_path="$6" + local script_relative="$7" + local script_dir_relative="$8" + local param_line="$9" echo "----------------------------------------" - echo "Target: $node_name ($node_ip)" + echo "Target: $node_name ($node_ip:$port)" echo "----------------------------------------" echo @@ -186,7 +215,7 @@ __execute_on_remote_node__() { __log_info__ "Cleaning and creating remote directory structure on $node_name" "REMOTE" local ssh_output - if ! ssh_output=$(__ssh_exec__ "$node_ip" "$node_pass" "$username" \ + if ! ssh_output=$(__ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" \ "rm -rf $REMOTE_TEMP_DIR && mkdir -p $REMOTE_TEMP_DIR/{Utilities,Host,LXC,Storage,VirtualMachines,Networking,Cluster,Security,HighAvailability,Firewall,Resources,RemoteManagement}" 2>&1); then # Check if interrupted if [[ $REMOTE_INTERRUPTED -eq 1 ]]; then @@ -213,8 +242,8 @@ __execute_on_remote_node__() { __info__ "Transferring files..." __log_info__ "Transferring utilities and script to $node_name" "REMOTE" - # Create tarball for faster transfer - local temp_tar="/tmp/remote_exec_$$.tar.gz" + # Create tarball for faster transfer (use timestamp to avoid collision in same GUI session) + local temp_tar="/tmp/remote_exec_$$_$(date +%s%N).tar.gz" tar -czf "$temp_tar" -C . Utilities "$script_dir_relative/$(basename "$script_path")" 2>/dev/null || { # Fallback to individual transfers if tar fails __update__ "Tar failed, using individual transfers..." @@ -224,7 +253,7 @@ __execute_on_remote_node__() { return 1 fi - if ! __scp_exec_recursive__ "$node_ip" "$node_pass" "$username" "Utilities/*.sh" "$REMOTE_TEMP_DIR/Utilities/" 2>/dev/null; then + if ! __scp_exec_recursive__ "$node_ip" "$node_pass" "$username" "$port" "Utilities/*.sh" "$REMOTE_TEMP_DIR/Utilities/" 2>/dev/null; then # Check if interrupted or actual failure if [[ $REMOTE_INTERRUPTED -eq 1 ]]; then return 1 @@ -239,7 +268,7 @@ __execute_on_remote_node__() { return 1 fi - if ! __scp_exec__ "$node_ip" "$node_pass" "$username" "$script_path" "$REMOTE_TEMP_DIR/$script_dir_relative/" 2>/dev/null; then + if ! __scp_exec__ "$node_ip" "$node_pass" "$username" "$port" "$script_path" "$REMOTE_TEMP_DIR/$script_dir_relative/" 2>/dev/null; then # Check if interrupted or actual failure if [[ $REMOTE_INTERRUPTED -eq 1 ]]; then return 1 @@ -260,14 +289,14 @@ __execute_on_remote_node__() { # If tar succeeded, transfer and extract if [[ -f "$temp_tar" ]]; then - if __scp_exec__ "$node_ip" "$node_pass" "$username" "$temp_tar" "/tmp/" 2>/dev/null; then + if __scp_exec__ "$node_ip" "$node_pass" "$username" "$port" "$temp_tar" "/tmp/" 2>/dev/null; then # Check if interrupted if [[ $REMOTE_INTERRUPTED -eq 1 ]]; then rm -f "$temp_tar" return 1 fi - __ssh_exec__ "$node_ip" "$node_pass" "$username" \ + __ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" \ "tar -xzf /tmp/$(basename "$temp_tar") -C $REMOTE_TEMP_DIR && rm /tmp/$(basename "$temp_tar")" 2>/dev/null __ok__ "Files transferred (tarball)" __log_info__ "Files transferred successfully (tarball)" "REMOTE" @@ -290,80 +319,79 @@ __execute_on_remote_node__() { return 1 fi - # Execute script + # Execute script remotely __info__ "Executing script..." __log_info__ "Executing $script_relative on remote with args: $param_line" "REMOTE" __log_debug__ "REMOTE_LOG_LEVEL in RemoteExecutor: $REMOTE_LOG_LEVEL" "REMOTE" - local remote_log="/tmp/proxmox_remote_execution_$$.log" - local remote_debug_log="/tmp/proxmox_remote_debug_$$.log" + # Use timestamp-based unique ID instead of PID to avoid log collision across multiple executions + local exec_id="$$_$(date +%s%N)" + local remote_log="/tmp/proxmox_remote_execution_${exec_id}.log" + local remote_debug_log="/tmp/proxmox_remote_debug_${exec_id}.log" local ssh_exit_code=0 + # Execute script and capture output to log file if [[ -n "$param_line" ]]; then - __ssh_exec__ "$node_ip" "$node_pass" "$username" \ - "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin NON_INTERACTIVE=1 DEBIAN_FRONTEND=noninteractive UTILITYPATH='$REMOTE_TEMP_DIR/Utilities' LOG_FILE='$remote_debug_log' LOG_LEVEL=$REMOTE_LOG_LEVEL LOG_CONSOLE=0 && cd $REMOTE_TEMP_DIR && echo '=== Remote Execution Start ===' > $remote_log 2>&1 && echo 'Script: bash $script_relative' >> $remote_log 2>&1 && echo 'Arguments: $param_line' >> $remote_log 2>&1 && echo 'Working directory: '\$(pwd) >> $remote_log 2>&1 && echo 'UTILITYPATH: '\$UTILITYPATH >> $remote_log 2>&1 && echo 'LOG_FILE: '\$LOG_FILE >> $remote_log 2>&1 && echo 'LOG_LEVEL: '\$LOG_LEVEL >> $remote_log 2>&1 && echo 'LOG_LEVEL (actual): $REMOTE_LOG_LEVEL' >> $remote_log 2>&1 && echo '===================================' >> $remote_log 2>&1 && eval bash $script_relative $param_line >> $remote_log 2>&1; echo \$? > ${remote_log}.exit" + __ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" \ + "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin NON_INTERACTIVE=1 DEBIAN_FRONTEND=noninteractive UTILITYPATH='$REMOTE_TEMP_DIR/Utilities' LOG_FILE='$remote_debug_log' LOG_LEVEL=$REMOTE_LOG_LEVEL LOG_CONSOLE=0 && cd $REMOTE_TEMP_DIR && { echo '=== Remote Execution Start ==='; echo 'Script: bash $script_relative'; echo 'Arguments: $param_line'; echo 'Working directory: '\$(pwd); echo 'UTILITYPATH: '\$UTILITYPATH; echo 'LOG_FILE: '\$LOG_FILE; echo 'LOG_LEVEL: '\$LOG_LEVEL; echo 'LOG_LEVEL (actual): $REMOTE_LOG_LEVEL'; echo '==================================='; eval bash $script_relative $param_line; echo \$? > ${remote_log}.exit; } >> $remote_log 2>&1" + ssh_exit_code=$? else - __ssh_exec__ "$node_ip" "$node_pass" "$username" \ - "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin NON_INTERACTIVE=1 DEBIAN_FRONTEND=noninteractive UTILITYPATH='$REMOTE_TEMP_DIR/Utilities' LOG_FILE='$remote_debug_log' LOG_LEVEL=$REMOTE_LOG_LEVEL LOG_CONSOLE=0 && cd $REMOTE_TEMP_DIR && echo '=== Remote Execution Start ===' > $remote_log 2>&1 && echo 'Script: bash $script_relative' >> $remote_log 2>&1 && echo 'Working directory: '\$(pwd) >> $remote_log 2>&1 && echo 'UTILITYPATH: '\$UTILITYPATH >> $remote_log 2>&1 && echo 'LOG_FILE: '\$LOG_FILE >> $remote_log 2>&1 && echo 'LOG_LEVEL: '\$LOG_LEVEL >> $remote_log 2>&1 && echo 'LOG_LEVEL (actual): $REMOTE_LOG_LEVEL' >> $remote_log 2>&1 && echo '===================================' >> $remote_log 2>&1 && bash $script_relative >> $remote_log 2>&1; echo \$? > ${remote_log}.exit" + __ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" \ + "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin NON_INTERACTIVE=1 DEBIAN_FRONTEND=noninteractive UTILITYPATH='$REMOTE_TEMP_DIR/Utilities' LOG_FILE='$remote_debug_log' LOG_LEVEL=$REMOTE_LOG_LEVEL LOG_CONSOLE=0 && cd $REMOTE_TEMP_DIR && { echo '=== Remote Execution Start ==='; echo 'Script: bash $script_relative'; echo 'Working directory: '\$(pwd); echo 'UTILITYPATH: '\$UTILITYPATH; echo 'LOG_FILE: '\$LOG_FILE; echo 'LOG_LEVEL: '\$LOG_LEVEL; echo 'LOG_LEVEL (actual): $REMOTE_LOG_LEVEL'; echo '==================================='; bash $script_relative; echo \$? > ${remote_log}.exit; } >> $remote_log 2>&1" + ssh_exit_code=$? fi - __ok__ "Script execution complete" - - # Retrieve exit code - local temp_exit_file="/tmp/remote_exit_$$" - if __scp_download__ "$node_ip" "$node_pass" "$username" "${remote_log}.exit" "$temp_exit_file" 2>/dev/null; then - ssh_exit_code=$(cat "$temp_exit_file") + # Get final exit code from file if available + local temp_exit_file="/tmp/remote_exit_${exec_id}" + if __scp_download__ "$node_ip" "$node_pass" "$username" "$port" "${remote_log}.exit" "$temp_exit_file" 2>/dev/null; then + local script_exit_code=$(cat "$temp_exit_file" 2>/dev/null) + if [[ -n "$script_exit_code" ]] && [[ "$script_exit_code" =~ ^[0-9]+$ ]]; then + ssh_exit_code=$script_exit_code + fi rm -f "$temp_exit_file" - else - ssh_exit_code=1 fi + __ok__ "Script execution complete" + __log_info__ "Script execution completed with exit code: $ssh_exit_code" "REMOTE" - # Retrieve and display both log files - local local_remote_log="/tmp/remote_${node_name}_$$.log" - local local_debug_log="/tmp/remote_${node_name}_$$.debug.log" - - # Retrieve stdout/stderr log - if __scp_download__ "$node_ip" "$node_pass" "$username" "$remote_log" "$local_remote_log" 2>/dev/null; then - __log_info__ "Retrieved remote execution log from $node_name" "REMOTE" - echo - echo "--- Output from $node_name ---" - cat "$local_remote_log" - echo "--- End output ---" - echo - - # Retrieve debug log if it exists - if __scp_download__ "$node_ip" "$node_pass" "$username" "$remote_debug_log" "$local_debug_log" 2>/dev/null; then - __log_info__ "Retrieved debug log from $node_name" "REMOTE" - - # Only show debug log if it has content - if [[ -s "$local_debug_log" ]]; then - echo - echo "--- Debug Log from $node_name (LOG_LEVEL=$REMOTE_LOG_LEVEL) ---" - cat "$local_debug_log" - echo "--- End debug log ---" - echo - fi + # Download final log file and debug logs + local local_remote_log="/tmp/remote_${node_name}_${exec_id}.log" + local local_debug_log="/tmp/remote_${node_name}_${exec_id}.debug.log" - # Append debug log to main log file - cat "$local_debug_log" >>"$LOG_FILE" 2>/dev/null || true - echo "Debug log saved to: $local_debug_log" - else - __log_debug__ "No debug log available from $node_name" "REMOTE" + # Ensure we have the final version of the log + __scp_download__ "$node_ip" "$node_pass" "$username" "$port" "$remote_log" "$local_remote_log" 2>/dev/null || true + + # Retrieve debug log if it exists + if __scp_download__ "$node_ip" "$node_pass" "$username" "$port" "$remote_debug_log" "$local_debug_log" 2>/dev/null; then + __log_info__ "Retrieved debug log from $node_name" "REMOTE" + + # Show debug log if it has content (not shown live) + if [[ -s "$local_debug_log" ]]; then + echo + echo "--- Debug Log from $node_name (LOG_LEVEL=$REMOTE_LOG_LEVEL) ---" + cat "$local_debug_log" + echo "--- End debug log ---" + echo fi - echo "Output log saved to: $local_remote_log" - cat "$local_remote_log" >>"$LOG_FILE" + # Append debug log to main log file + cat "$local_debug_log" >>"$LOG_FILE" 2>/dev/null || true + echo "Debug log saved to: $local_debug_log" else - __log_warn__ "Could not retrieve remote log from $node_name" "REMOTE" + __log_debug__ "No debug log available from $node_name" "REMOTE" + fi + + echo "Output log saved to: $local_remote_log" + if [[ -f "$local_remote_log" ]]; then + cat "$local_remote_log" >>"$LOG_FILE" fi # Cleanup CURRENT_MESSAGE="Cleaning up..." __update__ "$CURRENT_MESSAGE" __log_info__ "Cleaning up remote directory: $REMOTE_TEMP_DIR" "REMOTE" - __ssh_exec__ "$node_ip" "$node_pass" "$username" \ + __ssh_exec__ "$node_ip" "$node_pass" "$username" "$port" \ "rm -rf $REMOTE_TEMP_DIR $remote_log $remote_debug_log ${remote_log}.exit" 2>/dev/null || __log_warn__ "Cleanup failed (non-critical)" "REMOTE" __ok__ "Cleanup complete" @@ -376,7 +404,13 @@ __execute_on_remote_node__() { __log_error__ "$node_name execution failed with exit code: $ssh_exit_code" "REMOTE" fi - return "$ssh_exit_code" + # Ensure exit code is numeric before returning + if [[ "$ssh_exit_code" =~ ^[0-9]+$ ]]; then + return "$ssh_exit_code" + else + __log_error__ "Invalid exit code: '$ssh_exit_code', returning 1" "REMOTE" + return 1 + fi } # Execute script on remote target(s) @@ -433,8 +467,11 @@ __execute_remote_script__() { # Get username safely local node_username="${NODE_USERNAMES[$node_name]:-$DEFAULT_USERNAME}" + # Get port safely + local node_port="${NODE_PORTS[$node_name]:-$DEFAULT_PORT}" + # Execute on this node (always continue to next node regardless of result) - if __execute_on_remote_node__ "$node_name" "$node_ip" "$node_pass" "$node_username" "$script_path" "$script_relative" "$script_dir_relative" "$param_line"; then + if __execute_on_remote_node__ "$node_name" "$node_ip" "$node_pass" "$node_username" "$node_port" "$script_path" "$script_relative" "$script_dir_relative" "$param_line"; then ((success_count += 1)) else ((fail_count += 1)) @@ -470,9 +507,12 @@ __execute_remote_script__() { ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Fixed log filename collision issue when GUI.sh runs multiple executions +# - 2026-01-08: Added __test_remote_connection__ for password validation +# - 2025-12-18: Added SSH port configuration support for proxy/jump host setups # - 2025-11-24: Validated against CONTRIBUTING.md and fixed ShellCheck issues # - 2025-11-24: Added conditional sourcing of utility dependencies # - 2025-11-24: Fixed variable quoting issues (SC2086) diff --git a/Utilities/_Utilities.md b/Utilities/_Utilities.md index a662fd3..c28e4f3 100644 --- a/Utilities/_Utilities.md +++ b/Utilities/_Utilities.md @@ -1,6 +1,6 @@ # ProxmoxScripts Utility Functions Reference -**Auto-generated documentation** - Last updated: 2025-11-25 10:18:58 +**Auto-generated documentation** - Last updated: 2026-01-09 14:42:42 --- @@ -252,7 +252,7 @@ __validate_ip__ "192.168.1.1" "IP Address" | `__get_server_lxc__` | Cluster | Retrieves the VMIDs for all LXC containers on a specific server | | `__get_server_vms__` | Cluster | Retrieves the VMIDs for all VMs (QEMU) on a specific server | | `__get_vm_info__` | Operations | Get comprehensive VM information | -| `__get_vm_node__` | Cluster | Gets the node name where a specific VM is located in the cluster | +| `__get_vm_node__` | Cluster | Get the node name where a VM is located | | `__gradient_print__` | Colors | Prints multi-line text with a vertical color gradient | | `__handle_err__` | Communication | Handles errors by stopping the spinner and printing error details including the line number, exit code, and failing command | | `__info__` | Communication | Prints an informational message in bold yellow and starts the rainbow spinner | @@ -273,6 +273,7 @@ __validate_ip__ "192.168.1.1" "IP Address" | `__log_function_exit__` | Logger | Log function exit with return code | | `__log_info__` | Logger | Log info message | | `__log_section__` | Logger | Log section separator | +| `__log_trace__` | Logger | Log trace message | | `__log_var__` | Logger | Log variable value | | `__log_warn__` | Logger | Log warning message | | `__net_bulk_set_bridge__` | Network | Change bridge for multiple VMs | @@ -336,6 +337,7 @@ __validate_ip__ "192.168.1.1" "IP Address" | `__state_validate__` | StateManager | Validate a state file | | `__stop_spin__` | Communication | Stops the running spinner process (if any) and restores the cursor | | `__success__` | Communication | Alias for __ok__ for backward compatibility | +| `__test_remote_connection__` | RemoteExecutor | Test SSH connection to remote node with given credentials | | `__update__` | Communication | Updates the text displayed next to the spinner without stopping it | | `__validate_ctid__` | Cluster | Validates that a CTID exists and is a container (lxc), not a VM | | `__validate_vm_id_range__` | Cluster | Validates that VM IDs are numeric and in correct order | @@ -734,17 +736,18 @@ For __get_server_vms__ "local", the output might be: 401 402 ``` --- ### `__get_vm_node__` -**Description**: Gets the node name where a specific VM is located in the cluster. Returns empty string if VM is not found. +**Description**: Get the node name where a VM is located **Usage**: ```bash -local node=$(__get_vm_node__ 400) +local node=$(__get_vm_node__ ) ``` **Parameters**: - 1 The VMID to locate. -**Returns**: Prints the node name to stdout, or empty string if not found. +- 1 VM ID +**Returns**: Prints node name to stdout **Example Output**: ``` -For __get_vm_node__ 400, the output might be: pve01 +For __get_vm_node__ 400, the output might be: pve01 --- __get_vm_node__ --------------------------------------------------------- ``` --- ### `__get_ct_node__` @@ -1160,6 +1163,7 @@ Shows "Selected script", top comments, and example invocations sections. - `__clear_remote_targets__` - `__get_node_ip__` - `__get_node_username__` +- `__get_node_port__` - `__node_exists__` - `__get_available_nodes__` - `__count_available_nodes__` @@ -1338,7 +1342,7 @@ For __get_name_from_ip__ "192.168.1.23", the output is: pve03 # Logger.sh -**Purpose**: !/bin/bash Centralized logging utility for ProxmoxScripts Provides consistent, structured logging across all scripts __log__ "INFO" "Message here" __log__ "ERROR" "Something failed" __log__ "DEBUG" "Debug information" Environment Variables: LOG_LEVEL - Minimum level to log (DEBUG, INFO, WARN, ERROR) - default: INFO +**Purpose**: !/bin/bash Centralized logging utility for ProxmoxScripts Provides consistent, structured logging across all scripts __log__ "INFO" "Message here" __log__ "ERROR" "Something failed" __log__ "DEBUG" "Debug information" Environment Variables: LOG_LEVEL - Minimum level to log (TRACE, DEBUG, INFO, WARN, ERROR) - default: INFO **Usage**: ```bash @@ -1348,6 +1352,7 @@ source "${UTILITYPATH}/Logger.sh" **Functions**: - `__get_log_priority__` - `__log__` +- `__log_trace__` - `__log_debug__` - `__log_info__` - `__log_warn__` @@ -1369,10 +1374,13 @@ source "${UTILITYPATH}/Logger.sh" __log__ [category] ``` **Parameters**: -- level Log level (DEBUG, INFO, WARN, ERROR) +- level Log level (TRACE, DEBUG, INFO, WARN, ERROR) - message Message to log - category Optional category/component name --- +### `__log_trace__` +**Description**: Log trace message +--- ### `__log_debug__` **Description**: Log debug message --- @@ -1791,6 +1799,7 @@ __net_migrate_network__ [options] - Consistent return codes and error messages - Testable and mockable functions - State management helpers +- Optimized to minimize redundant API queries **Usage**: ```bash @@ -1860,14 +1869,19 @@ source "${UTILITYPATH}/Operations.sh" #### Functions in Operations.sh ### `__vm_exists__` -**Description**: Check if a VM exists (cluster-wide). +**Description**: Check if a VM exists (cluster-wide). Optionally returns node via stdout. **Usage**: ```bash -__vm_exists__ +if __vm_exists__ "$vmid"; then ...; fi OR node=$(__vm_exists__ "$vmid" --get-node) ``` **Parameters**: - 1 VM ID -**Returns**: 0 if exists, 1 if not +- 2 Optional: --get-node to output node name to stdout +**Returns**: 0 if exists (with node on stdout if --get-node), 1 if not +**Example Output**: +``` +node=$(__vm_exists__ "$vmid" --get-node) && echo "VM on $node" +``` --- ### `__vm_get_status__` **Description**: Get VM status (running, stopped, paused, etc). @@ -1901,16 +1915,18 @@ __vm_start__ [options] **Returns**: 0 on success, 1 on error --- ### `__vm_stop__` -**Description**: Stop a VM (cluster-aware). +**Description**: Stop a VM (cluster-aware). Sends SIGTERM then SIGKILL. **Usage**: ```bash __vm_stop__ [--timeout ] [--force] ``` **Parameters**: - 1 VM ID -- --timeout Timeout in seconds before force stop -- --force Force stop immediately +- --timeout Timeout in seconds (optional) +- --force Ignored for compatibility (qm stop is forceful by default) **Returns**: 0 on success, 1 on error +**Notes**: +- The --force flag is accepted but ignored for Proxmox version compatibility --- ### `__vm_set_config__` **Description**: Set VM configuration parameter. @@ -1935,14 +1951,15 @@ __vm_get_config__ **Returns**: Prints value to stdout, returns 1 on error --- ### `__ct_exists__` -**Description**: Check if a CT exists. +**Description**: Check if a CT exists. Optionally returns node via stdout. **Usage**: ```bash -__ct_exists__ +if __ct_exists__ "$ctid"; then ...; fi OR node=$(__ct_exists__ "$ctid" --get-node) ``` **Parameters**: - 1 CT ID -**Returns**: 0 if exists, 1 if not +- 2 Optional: --get-node to output node name to stdout +**Returns**: 0 if exists (with node on stdout if --get-node), 1 if not --- ### `__ct_get_status__` **Description**: Get CT status. @@ -2589,6 +2606,7 @@ __require_root_and_proxmox__ **Functions**: - `__remote_cleanup__` - `__prompt_for_params__` +- `__test_remote_connection__` - `__ssh_exec__` - `__scp_exec__` - `__scp_exec_recursive__` @@ -2598,6 +2616,21 @@ __require_root_and_proxmox__ --- +#### Functions in RemoteExecutor.sh + +### `__test_remote_connection__` +**Description**: Test SSH connection to remote node with given credentials +**Usage**: +```bash +__test_remote_connection__ +``` +**Parameters**: +- 1 Node IP address +- 2 Password (empty if using SSH keys) +- 3 Username +- 4 Port number +**Returns**: 0 if connection successful, 1 otherwise +--- # RemoteRunAllTests.sh **Purpose**: !/bin/bash Run all test suites on remote Proxmox nodes using RemoteExecutor.sh (same as GUI.sh). Copies entire repository and executes RunAllTests.sh on remote nodes. RemoteRunAllTests.sh --node pt01 RemoteRunAllTests.sh --node 192.168.1.81 RemoteRunAllTests.sh --all-nodes RemoteRunAllTests.sh --node pt01 --verbose RemoteRunAllTests.sh --all-nodes --debug Options: diff --git a/VirtualMachines/Operations/BulkCloneSetIP_Windows.sh b/VirtualMachines/Operations/BulkCloneSetIP_Windows.sh index bcc56da..060dcf9 100644 --- a/VirtualMachines/Operations/BulkCloneSetIP_Windows.sh +++ b/VirtualMachines/Operations/BulkCloneSetIP_Windows.sh @@ -54,8 +54,9 @@ netmask="$(__cidr_to_netmask__ "$startMask")" ############################################################################### # Create a temporary .bat file with netsh commands for Windows IP reconfiguration +# Use timestamp to avoid collision when running multiple instances in same session ############################################################################### -tempBat="/tmp/ChangeIP.bat.$$" +tempBat="/tmp/ChangeIP.bat.$$_$(date +%s%N)" cat <<'EOF' >"$tempBat" @echo off :: ChangeIP.bat @@ -140,9 +141,10 @@ main ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Fixed temp file collision when running multiple instances in same session # - 2025-11-24: Fixed script name in header to match filename # - 2025-11-24: Refactored main logic into main() function # - 2025-11-24: Fixed variable name mismatches (COUNT, BASE_VM_ID) diff --git a/VirtualMachines/Operations/BulkDelete.sh b/VirtualMachines/Operations/BulkDelete.sh index 0521a83..a677e39 100644 --- a/VirtualMachines/Operations/BulkDelete.sh +++ b/VirtualMachines/Operations/BulkDelete.sh @@ -70,13 +70,20 @@ main() { delete_vm_callback() { local vmid="$1" - # Unprotect VM - __vm_set_config__ "$vmid" --protection 0 2>/dev/null || return 1 - - # Stop if running + # Stop if running (qm stop is forceful by default) if __vm_is_running__ "$vmid"; then - __vm_stop__ "$vmid" --force 2>/dev/null || true - __vm_wait_for_status__ "$vmid" "stopped" --timeout 30 2>/dev/null || true + if __vm_stop__ "$vmid"; then + # Stop command succeeded, wait for VM to reach stopped state + __vm_wait_for_status__ "$vmid" "stopped" --timeout 30 2>/dev/null || { + # Timeout waiting for stopped status - skip this VM + echo "Error: VM $vmid did not stop within timeout" >&2 + return 1 + } + else + # Stop command failed immediately - skip this VM + echo "Error: Failed to issue stop command for VM $vmid" >&2 + return 1 + fi fi # Destroy VM using remote execution utility @@ -104,9 +111,11 @@ main ############################################################################### # Script notes: ############################################################################### -# Last checked: 2025-11-24 +# Last checked: 2026-01-08 # # Changes: +# - 2026-01-08: Removed --protection 0 call (unprotection must be handled by Options/BulkToggleProtectionMode.sh) +# - 2026-01-08: Improved stop command error handling with better timeout and error messages # - 2025-10-14: Converted to cluster-wide with safety features # - 2025-10-27: Updated to follow contributing guide with proper BulkOperations framework usage # diff --git a/nodes.json.template b/nodes.json.template index b0d1370..3490845 100644 --- a/nodes.json.template +++ b/nodes.json.template @@ -3,12 +3,14 @@ { "name": "node1", "ip": "192.168.1.11", + "port": 22, "username": "root", "ssh_keys": true }, { "name": "node2", "ip": "192.168.1.12", + "port": 22, "username": "root", "ssh_keys": false }