From 2bfeb4e0d63711360ec75e7758d4b4e0eb3c47c7 Mon Sep 17 00:00:00 2001
From: hkumarmk <hkumar@d4devops.org>
Date: Tue, 5 Apr 2016 03:08:41 +0530
Subject: [PATCH 1/5] Use ssh agent to connect between remote machines

Current method - copy ssh private key to remote node1 and then try to
ssh using that key file - is not required, as ssh-agent forwarding will
enable such facility without ssh private keys present in subsequent
machines. You just need private key on test node and no need to copy
private key to anywhere else, just run ssh-agent on test node and run
the command on remote nodes (even between remote nodes with agent
forwarding enabled.

Added tools/common.sh with common bash functions.
Added a new function "prepare"  and also added a new
parameter to run only "prepare". This will be helpful in case of
manually running tests which actually need running tools/configure.py as
well as starting ssh-agent on test node.
---
 fixtures/vm_test.py | 19 ++++++-------------
 run_ci.sh           | 16 ++++++----------
 tcutils/util.py     |  1 +
 tools/common.sh     | 23 +++++++++++++++++++++++
 4 files changed, 36 insertions(+), 23 deletions(-)
 create mode 100644 tools/common.sh

diff --git a/fixtures/vm_test.py b/fixtures/vm_test.py
index be49ba1f1..79c3fd445 100644
--- a/fixtures/vm_test.py
+++ b/fixtures/vm_test.py
@@ -1714,21 +1714,14 @@ def scp_file_to_vm(self, file, vm_ip, dest_vm_username='ubuntu'):
         timeout = math.floor(40 * float(delay_factor))
 
         try:
-            self.orch.put_key_file_to_host(self.vm_node_ip)
-            with hide('everything'):
-                with settings(host_string='%s@%s' % (
-                              host['username'], self.vm_node_ip),
-                              password=host['password'],
-                              warn_only=True, abort_on_prompts=False):
-                    self.get_rsa_to_vm()
-                    i = 'timeout %d scp -o StrictHostKeyChecking=no -i id_rsa %s %s@[%s]:' % (
-                        timeout, file, dest_vm_username, vm_ip)
-                    cmd_outputs = self.run_cmd_on_vm(
-                        cmds=[i], timeout=timeout + 10)
-                    self.logger.debug(cmd_outputs)
+            i = 'timeout %d scp -o StrictHostKeyChecking=no %s %s@[%s]:' % (
+                timeout, file, dest_vm_username, vm_ip)
+            cmd_outputs = self.run_cmd_on_vm(
+                cmds=[i], timeout=timeout + 10)
+            self.logger.debug(cmd_outputs)
         except Exception, e:
             self.logger.exception(
-                'Exception occured while trying to scp the file ')
+                'Exception occured while trying to scp the file\n%s' % e)
     # end scp_file_to_vm
 
     def put_pub_key_to_vm(self):
diff --git a/run_ci.sh b/run_ci.sh
index 803659da3..c84fafded 100755
--- a/run_ci.sh
+++ b/run_ci.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+source tools/common.sh
 
 function die
 {
@@ -13,6 +14,7 @@ function usage {
   echo "Usage: $0 [OPTION]..."
   echo "Run Contrail test suite"
   echo ""
+  echo "  -p, --prepare            Run only preparation steps"
   echo "  -t, --parallel           Run testr in parallel"
   echo "  -h, --help               Print this usage message"
   echo "  -m, --send-mail          Send the report at the end"
@@ -20,6 +22,7 @@ function usage {
   echo "  --contrail-fab-path      Contrail fab path, default to /opt/contrail/utils"
   echo "  -- [TESTROPTIONS]        After the first '--' you can pass arbitrary arguments to testr "
 }
+
 testrargs=""
 debug=0
 force=0
@@ -34,7 +37,7 @@ contrail_fab_path='/opt/contrail/utils'
 test_tag='suite1'
 export SCRIPT_TS=${SCRIPT_TS:-$(date +"%Y_%m_%d_%H_%M_%S")}
 
-if ! options=$(getopt -o UthdC:lLmc: -l upload,parallel,help,debug,config:,logging,logging-config,send-mail,concurrency:,contrail-fab-path: -- "$@")
+if ! options=$(getopt -o pUthdC:lLmc: -l prepare,upload,parallel,help,debug,config:,logging,logging-config,send-mail,concurrency:,contrail-fab-path: -- "$@")
 then
     # parse error
     usage
@@ -45,6 +48,7 @@ eval set -- $options
 first_uu=yes
 while [ $# -gt 0 ]; do
   case "$1" in
+    -p|--prepare) prepare; exit;;
     -h|--help) usage; exit;;
     -U|--upload) upload=1;;
     -d|--debug) debug=1;;
@@ -64,15 +68,7 @@ done
 testrargs+=" $test_tag"
 export TAGS="$test_tag"
 
-if [ -n "$config_file" ]; then
-    config_file=`readlink -f "$config_file"`
-    export TEST_CONFIG_DIR=`dirname "$config_file"`
-    export TEST_CONFIG_FILE=`basename "$config_file"`
-fi
-
-if [ ! -f "$config_file" ]; then
-    python tools/configure.py $(readlink -f .) -p $contrail_fab_path
-fi
+prepare
 
 if [ $logging -eq 1 ]; then
     if [ ! -f "$logging_config" ]; then
diff --git a/tcutils/util.py b/tcutils/util.py
index a66de8b76..ba304fd18 100644
--- a/tcutils/util.py
+++ b/tcutils/util.py
@@ -252,6 +252,7 @@ def run_cmd_through_node(host_string, cmd, password=None, gateway=None,
                                       shell=shell,
                                       disable_known_hosts=True,
                                       abort_on_prompts=False):
+        env.forward_agent = True
         gateway_hoststring = gateway if re.match(r'\w+@[\d\.]+:\d+', gateway) else gateway + ':22'
         node_hoststring = host_string if re.match(r'\w+@[\d\.]+:\d+', host_string) else host_string + ':22'
         if password:
diff --git a/tools/common.sh b/tools/common.sh
new file mode 100644
index 000000000..6e5c18e46
--- /dev/null
+++ b/tools/common.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+function prepare {
+  if [ -n "$config_file" ]; then
+    config_file=`readlink -f "$config_file"`
+    export TEST_CONFIG_DIR=`dirname "$config_file"`
+    export TEST_CONFIG_FILE=`basename "$config_file"`
+  fi
+
+  if [ ! -f "$config_file" ]; then
+    python tools/configure.py $(readlink -f .) -p $contrail_fab_path
+  fi
+
+  #Start ssh-agent if not there
+  if [ -z "$SSH_AUTH_SOCK" ] ; then
+    eval `ssh-agent -s`
+  fi
+  if [ -z "$SSH_AUTH_SOCK" ] ; then
+    echo "Error: SSH agent failed to start"
+    exit 1
+  fi
+  ssh-add
+}

From af10d149e9aaa9554b3ffb756ea714ec9e178a5c Mon Sep 17 00:00:00 2001
From: hkumarmk <hkumar@d4devops.org>
Date: Tue, 5 Apr 2016 12:18:12 +0530
Subject: [PATCH 2/5] Added missing dependency modules in requirements.txt

---
 requirements.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/requirements.txt b/requirements.txt
index cb529ca81..755d30eb3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,5 @@ linecache2
 eventlet
 nose
 tabulate
+fabric
+netaddr

From f57a51213dd05092ffd81a887612857be7b95612 Mon Sep 17 00:00:00 2001
From: hkumarmk <hkumar@d4devops.org>
Date: Tue, 5 Apr 2016 12:24:05 +0530
Subject: [PATCH 3/5] Moved fabric related methods to tcuils.fabutils

This is to refactor certain common code to central place and to reuse
them. Current effort is to consolidate methods which use fabric
operations to single place and avoid importing fabric module to various
other files but to use the methods from fabutils.
---
 tcutils/fabutils.py | 337 ++++++++++++++++++++++++++++++++++++++++++++
 tcutils/util.py     | 323 +-----------------------------------------
 2 files changed, 339 insertions(+), 321 deletions(-)
 create mode 100644 tcutils/fabutils.py

diff --git a/tcutils/fabutils.py b/tcutils/fabutils.py
new file mode 100644
index 000000000..e6fbb8db6
--- /dev/null
+++ b/tcutils/fabutils.py
@@ -0,0 +1,337 @@
+from fabric.operations import get, put, sudo
+from fabric.api import run, env
+from fabric.exceptions import CommandTimeout, NetworkError
+from fabric.contrib.files import exists
+from fabric.context_managers import settings, hide, cd
+
+
+import re
+import logging
+import time
+import random
+import os
+
+log = logging.getLogger('log01')
+sku_dict = {'2014.1': 'icehouse', '2014.2': 'juno', '2015.1': 'kilo', '12.0': 'liberty'}
+
+
+def get_os_env(var):
+    if var in os.environ:
+        return os.environ.get(var)
+    else:
+        return None
+# end get_os_env
+
+
+def remove_unwanted_output(text):
+    """ Fab output usually has content like [ x.x.x.x ] out : <content>
+    Args:
+        text: text to parse
+    """
+    if not text:
+        return None
+
+    return_list = text.split('\n')
+
+    return_list1 = []
+    for line in return_list:
+        line_split = line.split(' out: ')
+        if len(line_split) == 2:
+            return_list1.append(line_split[1])
+        else:
+            if ' out:' not in line:
+                return_list1.append(line)
+    real_output = '\n'.join(return_list1)
+    return real_output
+
+
+def run_cmd(host_string, cmd, password=None, gateway=None,
+            gateway_password=None, with_sudo=False, timeout=120,
+            as_daemon=False, raw=False, cwd=None, warn_only=True):
+    """ Run command on remote node through another node (gateway).
+        This is useful to run commands on VMs through compute node
+    Args:
+        host_string: host_string on which the command to run
+        password: Password
+        cmd: command
+        gateway: host_string of the node through which host_string will connect
+        gateway_password: Password of gateway hoststring
+        with_sudo: use Sudo
+        timeout: timeout
+        cwd: change directory to provided parameter
+        as_daemon: run in background
+        warn_only: run fab with warn_only
+        raw: If raw is True, will return the fab _AttributeString object itself without removing any unwanted output
+    """
+    if as_daemon:
+        cmd = 'nohup ' + cmd + ' &'
+
+    if cwd:
+        cmd = 'cd %s; %s' % (cd, cmd)
+
+    (username, host_ip) = host_string.split('@')
+
+    if username == 'root':
+        with_sudo = False
+
+    shell = '/bin/bash -l -c'
+
+    if username == 'cirros':
+        shell = '/bin/sh -l -c'
+
+    _run = sudo if with_sudo else run
+
+    # with hide('everything'), settings(host_string=host_string,
+    with settings(
+            host_string=host_string,
+            gateway=gateway,
+            warn_only=warn_only,
+            shell=shell,
+            disable_known_hosts=True,
+            abort_on_prompts=False):
+        env.forward_agent = True
+        gateway_hoststring = (gateway if re.match(r'\w+@[\d\.]+:\d+', gateway)
+                              else gateway + ':22')
+        node_hoststring = (host_string
+                           if re.match(r'\w+@[\d\.]+:\d+', host_string)
+                           else host_string + ':22')
+        if password:
+            env.passwords.update({node_hoststring: password})
+            # If gateway_password is not set, guess same password
+            # (if key is used, it will be tried before password)
+            if not gateway_password:
+                env.passwords.update({gateway_hoststring: password})
+
+        if gateway_password:
+            env.passwords.update({gateway_hoststring: gateway_password})
+            if not password:
+                env.passwords.update({node_hoststring: gateway_password})
+
+        log.debug(cmd)
+        tries = 1
+        output = None
+        while tries > 0:
+            if timeout:
+                try:
+                    output = _run(cmd, timeout=timeout)
+                except CommandTimeout:
+                    pass
+            else:
+                output = _run(cmd)
+            if output and 'Fatal error' in output:
+                tries -= 1
+                time.sleep(5)
+            else:
+                break
+        # end while
+
+        if not raw:
+            real_output = remove_unwanted_output(output)
+        else:
+            real_output = output
+        return real_output
+
+
+def run_netconf_on_node(host_string, password, cmds, op_format='text'):
+    '''
+    Run netconf from node to a VM.Usecase: vSRX or vMX or any netconf supporting device.
+    '''
+    (username, host_ip) = host_string.split('@')
+    timeout = 10
+    device = 'junos'
+    hostkey_verify = "False"
+    # Sometimes, during bootup, there could be some intermittent conn. issue
+    tries = 1
+    output = None
+    copy_fabfile_to_agent()
+    while tries > 0:
+        if 'show' in cmds:
+            cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running get_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"' % (
+                username, password, host_ip, cmds, timeout, device, hostkey_verify, op_format)
+        else:
+            cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running config_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\"' % (
+                username, password, host_ip, cmds, timeout, device, hostkey_verify)
+        log.debug(cmd_str)
+        output = run(cmd_str)
+        log.debug(output)
+        if ((output) and ('Fatal error' in output)):
+            tries -= 1
+            time.sleep(5)
+        else:
+            break
+    # end while
+    return output
+# end run_netconf_on_node
+
+
+def copy_fabfile_to_agent():
+    src = 'tcutils/fabfile.py'
+    dst = '~/fabfile.py'
+    if 'fab_copied_to_hosts' not in env.keys():
+        env.fab_copied_to_hosts = list()
+    if not env.host_string in env.fab_copied_to_hosts:
+        if not exists(dst):
+            put(src, dst)
+        env.fab_copied_to_hosts.append(env.host_string)
+
+
+def _escape_some_chars(text):
+    chars = ['"', '=']
+    for char in chars:
+        text = text.replace(char, '\\\\' + char)
+    return text
+# end escape_chars
+
+
+def run_fab_cmd_on_node(host_string, password, cmd, as_sudo=False, timeout=120, as_daemon=False, raw=False,
+                        warn_only=True):
+    """
+    Run fab command on a node. Usecase : as part of script running on cfgm node, can run a cmd on VM from compute node
+
+    If raw is True, will return the fab _AttributeString object itself without removing any unwanted output
+    """
+    cmd = _escape_some_chars(cmd)
+    (username, host_ip) = host_string.split('@')
+    copy_fabfile_to_agent()
+    cmd_args = '-u %s -p "%s" -H %s -D --hide status,user,running' % (username,
+                password, host_ip)
+    if warn_only:
+        cmd_args+= ' -w '
+    cmd_str = 'fab %s ' % (cmd_args)
+    if as_daemon:
+        cmd_str += '--no-pty '
+        cmd = 'nohup ' + cmd + ' &'
+    if username == 'root':
+        as_sudo = False
+    elif username == 'cirros':
+        cmd_str += ' -s "/bin/sh -l -c" '
+    if as_sudo:
+        cmd_str += 'sudo_command:\"%s\"' % (cmd)
+    else:
+        cmd_str += 'command:\"%s\"' % (cmd)
+    # Sometimes, during bootup, there could be some intermittent conn. issue
+    log.debug(cmd_str)
+    tries = 1
+    output = None
+    while tries > 0:
+        if timeout:
+            try:
+                output = sudo(cmd_str, timeout=timeout)
+                log.debug(output)
+            except CommandTimeout:
+                return output
+        else:
+            output = run(cmd_str)
+        if ((output) and ('Fatal error' in output)):
+            tries -= 1
+            time.sleep(5)
+        else:
+            break
+    # end while
+
+    if not raw:
+        real_output = remove_unwanted_output(output)
+    else:
+        real_output = output
+    return real_output
+# end run_fab_cmd_on_node
+
+
+def fab_put_file_to_vm(host_string, password, src, dest):
+    copy_fabfile_to_agent()
+    (username, host_ip) = host_string.split('@')
+    cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running fput:\"%s\",\"%s\"' % (
+        username, password, host_ip, src, dest)
+    log.debug(cmd_str)
+    output = run(cmd_str)
+    real_output = remove_unwanted_output(output)
+# end fab_put_file_to_vm
+
+
+def sshable(host_string, password=None, gateway=None, gateway_password=None):
+    host_string_split = re.split(r"[@:]", host_string)
+    host_port = host_string_split[2] if len(host_string_split) > 2 else '22'
+    with hide('everything'), settings(host_string=gateway,
+                                      password=gateway_password,
+                                      warn_only=True):
+        if run('nc -w 1 -z %s %s' % (host_string_split[1], host_port)).succeeded:
+            try:
+                run_cmd(host_string, 'uname', password, gateway,
+                        gateway_password, timeout=10)
+                return True
+            except Exception as e:
+                log.error("Error on ssh to %s" % host_string)
+                log.debug(str(e))
+                return False
+
+
+def fab_check_ssh(host_string, password):
+    copy_fabfile_to_agent()
+    (username, host_ip) = host_string.split('@')
+    cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running verify_socket_connection:22' % (
+        username, password, host_ip)
+    log.debug(cmd_str)
+    output = run(cmd_str)
+    log.debug(output)
+    if 'True' in output:
+        return True
+    return False
+# end fab_check_ssh
+
+
+def run_cmd_on_server(issue_cmd, server_ip, username,
+                      password, pty=True, as_sudo=False):
+    with hide('everything'):
+        with settings(
+            host_string='%s@%s' % (username, server_ip), password=password,
+                warn_only=True, abort_on_prompts=False):
+            if as_sudo:
+                output = sudo('%s' % (issue_cmd), pty=pty)
+            else:
+                output = run('%s' % (issue_cmd), pty=pty)
+            return output
+# end run_cmd_on_server
+
+
+def copy_file_to_server(host, src, dest, filename, force=False):
+
+    fname = "%s/%s" % (dest, filename)
+    with settings(host_string='%s@%s' % (host['username'],
+                                         host['ip']), password=host['password'],
+                  warn_only=True, abort_on_prompts=False):
+        if not exists(fname) or force:
+            time.sleep(random.randint(1, 10))
+            put(src, dest)
+# end copy_file_to_server
+
+def get_host_domain_name(host):
+    output = None
+    with settings(hide('everything'), host_string='%s@%s' % (host['username'],
+        host['ip']), password=host['password'],
+        warn_only=True, abort_on_prompts=False):
+        output = run('hostname -d')
+
+    return output
+# end get_host_domain_name
+
+
+def get_build_sku(openstack_node_ip, openstack_node_password='c0ntrail123', user='root'):
+    build_sku = get_os_env("SKU")
+    if build_sku is not None:
+        return str(build_sku).lower()
+    else:
+        host_str='%s@%s' % (user, openstack_node_ip)
+        cmd = 'nova-manage version'
+        tries = 10
+        while not build_sku and tries:
+            try:
+                with hide('everything'), settings(host_string=host_str,
+                                                  user=user,
+                                                  password=openstack_node_password):
+                    output = sudo(cmd)
+                    build_sku = sku_dict[re.findall("[0-9]+.[0-9]+",output)[0]]
+            except NetworkError, e:
+                time.sleep(1)
+                pass
+            tries -= 1
+        return build_sku
diff --git a/tcutils/util.py b/tcutils/util.py
index ba304fd18..89f3bc22c 100644
--- a/tcutils/util.py
+++ b/tcutils/util.py
@@ -5,9 +5,6 @@
 import time
 from collections import defaultdict, MutableMapping
 from netaddr import *
-import pprint
-from fabric.operations import get, put, sudo
-from fabric.api import run, env
 import logging
 import threading
 from functools import wraps
@@ -19,23 +16,17 @@
 import fcntl
 import socket
 import struct
-from fabric.exceptions import CommandTimeout, NetworkError
-from fabric.contrib.files import exists
-from fabric.context_managers import settings, hide, cd
 import ConfigParser
 from testtools.testcase import TestSkipped
 import functools
 import testtools
-from fabfile import *
+
+from fabutils import *
 
 log = logging.getLogger('log01')
-#log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG)
 
-sku_dict = {'2014.1': 'icehouse', '2014.2': 'juno', '2015.1': 'kilo', '12.0': 'liberty'}
 
 # Code borrowed from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
-
-
 def retry(tries=5, delay=3):
     '''Retries a function or method until it returns True.
     delay sets the initial delay in seconds.
@@ -132,262 +123,6 @@ def get_string_match_count(string_list, string_where_to_search):
     return d
 
 
-def get_os_env(var):
-    if var in os.environ:
-        return os.environ.get(var)
-    else:
-        return None
-# end get_os_env
-
-
-def _escape_some_chars(text):
-    chars = ['"', '=']
-    for char in chars:
-        text = text.replace(char, '\\\\' + char)
-    return text
-# end escape_chars
-
-
-def remove_unwanted_output(text):
-    ''' Fab output usually has content like [ x.x.x.x ] out : <content>
-    '''
-    if not text:
-        return None
-
-    return_list = text.split('\n')
-
-    return_list1 = []
-    for line in return_list:
-        line_split = line.split(' out: ')
-        if len(line_split) == 2:
-            return_list1.append(line_split[1])
-        else:
-            if ' out:' not in line:
-                return_list1.append(line)
-    real_output = '\n'.join(return_list1)
-    return real_output
-
-
-def run_netconf_on_node(host_string, password, cmds, op_format='text'):
-    '''
-    Run netconf from node to a VM.Usecase: vSRX or vMX or any netconf supporting device.
-    '''
-    (username, host_ip) = host_string.split('@')
-    timeout = 10
-    device = 'junos'
-    hostkey_verify = "False"
-    # Sometimes, during bootup, there could be some intermittent conn. issue
-    tries = 1
-    output = None
-    copy_fabfile_to_agent()
-    while tries > 0:
-        if 'show' in cmds:
-            cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running get_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"' % (
-                username, password, host_ip, cmds, timeout, device, hostkey_verify, op_format)
-        else:
-            cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running config_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\"' % (
-                username, password, host_ip, cmds, timeout, device, hostkey_verify)
-        log.debug(cmd_str)
-        output = run(cmd_str)
-        log.debug(output)
-        if ((output) and ('Fatal error' in output)):
-            tries -= 1
-            time.sleep(5)
-        else:
-            break
-    # end while
-    return output
-# end run_netconf_on_node
-
-
-def copy_fabfile_to_agent():
-    src = 'tcutils/fabfile.py'
-    dst = '~/fabfile.py'
-    if 'fab_copied_to_hosts' not in env.keys():
-        env.fab_copied_to_hosts = list()
-    if not env.host_string in env.fab_copied_to_hosts:
-        if not exists(dst):
-            put(src, dst)
-        env.fab_copied_to_hosts.append(env.host_string)
-
-def run_cmd_through_node(host_string, cmd, password=None, gateway=None,
-                         gateway_password=None, with_sudo=False, timeout=120,
-                         as_daemon=False, raw=False, cd=None, warn_only=True):
-    """ Run command on remote node through another node (gateway).
-        This is useful to run commands on VMs through compute node
-    Args:
-        host_string: host_string on which the command to run
-        password: Password
-        cmd: command
-        gateway: host_string of the node through which host_string will connect
-        gateway_password: Password of gateway hoststring
-        with_sudo: use Sudo
-        timeout: timeout
-        cd: change directory to provided parameter
-        as_daemon: run in background
-        raw: If raw is True, will return the fab _AttributeString object itself without removing any unwanted output
-    """
-    if as_daemon:
-        cmd = 'nohup ' + cmd + ' &'
-
-    if cd:
-        cmd = 'cd %s; %s' % (cd, cmd)
-
-    (username, host_ip) = host_string.split('@')
-
-    if username == 'root':
-        with_sudo = False
-
-    shell = '/bin/bash -l -c'
-
-    if username == 'cirros':
-        shell = '/bin/sh -l -c'
-
-    _run = sudo if with_sudo else run
-
-    #with hide('everything'), settings(host_string=host_string,
-    with settings(host_string=host_string,
-                                      gateway=gateway,
-                                      warn_only=warn_only,
-                                      shell=shell,
-                                      disable_known_hosts=True,
-                                      abort_on_prompts=False):
-        env.forward_agent = True
-        gateway_hoststring = gateway if re.match(r'\w+@[\d\.]+:\d+', gateway) else gateway + ':22'
-        node_hoststring = host_string if re.match(r'\w+@[\d\.]+:\d+', host_string) else host_string + ':22'
-        if password:
-            env.passwords.update({node_hoststring: password})
-            # If gateway_password is not set, guess same password
-            # (if key is used, it will be tried before password)
-            if not gateway_password:
-                env.passwords.update({gateway_hoststring: password})
-
-        if gateway_password:
-            env.passwords.update({gateway_hoststring: gateway_password})
-            if not password:
-                env.passwords.update({node_hoststring: gateway_password})
-
-        log.debug(cmd)
-        tries = 1
-        output = None
-        while tries > 0:
-            if timeout:
-                try:
-                    output = _run(cmd, timeout=timeout)
-                except CommandTimeout:
-                    pass
-            else:
-                output = _run(cmd)
-            if (output) and ('Fatal error' in output):
-                tries -= 1
-                time.sleep(5)
-            else:
-                break
-        # end while
-
-        if not raw:
-            real_output = remove_unwanted_output(output)
-        else:
-            real_output = output
-        return real_output
-
-
-def run_fab_cmd_on_node(host_string, password, cmd, as_sudo=False, timeout=120, as_daemon=False, raw=False,
-                        warn_only=True):
-    """
-    Run fab command on a node. Usecase : as part of script running on cfgm node, can run a cmd on VM from compute node
-
-    If raw is True, will return the fab _AttributeString object itself without removing any unwanted output
-    """
-    cmd = _escape_some_chars(cmd)
-    (username, host_ip) = host_string.split('@')
-    copy_fabfile_to_agent()
-    cmd_args = '-u %s -p "%s" -H %s -D --hide status,user,running' % (username,
-                password, host_ip)
-    if warn_only:
-        cmd_args+= ' -w '
-    cmd_str = 'fab %s ' % (cmd_args)
-    if as_daemon:
-        cmd_str += '--no-pty '
-        cmd = 'nohup ' + cmd + ' &'
-    if username == 'root':
-        as_sudo = False
-    elif username == 'cirros':
-        cmd_str += ' -s "/bin/sh -l -c" '
-    if as_sudo:
-        cmd_str += 'sudo_command:\"%s\"' % (cmd)
-    else:
-        cmd_str += 'command:\"%s\"' % (cmd)
-    # Sometimes, during bootup, there could be some intermittent conn. issue
-    log.debug(cmd_str)
-    tries = 1
-    output = None
-    while tries > 0:
-        if timeout:
-            try:
-                output = sudo(cmd_str, timeout=timeout)
-                log.debug(output)
-            except CommandTimeout:
-                return output
-        else:
-            output = run(cmd_str)
-        if ((output) and ('Fatal error' in output)):
-            tries -= 1
-            time.sleep(5)
-        else:
-            break
-    # end while
-
-    if not raw:
-        real_output = remove_unwanted_output(output)
-    else:
-        real_output = output
-    return real_output
-# end run_fab_cmd_on_node
-
-
-def fab_put_file_to_vm(host_string, password, src, dest):
-    copy_fabfile_to_agent()
-    (username, host_ip) = host_string.split('@')
-    cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running fput:\"%s\",\"%s\"' % (
-        username, password, host_ip, src, dest)
-    log.debug(cmd_str)
-    output = run(cmd_str)
-    real_output = remove_unwanted_output(output)
-# end fab_put_file_to_vm
-
-
-def sshable(host_string, password=None, gateway=None, gateway_password=None):
-    host_string_split = re.split(r"[@:]", host_string)
-    host_port = host_string_split[2] if len(host_string_split) > 2 else '22'
-    with hide('everything'), settings(host_string=gateway,
-                                      password=gateway_password,
-                                      warn_only=True):
-        if run('nc -w 1 -z %s %s' % (host_string_split[1], host_port)).succeeded:
-            try:
-                run_cmd_through_node(host_string, 'uname', password, gateway,
-                                     gateway_password, timeout=10)
-                return True
-            except Exception as e:
-                log.error("Error on ssh to %s" % host_string)
-                log.debug(str(e))
-                return False
-
-
-def fab_check_ssh(host_string, password):
-    copy_fabfile_to_agent()
-    (username, host_ip) = host_string.split('@')
-    cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running verify_socket_connection:22' % (
-        username, password, host_ip)
-    log.debug(cmd_str)
-    output = run(cmd_str)
-    log.debug(output)
-    if 'True' in output:
-        return True
-    return False
-# end fab_check_ssh
-
-
 def retry_for_value(tries=5, delay=3):
     '''Retries a function or method until it returns True.
         delay sets the initial delay in seconds.
@@ -749,18 +484,6 @@ def wrapper(*args, **kwargs):
     wrapper.has_run = False
     return wrapper
 
-def run_cmd_on_server(issue_cmd, server_ip, username,
-                      password, pty=True, as_sudo=False):
-    with hide('everything'):
-        with settings(
-            host_string='%s@%s' % (username, server_ip), password=password,
-                warn_only=True, abort_on_prompts=False):
-            if as_sudo:
-                output = sudo('%s' % (issue_cmd), pty=pty) 
-            else:
-                output = run('%s' % (issue_cmd), pty=pty)
-            return output
-# end run_cmd_on_server
 
 class Lock:
 
@@ -808,28 +531,6 @@ def read_config_option(config, section, option, default_option):
 # end read_config_option
 
 
-def copy_file_to_server(host, src, dest, filename, force=False):
-
-    fname = "%s/%s" % (dest, filename)
-    with settings(host_string='%s@%s' % (host['username'],
-                                         host['ip']), password=host['password'],
-                  warn_only=True, abort_on_prompts=False):
-        if not exists(fname) or force:
-            time.sleep(random.randint(1, 10))
-            put(src, dest)
-# end copy_file_to_server
-
-def get_host_domain_name(host):
-    output = None
-    with settings(hide('everything'), host_string='%s@%s' % (host['username'],
-        host['ip']), password=host['password'],
-        warn_only=True, abort_on_prompts=False):
-        output = run('hostname -d')
-
-    return output
-# end get_host_domain_name
-
-
 def get_random_vxlan_id():
     return random.randint(1, 16777215)
 
@@ -959,23 +660,3 @@ def wrapper(self, *func_args, **func_kwargs):
     return decorator
 
 
-def get_build_sku(openstack_node_ip, openstack_node_password='c0ntrail123', user='root'):
-    build_sku = get_os_env("SKU")
-    if build_sku is not None:
-        return str(build_sku).lower()
-    else:
-        host_str='%s@%s' % (user, openstack_node_ip)
-        cmd = 'nova-manage version'
-        tries = 10
-        while not build_sku and tries:
-            try:
-                with hide('everything'), settings(host_string=host_str,
-                                                  user=user,
-                                                  password=openstack_node_password):
-                    output = sudo(cmd)
-                    build_sku = sku_dict[re.findall("[0-9]+.[0-9]+",output)[0]]
-            except NetworkError, e:
-                time.sleep(1)
-                pass
-            tries -= 1
-        return build_sku

From 95a82df66581045113aec67ae688f69b1f7ef389 Mon Sep 17 00:00:00 2001
From: hkumarmk <hkumar@d4devops.org>
Date: Tue, 5 Apr 2016 12:27:37 +0530
Subject: [PATCH 4/5] Renamed run_cmd_through_node to run_cmd to use it in
 wider usecases

This method to be used to run command on any remote nodes - whether it
is a remote server or VM or between VMs. This method has capability to
handle:

1. run remote command on remote server from test node
2. Run remote command on node-a through node-b from the test node
  * in this case node-a is the target node, node-b is gateway, and the
nodes will be connect from testnode
  * This is to avoid situation to login to remote node (node-b in this
    case) and run script (fab script or pexpect or any such code) on
    that remote node (node-b) against running command on target node
    (node-a)
3. Run remote command on VM thorugh compute node - Same usecase as of #2
4. Run remote commands between VMs - say copy a file from vm1 to vm2
through compute node of vm1.
  * This will use ssh-agent forward to avoid copying ssh private keys to
    subsequent servers - Previously we used to copy ssh private keys to
    compute node and then copy the same file to vm1 in order to be able
    to connect from vm1 to vm2.
  * The commands will be running sitting on the test node then run an
  * "ssh/scp" command on vm1 through compute node of vm1 with
    agent_forward on
  * in this case flow is like this: test_node ->
    compute_of_vm1(gateway - passthrough) -> vm1 (run ssh/scp there) ->
    vm2 (final command is run)

NOTE that in all above situations ssh is initiated from the test node
and NO NEED TO SSH TO ANY OTHER NODE AND RUN SSH/FAB COMMANDS FROM
THERE.
---
 common/neutron/lbaas/base.py | 34 ++++++++++++++++------------------
 fixtures/vm_test.py          | 10 +++++-----
 2 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/common/neutron/lbaas/base.py b/common/neutron/lbaas/base.py
index ed024718a..829ce5735 100644
--- a/common/neutron/lbaas/base.py
+++ b/common/neutron/lbaas/base.py
@@ -1,7 +1,6 @@
 from common.neutron.base import BaseNeutronTest
 from tcutils.commands import ssh, execute_cmd, execute_cmd_out
-from fabric.context_managers import settings, hide
-from tcutils.util import run_cmd_through_node, retry
+from tcutils.util import run_cmd, retry
 from fabric.exceptions import CommandTimeout
 import re
 from time import sleep
@@ -102,21 +101,20 @@ def start_simpleHTTPserver(self, servers):
         for server in servers:
             cmd1 = 'hostname > index.html'
             cmd2 = 'python -m SimpleHTTPServer 80 &> /tmp/http.log'
-            run_cmd_through_node(host_string='%s@%s'%(server.vm_username,
-                                                      server.local_ip),
-                                 password=server.vm_password, cmd=cmd1,
-                                 gateway='@'.join([self.inputs.username,
-                                                   server.vm_node_ip]),
-                                 gateway_password=self.inputs.password,
-                                 cd='/tmp')
+            run_cmd(host_string='%s@%s'%(server.vm_username,
+                                         server.local_ip),
+                    password=server.vm_password, cmd=cmd1,
+                    gateway='@'.join([self.inputs.username,
+                                      server.vm_node_ip]),
+                    gateway_password=self.inputs.password, cwd='/tmp')
             try:
-                run_cmd_through_node(host_string = '%s@%s'%(server.vm_username,
-                                                            server.local_ip),
-                                     password=server.vm_password, cmd=cmd2,
-                                     gateway='@'.join([self.inputs.username,
-                                                   server.vm_node_ip]),
-                                     gateway_password=self.inputs.password,
-                                     with_sudo=True, timeout=1, cd='/tmp')
+                run_cmd(host_string = '%s@%s'%(server.vm_username,
+                                               server.local_ip),
+                        password=server.vm_password, cmd=cmd2,
+                        gateway='@'.join([self.inputs.username,
+                                          server.vm_node_ip]),
+                        gateway_password=self.inputs.password,
+                        with_sudo=True, timeout=1, cwd='/tmp')
             except CommandTimeout:
                 pass
         return
@@ -127,7 +125,7 @@ def run_wget(self, vm, vip):
         result = False
         cmd1 = 'wget http://%s' % vip
         cmd2 = 'cat index.html'
-        result = run_cmd_through_node(
+        result = run_cmd(
             host_string='%s@%s'%(vm.vm_username, vm.local_ip),
             password=vm.vm_password, cmd=cmd1,
             gateway='@'.join([self.inputs.username, vm.vm_node_ip]),
@@ -137,7 +135,7 @@ def run_wget(self, vm, vip):
         if result.count('200 OK'):
             result = True
             self.logger.info("connections to vip %s successful" % (vip))
-            response = run_cmd_through_node(
+            response = run_cmd(
                 host_string='%s@%s'%(vm.vm_username, vm.local_ip),
                 password=vm.vm_password, cmd=cmd2,
                 gateway='@'.join([self.inputs.username, vm.vm_node_ip]),
diff --git a/fixtures/vm_test.py b/fixtures/vm_test.py
index 79c3fd445..b5a2aa364 100644
--- a/fixtures/vm_test.py
+++ b/fixtures/vm_test.py
@@ -1026,7 +1026,7 @@ def ping_to_ip(self, ip, return_output=False, other_opt='', size='56', count='5'
             vm_host_string = '%s@%s' % (self.vm_username, self.local_ip)
             if af is None:
                 cmd = """python -c 'import socket;socket.getaddrinfo("%s", None, socket.AF_INET6)'""" % ip
-                output = run_cmd_through_node(
+                output = run_cmd(
                     vm_host_string, cmd, gateway_password=host['password'],
                     gateway='%s@%s' % (host['username'], self.vm_node_ip),
                     with_sudo=True
@@ -1039,7 +1039,7 @@ def ping_to_ip(self, ip, return_output=False, other_opt='', size='56', count='5'
                 util, str(size), str(count), other_opt, ip
             )
 
-            output = run_cmd_through_node(
+            output = run_cmd(
                 vm_host_string, cmd, gateway_password=host['password'],
                 gateway='%s@%s' % (host['username'], self.vm_node_ip),
                 with_sudo=True
@@ -1880,7 +1880,7 @@ def run_cmd_on_vm(self, cmds=[], as_sudo=False, timeout=30,
             for cmd in cmdList:
                 self.logger.debug('Running Cmd on %s: %s' % (
                     self.vm_node_ip, cmd))
-                output = run_cmd_through_node(
+                output = run_cmd(
                     vm_host_string, cmd, gateway_password=host['password'],
                     gateway='%s@%s' % (host['username'], self.vm_node_ip),
                     with_sudo=as_sudo, timeout=timeout, as_daemon=as_daemon,
@@ -2189,13 +2189,13 @@ def start_webserver(self, listen_port=8000, content=None):
         try:
             vm_host_string = '%s@%s'%(self.vm_username, self.local_ip)
             cmd = 'echo %s >& index.html'%(content or self.vm_name)
-            output = run_cmd_through_node(
+            output = run_cmd(
                 vm_host_string, cmd, gateway_password=host['password'],
                 gateway='%s@%s' % (host['username'], self.vm_node_ip),
                 with_sudo=True
             )
             cmd = 'python -m SimpleHTTPServer %d &> /dev/null' % listen_port
-            output = run_cmd_through_node(
+            output = run_cmd(
                 vm_host_string, cmd, gateway_password=host['password'],
                 gateway='%s@%s' % (host['username'], self.vm_node_ip),
                 with_sudo=True, as_daemon=True

From ec1833a6b3b130f4a5aaa4f8f9928914696977f8 Mon Sep 17 00:00:00 2001
From: hkumarmk <hkumar@d4devops.org>
Date: Tue, 5 Apr 2016 13:46:33 +0530
Subject: [PATCH 5/5] Replace run_fab_cmd_on_node with run_cmd

Removed run_fab_cmd_on_node from the codebase and replaced all
occurances of it with run_cmd.
---
 tcutils/fabutils.py                          | 55 ------------------
 tcutils/pkgs/Traffic/traffic/core/helpers.py | 60 +++++++++-----------
 tcutils/pkgs/install.py                      | 43 +++++++-------
 3 files changed, 45 insertions(+), 113 deletions(-)

diff --git a/tcutils/fabutils.py b/tcutils/fabutils.py
index e6fbb8db6..61e8802c3 100644
--- a/tcutils/fabutils.py
+++ b/tcutils/fabutils.py
@@ -182,61 +182,6 @@ def _escape_some_chars(text):
     return text
 # end escape_chars
 
-
-def run_fab_cmd_on_node(host_string, password, cmd, as_sudo=False, timeout=120, as_daemon=False, raw=False,
-                        warn_only=True):
-    """
-    Run fab command on a node. Usecase : as part of script running on cfgm node, can run a cmd on VM from compute node
-
-    If raw is True, will return the fab _AttributeString object itself without removing any unwanted output
-    """
-    cmd = _escape_some_chars(cmd)
-    (username, host_ip) = host_string.split('@')
-    copy_fabfile_to_agent()
-    cmd_args = '-u %s -p "%s" -H %s -D --hide status,user,running' % (username,
-                password, host_ip)
-    if warn_only:
-        cmd_args+= ' -w '
-    cmd_str = 'fab %s ' % (cmd_args)
-    if as_daemon:
-        cmd_str += '--no-pty '
-        cmd = 'nohup ' + cmd + ' &'
-    if username == 'root':
-        as_sudo = False
-    elif username == 'cirros':
-        cmd_str += ' -s "/bin/sh -l -c" '
-    if as_sudo:
-        cmd_str += 'sudo_command:\"%s\"' % (cmd)
-    else:
-        cmd_str += 'command:\"%s\"' % (cmd)
-    # Sometimes, during bootup, there could be some intermittent conn. issue
-    log.debug(cmd_str)
-    tries = 1
-    output = None
-    while tries > 0:
-        if timeout:
-            try:
-                output = sudo(cmd_str, timeout=timeout)
-                log.debug(output)
-            except CommandTimeout:
-                return output
-        else:
-            output = run(cmd_str)
-        if ((output) and ('Fatal error' in output)):
-            tries -= 1
-            time.sleep(5)
-        else:
-            break
-    # end while
-
-    if not raw:
-        real_output = remove_unwanted_output(output)
-    else:
-        real_output = output
-    return real_output
-# end run_fab_cmd_on_node
-
-
 def fab_put_file_to_vm(host_string, password, src, dest):
     copy_fabfile_to_agent()
     (username, host_ip) = host_string.split('@')
diff --git a/tcutils/pkgs/Traffic/traffic/core/helpers.py b/tcutils/pkgs/Traffic/traffic/core/helpers.py
index 0fc3ea7fe..7204e5525 100644
--- a/tcutils/pkgs/Traffic/traffic/core/helpers.py
+++ b/tcutils/pkgs/Traffic/traffic/core/helpers.py
@@ -6,7 +6,7 @@
 from fabric.api import run
 from fabric.operations import put
 from fabric.context_managers import settings, hide
-from tcutils.util import run_fab_cmd_on_node
+from tcutils.util import run_cmd
 
 try:
     # Running from the source repo "test".
@@ -58,43 +58,35 @@ def get_sshkey(self):
     def runcmd(self, cmd):
         """Run remote command."""
         output = None
-#        keyfile = self.get_sshkey()
-#        ssh_cmd = 'ssh -o StrictHostKeyChecking=no -i %s %s@%s \"%s\"' % (
-#                  keyfile, self.rhost.user, self.rhost.ip, cmd)
         self.log.debug('On host %s exec: %s'%(self.rhost.ip, cmd))
-        with hide('everything'):
-            with settings(
-                host_string='%s@%s' % (self.lhost.user, self.lhost.ip),
-                    password=self.lhost.password, warn_only=True, abort_on_prompts=False):
-                self.log.debug("Executing: %s", cmd)
-                retry = 6
-                while True:
-                    output = ''
-#                    output=run(ssh_cmd)
-                    output = run_fab_cmd_on_node(
-                        host_string='%s@%s' % (self.rhost.user, self.rhost.ip),
-                        password='ubuntu', as_sudo=True, cmd=cmd)
-                    if (not output) and retry:
-                        self.log.error(
-                            "Scapy issue while sending/receiving packets. Will retry after 5 secs.")
-                        sleep(5)
-                        retry -= 1
-                        continue
-                    if ("Connection timed out" in output or
-                            "Connection refused" in output) and retry:
-                        self.log.debug(
-                            "SSH timeout, sshd might not be up yet. will retry after 5 secs.")
-                        sleep(5)
-                        retry -= 1
-                        continue
-                    elif "Connection timed out" in output:
-                        raise SSHError(output)
-                    else:
-                        break
+        self.log.debug("Executing: %s", cmd)
+        retry = 6
+        while True:
+            output = ''
+            output = run_cmd(
+                '%s@%s' % (self.rhost.user, self.rhost.ip),
+                cmd, 'ubuntu', '%s@%s' % (self.lhost.user, self.lhost.ip),
+                self.lhost.password, with_sudo=True)
+            if (not output) and retry:
+                self.log.error(
+                    "Scapy issue while sending/receiving packets. Will retry after 5 secs.")
+                sleep(5)
+                retry -= 1
+                continue
+            if ("Connection timed out" in output or
+                    "Connection refused" in output) and retry:
+                self.log.debug(
+                    "SSH timeout, sshd might not be up yet. will retry after 5 secs.")
+                sleep(5)
+                retry -= 1
+                continue
+            elif "Connection timed out" in output:
+                raise SSHError(output)
+            else:
+                break
         self.log.debug(output)
         return output
 
-
 class Sender(Helper):
 
     def __init__(self, name, profile, lhost, rhost, log=LOG):
diff --git a/tcutils/pkgs/install.py b/tcutils/pkgs/install.py
index 8a74131cc..d189429de 100644
--- a/tcutils/pkgs/install.py
+++ b/tcutils/pkgs/install.py
@@ -8,7 +8,7 @@
 from fabric.api import run
 from fabric.operations import put
 from fabric.context_managers import settings, hide
-from tcutils.util import run_fab_cmd_on_node, fab_put_file_to_vm
+from tcutils.util import run_cmd, fab_put_file_to_vm
 
 LOG.basicConfig(format='%(levelname)s: %(message)s', level=LOG.DEBUG)
 
@@ -138,29 +138,24 @@ def copy_to_vm(self, pkg, host):
 
     def execute_in_vm(self, cmd, host):
         output = None
-        with hide('everything'):
-            with settings(host_string='%s@%s' % (self.pkgsrc.user, host),
-                          password=self.pkgsrc.password, warn_only=True,
-                          abort_on_prompts=False):
-                retry = 6
-                while True:
-                    output = ''
-                    output = run_fab_cmd_on_node(
-                        host_string='%s@%s' % (
-                            self.pkgdst.user, self.pkgdst.host),
-                        password=self.pkgdst.password, cmd=cmd,
-                        as_sudo=True)
-                    if ("Connection timed out" in output or
-                            "Connection refused" in output) and retry:
-                        self.log.debug(
-                            "SSH timeout, sshd might not be up yet. will retry after 5 secs.")
-                        sleep(5)
-                        retry -= 1
-                        continue
-                    elif "Connection timed out" in output:
-                        raise SSHError(output)
-                    else:
-                        break
+        retry = 6
+        while True:
+            output = ''
+            output = run_cmd(
+                '%s@%s' % (self.pkgdst.user, self.pkgdst.host),
+                cmd, self.pkgdst.password, '%s@%s' % (self.pkgsrc.user, host),
+                self.pkgsrc.password, with_sudo=True)
+            if ("Connection timed out" in output or
+                 "Connection refused" in output) and retry:
+                self.log.debug(
+                    "SSH timeout, sshd might not be up yet. will retry after 5 secs.")
+                sleep(5)
+                retry -= 1
+                continue
+            elif "Connection timed out" in output:
+                raise SSHError(output)
+            else:
+                break
         self.log.debug(output)
         return output