diff --git a/common/neutron/lbaas/base.py b/common/neutron/lbaas/base.py index ed024718a..829ce5735 100644 --- a/common/neutron/lbaas/base.py +++ b/common/neutron/lbaas/base.py @@ -1,7 +1,6 @@ from common.neutron.base import BaseNeutronTest from tcutils.commands import ssh, execute_cmd, execute_cmd_out -from fabric.context_managers import settings, hide -from tcutils.util import run_cmd_through_node, retry +from tcutils.util import run_cmd, retry from fabric.exceptions import CommandTimeout import re from time import sleep @@ -102,21 +101,20 @@ def start_simpleHTTPserver(self, servers): for server in servers: cmd1 = 'hostname > index.html' cmd2 = 'python -m SimpleHTTPServer 80 &> /tmp/http.log' - run_cmd_through_node(host_string='%s@%s'%(server.vm_username, - server.local_ip), - password=server.vm_password, cmd=cmd1, - gateway='@'.join([self.inputs.username, - server.vm_node_ip]), - gateway_password=self.inputs.password, - cd='/tmp') + run_cmd(host_string='%s@%s'%(server.vm_username, + server.local_ip), + password=server.vm_password, cmd=cmd1, + gateway='@'.join([self.inputs.username, + server.vm_node_ip]), + gateway_password=self.inputs.password, cwd='/tmp') try: - run_cmd_through_node(host_string = '%s@%s'%(server.vm_username, - server.local_ip), - password=server.vm_password, cmd=cmd2, - gateway='@'.join([self.inputs.username, - server.vm_node_ip]), - gateway_password=self.inputs.password, - with_sudo=True, timeout=1, cd='/tmp') + run_cmd(host_string = '%s@%s'%(server.vm_username, + server.local_ip), + password=server.vm_password, cmd=cmd2, + gateway='@'.join([self.inputs.username, + server.vm_node_ip]), + gateway_password=self.inputs.password, + with_sudo=True, timeout=1, cwd='/tmp') except CommandTimeout: pass return @@ -127,7 +125,7 @@ def run_wget(self, vm, vip): result = False cmd1 = 'wget http://%s' % vip cmd2 = 'cat index.html' - result = run_cmd_through_node( + result = run_cmd( host_string='%s@%s'%(vm.vm_username, vm.local_ip), password=vm.vm_password, cmd=cmd1, gateway='@'.join([self.inputs.username, vm.vm_node_ip]), @@ -137,7 +135,7 @@ def run_wget(self, vm, vip): if result.count('200 OK'): result = True self.logger.info("connections to vip %s successful" % (vip)) - response = run_cmd_through_node( + response = run_cmd( host_string='%s@%s'%(vm.vm_username, vm.local_ip), password=vm.vm_password, cmd=cmd2, gateway='@'.join([self.inputs.username, vm.vm_node_ip]), diff --git a/fixtures/vm_test.py b/fixtures/vm_test.py index be49ba1f1..b5a2aa364 100644 --- a/fixtures/vm_test.py +++ b/fixtures/vm_test.py @@ -1026,7 +1026,7 @@ def ping_to_ip(self, ip, return_output=False, other_opt='', size='56', count='5' vm_host_string = '%s@%s' % (self.vm_username, self.local_ip) if af is None: cmd = """python -c 'import socket;socket.getaddrinfo("%s", None, socket.AF_INET6)'""" % ip - output = run_cmd_through_node( + output = run_cmd( vm_host_string, cmd, gateway_password=host['password'], gateway='%s@%s' % (host['username'], self.vm_node_ip), with_sudo=True @@ -1039,7 +1039,7 @@ def ping_to_ip(self, ip, return_output=False, other_opt='', size='56', count='5' util, str(size), str(count), other_opt, ip ) - output = run_cmd_through_node( + output = run_cmd( vm_host_string, cmd, gateway_password=host['password'], gateway='%s@%s' % (host['username'], self.vm_node_ip), with_sudo=True @@ -1714,21 +1714,14 @@ def scp_file_to_vm(self, file, vm_ip, dest_vm_username='ubuntu'): timeout = math.floor(40 * float(delay_factor)) try: - self.orch.put_key_file_to_host(self.vm_node_ip) - with hide('everything'): - with settings(host_string='%s@%s' % ( - host['username'], self.vm_node_ip), - password=host['password'], - warn_only=True, abort_on_prompts=False): - self.get_rsa_to_vm() - i = 'timeout %d scp -o StrictHostKeyChecking=no -i id_rsa %s %s@[%s]:' % ( - timeout, file, dest_vm_username, vm_ip) - cmd_outputs = self.run_cmd_on_vm( - cmds=[i], timeout=timeout + 10) - self.logger.debug(cmd_outputs) + i = 'timeout %d scp -o StrictHostKeyChecking=no %s %s@[%s]:' % ( + timeout, file, dest_vm_username, vm_ip) + cmd_outputs = self.run_cmd_on_vm( + cmds=[i], timeout=timeout + 10) + self.logger.debug(cmd_outputs) except Exception, e: self.logger.exception( - 'Exception occured while trying to scp the file ') + 'Exception occured while trying to scp the file\n%s' % e) # end scp_file_to_vm def put_pub_key_to_vm(self): @@ -1887,7 +1880,7 @@ def run_cmd_on_vm(self, cmds=[], as_sudo=False, timeout=30, for cmd in cmdList: self.logger.debug('Running Cmd on %s: %s' % ( self.vm_node_ip, cmd)) - output = run_cmd_through_node( + output = run_cmd( vm_host_string, cmd, gateway_password=host['password'], gateway='%s@%s' % (host['username'], self.vm_node_ip), with_sudo=as_sudo, timeout=timeout, as_daemon=as_daemon, @@ -2196,13 +2189,13 @@ def start_webserver(self, listen_port=8000, content=None): try: vm_host_string = '%s@%s'%(self.vm_username, self.local_ip) cmd = 'echo %s >& index.html'%(content or self.vm_name) - output = run_cmd_through_node( + output = run_cmd( vm_host_string, cmd, gateway_password=host['password'], gateway='%s@%s' % (host['username'], self.vm_node_ip), with_sudo=True ) cmd = 'python -m SimpleHTTPServer %d &> /dev/null' % listen_port - output = run_cmd_through_node( + output = run_cmd( vm_host_string, cmd, gateway_password=host['password'], gateway='%s@%s' % (host['username'], self.vm_node_ip), with_sudo=True, as_daemon=True diff --git a/requirements.txt b/requirements.txt index cb529ca81..755d30eb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,5 @@ linecache2 eventlet nose tabulate +fabric +netaddr diff --git a/run_ci.sh b/run_ci.sh index 803659da3..c84fafded 100755 --- a/run_ci.sh +++ b/run_ci.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +source tools/common.sh function die { @@ -13,6 +14,7 @@ function usage { echo "Usage: $0 [OPTION]..." echo "Run Contrail test suite" echo "" + echo " -p, --prepare Run only preparation steps" echo " -t, --parallel Run testr in parallel" echo " -h, --help Print this usage message" echo " -m, --send-mail Send the report at the end" @@ -20,6 +22,7 @@ function usage { echo " --contrail-fab-path Contrail fab path, default to /opt/contrail/utils" echo " -- [TESTROPTIONS] After the first '--' you can pass arbitrary arguments to testr " } + testrargs="" debug=0 force=0 @@ -34,7 +37,7 @@ contrail_fab_path='/opt/contrail/utils' test_tag='suite1' export SCRIPT_TS=${SCRIPT_TS:-$(date +"%Y_%m_%d_%H_%M_%S")} -if ! options=$(getopt -o UthdC:lLmc: -l upload,parallel,help,debug,config:,logging,logging-config,send-mail,concurrency:,contrail-fab-path: -- "$@") +if ! options=$(getopt -o pUthdC:lLmc: -l prepare,upload,parallel,help,debug,config:,logging,logging-config,send-mail,concurrency:,contrail-fab-path: -- "$@") then # parse error usage @@ -45,6 +48,7 @@ eval set -- $options first_uu=yes while [ $# -gt 0 ]; do case "$1" in + -p|--prepare) prepare; exit;; -h|--help) usage; exit;; -U|--upload) upload=1;; -d|--debug) debug=1;; @@ -64,15 +68,7 @@ done testrargs+=" $test_tag" export TAGS="$test_tag" -if [ -n "$config_file" ]; then - config_file=`readlink -f "$config_file"` - export TEST_CONFIG_DIR=`dirname "$config_file"` - export TEST_CONFIG_FILE=`basename "$config_file"` -fi - -if [ ! -f "$config_file" ]; then - python tools/configure.py $(readlink -f .) -p $contrail_fab_path -fi +prepare if [ $logging -eq 1 ]; then if [ ! -f "$logging_config" ]; then diff --git a/tcutils/fabutils.py b/tcutils/fabutils.py new file mode 100644 index 000000000..61e8802c3 --- /dev/null +++ b/tcutils/fabutils.py @@ -0,0 +1,282 @@ +from fabric.operations import get, put, sudo +from fabric.api import run, env +from fabric.exceptions import CommandTimeout, NetworkError +from fabric.contrib.files import exists +from fabric.context_managers import settings, hide, cd + + +import re +import logging +import time +import random +import os + +log = logging.getLogger('log01') +sku_dict = {'2014.1': 'icehouse', '2014.2': 'juno', '2015.1': 'kilo', '12.0': 'liberty'} + + +def get_os_env(var): + if var in os.environ: + return os.environ.get(var) + else: + return None +# end get_os_env + + +def remove_unwanted_output(text): + """ Fab output usually has content like [ x.x.x.x ] out : + Args: + text: text to parse + """ + if not text: + return None + + return_list = text.split('\n') + + return_list1 = [] + for line in return_list: + line_split = line.split(' out: ') + if len(line_split) == 2: + return_list1.append(line_split[1]) + else: + if ' out:' not in line: + return_list1.append(line) + real_output = '\n'.join(return_list1) + return real_output + + +def run_cmd(host_string, cmd, password=None, gateway=None, + gateway_password=None, with_sudo=False, timeout=120, + as_daemon=False, raw=False, cwd=None, warn_only=True): + """ Run command on remote node through another node (gateway). + This is useful to run commands on VMs through compute node + Args: + host_string: host_string on which the command to run + password: Password + cmd: command + gateway: host_string of the node through which host_string will connect + gateway_password: Password of gateway hoststring + with_sudo: use Sudo + timeout: timeout + cwd: change directory to provided parameter + as_daemon: run in background + warn_only: run fab with warn_only + raw: If raw is True, will return the fab _AttributeString object itself without removing any unwanted output + """ + if as_daemon: + cmd = 'nohup ' + cmd + ' &' + + if cwd: + cmd = 'cd %s; %s' % (cd, cmd) + + (username, host_ip) = host_string.split('@') + + if username == 'root': + with_sudo = False + + shell = '/bin/bash -l -c' + + if username == 'cirros': + shell = '/bin/sh -l -c' + + _run = sudo if with_sudo else run + + # with hide('everything'), settings(host_string=host_string, + with settings( + host_string=host_string, + gateway=gateway, + warn_only=warn_only, + shell=shell, + disable_known_hosts=True, + abort_on_prompts=False): + env.forward_agent = True + gateway_hoststring = (gateway if re.match(r'\w+@[\d\.]+:\d+', gateway) + else gateway + ':22') + node_hoststring = (host_string + if re.match(r'\w+@[\d\.]+:\d+', host_string) + else host_string + ':22') + if password: + env.passwords.update({node_hoststring: password}) + # If gateway_password is not set, guess same password + # (if key is used, it will be tried before password) + if not gateway_password: + env.passwords.update({gateway_hoststring: password}) + + if gateway_password: + env.passwords.update({gateway_hoststring: gateway_password}) + if not password: + env.passwords.update({node_hoststring: gateway_password}) + + log.debug(cmd) + tries = 1 + output = None + while tries > 0: + if timeout: + try: + output = _run(cmd, timeout=timeout) + except CommandTimeout: + pass + else: + output = _run(cmd) + if output and 'Fatal error' in output: + tries -= 1 + time.sleep(5) + else: + break + # end while + + if not raw: + real_output = remove_unwanted_output(output) + else: + real_output = output + return real_output + + +def run_netconf_on_node(host_string, password, cmds, op_format='text'): + ''' + Run netconf from node to a VM.Usecase: vSRX or vMX or any netconf supporting device. + ''' + (username, host_ip) = host_string.split('@') + timeout = 10 + device = 'junos' + hostkey_verify = "False" + # Sometimes, during bootup, there could be some intermittent conn. issue + tries = 1 + output = None + copy_fabfile_to_agent() + while tries > 0: + if 'show' in cmds: + cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running get_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"' % ( + username, password, host_ip, cmds, timeout, device, hostkey_verify, op_format) + else: + cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running config_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\"' % ( + username, password, host_ip, cmds, timeout, device, hostkey_verify) + log.debug(cmd_str) + output = run(cmd_str) + log.debug(output) + if ((output) and ('Fatal error' in output)): + tries -= 1 + time.sleep(5) + else: + break + # end while + return output +# end run_netconf_on_node + + +def copy_fabfile_to_agent(): + src = 'tcutils/fabfile.py' + dst = '~/fabfile.py' + if 'fab_copied_to_hosts' not in env.keys(): + env.fab_copied_to_hosts = list() + if not env.host_string in env.fab_copied_to_hosts: + if not exists(dst): + put(src, dst) + env.fab_copied_to_hosts.append(env.host_string) + + +def _escape_some_chars(text): + chars = ['"', '='] + for char in chars: + text = text.replace(char, '\\\\' + char) + return text +# end escape_chars + +def fab_put_file_to_vm(host_string, password, src, dest): + copy_fabfile_to_agent() + (username, host_ip) = host_string.split('@') + cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running fput:\"%s\",\"%s\"' % ( + username, password, host_ip, src, dest) + log.debug(cmd_str) + output = run(cmd_str) + real_output = remove_unwanted_output(output) +# end fab_put_file_to_vm + + +def sshable(host_string, password=None, gateway=None, gateway_password=None): + host_string_split = re.split(r"[@:]", host_string) + host_port = host_string_split[2] if len(host_string_split) > 2 else '22' + with hide('everything'), settings(host_string=gateway, + password=gateway_password, + warn_only=True): + if run('nc -w 1 -z %s %s' % (host_string_split[1], host_port)).succeeded: + try: + run_cmd(host_string, 'uname', password, gateway, + gateway_password, timeout=10) + return True + except Exception as e: + log.error("Error on ssh to %s" % host_string) + log.debug(str(e)) + return False + + +def fab_check_ssh(host_string, password): + copy_fabfile_to_agent() + (username, host_ip) = host_string.split('@') + cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running verify_socket_connection:22' % ( + username, password, host_ip) + log.debug(cmd_str) + output = run(cmd_str) + log.debug(output) + if 'True' in output: + return True + return False +# end fab_check_ssh + + +def run_cmd_on_server(issue_cmd, server_ip, username, + password, pty=True, as_sudo=False): + with hide('everything'): + with settings( + host_string='%s@%s' % (username, server_ip), password=password, + warn_only=True, abort_on_prompts=False): + if as_sudo: + output = sudo('%s' % (issue_cmd), pty=pty) + else: + output = run('%s' % (issue_cmd), pty=pty) + return output +# end run_cmd_on_server + + +def copy_file_to_server(host, src, dest, filename, force=False): + + fname = "%s/%s" % (dest, filename) + with settings(host_string='%s@%s' % (host['username'], + host['ip']), password=host['password'], + warn_only=True, abort_on_prompts=False): + if not exists(fname) or force: + time.sleep(random.randint(1, 10)) + put(src, dest) +# end copy_file_to_server + +def get_host_domain_name(host): + output = None + with settings(hide('everything'), host_string='%s@%s' % (host['username'], + host['ip']), password=host['password'], + warn_only=True, abort_on_prompts=False): + output = run('hostname -d') + + return output +# end get_host_domain_name + + +def get_build_sku(openstack_node_ip, openstack_node_password='c0ntrail123', user='root'): + build_sku = get_os_env("SKU") + if build_sku is not None: + return str(build_sku).lower() + else: + host_str='%s@%s' % (user, openstack_node_ip) + cmd = 'nova-manage version' + tries = 10 + while not build_sku and tries: + try: + with hide('everything'), settings(host_string=host_str, + user=user, + password=openstack_node_password): + output = sudo(cmd) + build_sku = sku_dict[re.findall("[0-9]+.[0-9]+",output)[0]] + except NetworkError, e: + time.sleep(1) + pass + tries -= 1 + return build_sku diff --git a/tcutils/pkgs/Traffic/traffic/core/helpers.py b/tcutils/pkgs/Traffic/traffic/core/helpers.py index 0fc3ea7fe..7204e5525 100644 --- a/tcutils/pkgs/Traffic/traffic/core/helpers.py +++ b/tcutils/pkgs/Traffic/traffic/core/helpers.py @@ -6,7 +6,7 @@ from fabric.api import run from fabric.operations import put from fabric.context_managers import settings, hide -from tcutils.util import run_fab_cmd_on_node +from tcutils.util import run_cmd try: # Running from the source repo "test". @@ -58,43 +58,35 @@ def get_sshkey(self): def runcmd(self, cmd): """Run remote command.""" output = None -# keyfile = self.get_sshkey() -# ssh_cmd = 'ssh -o StrictHostKeyChecking=no -i %s %s@%s \"%s\"' % ( -# keyfile, self.rhost.user, self.rhost.ip, cmd) self.log.debug('On host %s exec: %s'%(self.rhost.ip, cmd)) - with hide('everything'): - with settings( - host_string='%s@%s' % (self.lhost.user, self.lhost.ip), - password=self.lhost.password, warn_only=True, abort_on_prompts=False): - self.log.debug("Executing: %s", cmd) - retry = 6 - while True: - output = '' -# output=run(ssh_cmd) - output = run_fab_cmd_on_node( - host_string='%s@%s' % (self.rhost.user, self.rhost.ip), - password='ubuntu', as_sudo=True, cmd=cmd) - if (not output) and retry: - self.log.error( - "Scapy issue while sending/receiving packets. Will retry after 5 secs.") - sleep(5) - retry -= 1 - continue - if ("Connection timed out" in output or - "Connection refused" in output) and retry: - self.log.debug( - "SSH timeout, sshd might not be up yet. will retry after 5 secs.") - sleep(5) - retry -= 1 - continue - elif "Connection timed out" in output: - raise SSHError(output) - else: - break + self.log.debug("Executing: %s", cmd) + retry = 6 + while True: + output = '' + output = run_cmd( + '%s@%s' % (self.rhost.user, self.rhost.ip), + cmd, 'ubuntu', '%s@%s' % (self.lhost.user, self.lhost.ip), + self.lhost.password, with_sudo=True) + if (not output) and retry: + self.log.error( + "Scapy issue while sending/receiving packets. Will retry after 5 secs.") + sleep(5) + retry -= 1 + continue + if ("Connection timed out" in output or + "Connection refused" in output) and retry: + self.log.debug( + "SSH timeout, sshd might not be up yet. will retry after 5 secs.") + sleep(5) + retry -= 1 + continue + elif "Connection timed out" in output: + raise SSHError(output) + else: + break self.log.debug(output) return output - class Sender(Helper): def __init__(self, name, profile, lhost, rhost, log=LOG): diff --git a/tcutils/pkgs/install.py b/tcutils/pkgs/install.py index 8a74131cc..d189429de 100644 --- a/tcutils/pkgs/install.py +++ b/tcutils/pkgs/install.py @@ -8,7 +8,7 @@ from fabric.api import run from fabric.operations import put from fabric.context_managers import settings, hide -from tcutils.util import run_fab_cmd_on_node, fab_put_file_to_vm +from tcutils.util import run_cmd, fab_put_file_to_vm LOG.basicConfig(format='%(levelname)s: %(message)s', level=LOG.DEBUG) @@ -138,29 +138,24 @@ def copy_to_vm(self, pkg, host): def execute_in_vm(self, cmd, host): output = None - with hide('everything'): - with settings(host_string='%s@%s' % (self.pkgsrc.user, host), - password=self.pkgsrc.password, warn_only=True, - abort_on_prompts=False): - retry = 6 - while True: - output = '' - output = run_fab_cmd_on_node( - host_string='%s@%s' % ( - self.pkgdst.user, self.pkgdst.host), - password=self.pkgdst.password, cmd=cmd, - as_sudo=True) - if ("Connection timed out" in output or - "Connection refused" in output) and retry: - self.log.debug( - "SSH timeout, sshd might not be up yet. will retry after 5 secs.") - sleep(5) - retry -= 1 - continue - elif "Connection timed out" in output: - raise SSHError(output) - else: - break + retry = 6 + while True: + output = '' + output = run_cmd( + '%s@%s' % (self.pkgdst.user, self.pkgdst.host), + cmd, self.pkgdst.password, '%s@%s' % (self.pkgsrc.user, host), + self.pkgsrc.password, with_sudo=True) + if ("Connection timed out" in output or + "Connection refused" in output) and retry: + self.log.debug( + "SSH timeout, sshd might not be up yet. will retry after 5 secs.") + sleep(5) + retry -= 1 + continue + elif "Connection timed out" in output: + raise SSHError(output) + else: + break self.log.debug(output) return output diff --git a/tcutils/util.py b/tcutils/util.py index a66de8b76..89f3bc22c 100644 --- a/tcutils/util.py +++ b/tcutils/util.py @@ -5,9 +5,6 @@ import time from collections import defaultdict, MutableMapping from netaddr import * -import pprint -from fabric.operations import get, put, sudo -from fabric.api import run, env import logging import threading from functools import wraps @@ -19,23 +16,17 @@ import fcntl import socket import struct -from fabric.exceptions import CommandTimeout, NetworkError -from fabric.contrib.files import exists -from fabric.context_managers import settings, hide, cd import ConfigParser from testtools.testcase import TestSkipped import functools import testtools -from fabfile import * + +from fabutils import * log = logging.getLogger('log01') -#log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) -sku_dict = {'2014.1': 'icehouse', '2014.2': 'juno', '2015.1': 'kilo', '12.0': 'liberty'} # Code borrowed from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry - - def retry(tries=5, delay=3): '''Retries a function or method until it returns True. delay sets the initial delay in seconds. @@ -132,261 +123,6 @@ def get_string_match_count(string_list, string_where_to_search): return d -def get_os_env(var): - if var in os.environ: - return os.environ.get(var) - else: - return None -# end get_os_env - - -def _escape_some_chars(text): - chars = ['"', '='] - for char in chars: - text = text.replace(char, '\\\\' + char) - return text -# end escape_chars - - -def remove_unwanted_output(text): - ''' Fab output usually has content like [ x.x.x.x ] out : - ''' - if not text: - return None - - return_list = text.split('\n') - - return_list1 = [] - for line in return_list: - line_split = line.split(' out: ') - if len(line_split) == 2: - return_list1.append(line_split[1]) - else: - if ' out:' not in line: - return_list1.append(line) - real_output = '\n'.join(return_list1) - return real_output - - -def run_netconf_on_node(host_string, password, cmds, op_format='text'): - ''' - Run netconf from node to a VM.Usecase: vSRX or vMX or any netconf supporting device. - ''' - (username, host_ip) = host_string.split('@') - timeout = 10 - device = 'junos' - hostkey_verify = "False" - # Sometimes, during bootup, there could be some intermittent conn. issue - tries = 1 - output = None - copy_fabfile_to_agent() - while tries > 0: - if 'show' in cmds: - cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running get_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"' % ( - username, password, host_ip, cmds, timeout, device, hostkey_verify, op_format) - else: - cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running config_via_netconf:\"%s\",\"%s\",\"%s\",\"%s\"' % ( - username, password, host_ip, cmds, timeout, device, hostkey_verify) - log.debug(cmd_str) - output = run(cmd_str) - log.debug(output) - if ((output) and ('Fatal error' in output)): - tries -= 1 - time.sleep(5) - else: - break - # end while - return output -# end run_netconf_on_node - - -def copy_fabfile_to_agent(): - src = 'tcutils/fabfile.py' - dst = '~/fabfile.py' - if 'fab_copied_to_hosts' not in env.keys(): - env.fab_copied_to_hosts = list() - if not env.host_string in env.fab_copied_to_hosts: - if not exists(dst): - put(src, dst) - env.fab_copied_to_hosts.append(env.host_string) - -def run_cmd_through_node(host_string, cmd, password=None, gateway=None, - gateway_password=None, with_sudo=False, timeout=120, - as_daemon=False, raw=False, cd=None, warn_only=True): - """ Run command on remote node through another node (gateway). - This is useful to run commands on VMs through compute node - Args: - host_string: host_string on which the command to run - password: Password - cmd: command - gateway: host_string of the node through which host_string will connect - gateway_password: Password of gateway hoststring - with_sudo: use Sudo - timeout: timeout - cd: change directory to provided parameter - as_daemon: run in background - raw: If raw is True, will return the fab _AttributeString object itself without removing any unwanted output - """ - if as_daemon: - cmd = 'nohup ' + cmd + ' &' - - if cd: - cmd = 'cd %s; %s' % (cd, cmd) - - (username, host_ip) = host_string.split('@') - - if username == 'root': - with_sudo = False - - shell = '/bin/bash -l -c' - - if username == 'cirros': - shell = '/bin/sh -l -c' - - _run = sudo if with_sudo else run - - #with hide('everything'), settings(host_string=host_string, - with settings(host_string=host_string, - gateway=gateway, - warn_only=warn_only, - shell=shell, - disable_known_hosts=True, - abort_on_prompts=False): - gateway_hoststring = gateway if re.match(r'\w+@[\d\.]+:\d+', gateway) else gateway + ':22' - node_hoststring = host_string if re.match(r'\w+@[\d\.]+:\d+', host_string) else host_string + ':22' - if password: - env.passwords.update({node_hoststring: password}) - # If gateway_password is not set, guess same password - # (if key is used, it will be tried before password) - if not gateway_password: - env.passwords.update({gateway_hoststring: password}) - - if gateway_password: - env.passwords.update({gateway_hoststring: gateway_password}) - if not password: - env.passwords.update({node_hoststring: gateway_password}) - - log.debug(cmd) - tries = 1 - output = None - while tries > 0: - if timeout: - try: - output = _run(cmd, timeout=timeout) - except CommandTimeout: - pass - else: - output = _run(cmd) - if (output) and ('Fatal error' in output): - tries -= 1 - time.sleep(5) - else: - break - # end while - - if not raw: - real_output = remove_unwanted_output(output) - else: - real_output = output - return real_output - - -def run_fab_cmd_on_node(host_string, password, cmd, as_sudo=False, timeout=120, as_daemon=False, raw=False, - warn_only=True): - """ - Run fab command on a node. Usecase : as part of script running on cfgm node, can run a cmd on VM from compute node - - If raw is True, will return the fab _AttributeString object itself without removing any unwanted output - """ - cmd = _escape_some_chars(cmd) - (username, host_ip) = host_string.split('@') - copy_fabfile_to_agent() - cmd_args = '-u %s -p "%s" -H %s -D --hide status,user,running' % (username, - password, host_ip) - if warn_only: - cmd_args+= ' -w ' - cmd_str = 'fab %s ' % (cmd_args) - if as_daemon: - cmd_str += '--no-pty ' - cmd = 'nohup ' + cmd + ' &' - if username == 'root': - as_sudo = False - elif username == 'cirros': - cmd_str += ' -s "/bin/sh -l -c" ' - if as_sudo: - cmd_str += 'sudo_command:\"%s\"' % (cmd) - else: - cmd_str += 'command:\"%s\"' % (cmd) - # Sometimes, during bootup, there could be some intermittent conn. issue - log.debug(cmd_str) - tries = 1 - output = None - while tries > 0: - if timeout: - try: - output = sudo(cmd_str, timeout=timeout) - log.debug(output) - except CommandTimeout: - return output - else: - output = run(cmd_str) - if ((output) and ('Fatal error' in output)): - tries -= 1 - time.sleep(5) - else: - break - # end while - - if not raw: - real_output = remove_unwanted_output(output) - else: - real_output = output - return real_output -# end run_fab_cmd_on_node - - -def fab_put_file_to_vm(host_string, password, src, dest): - copy_fabfile_to_agent() - (username, host_ip) = host_string.split('@') - cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running fput:\"%s\",\"%s\"' % ( - username, password, host_ip, src, dest) - log.debug(cmd_str) - output = run(cmd_str) - real_output = remove_unwanted_output(output) -# end fab_put_file_to_vm - - -def sshable(host_string, password=None, gateway=None, gateway_password=None): - host_string_split = re.split(r"[@:]", host_string) - host_port = host_string_split[2] if len(host_string_split) > 2 else '22' - with hide('everything'), settings(host_string=gateway, - password=gateway_password, - warn_only=True): - if run('nc -w 1 -z %s %s' % (host_string_split[1], host_port)).succeeded: - try: - run_cmd_through_node(host_string, 'uname', password, gateway, - gateway_password, timeout=10) - return True - except Exception as e: - log.error("Error on ssh to %s" % host_string) - log.debug(str(e)) - return False - - -def fab_check_ssh(host_string, password): - copy_fabfile_to_agent() - (username, host_ip) = host_string.split('@') - cmd_str = 'fab -u %s -p "%s" -H %s -D -w --hide status,user,running verify_socket_connection:22' % ( - username, password, host_ip) - log.debug(cmd_str) - output = run(cmd_str) - log.debug(output) - if 'True' in output: - return True - return False -# end fab_check_ssh - - def retry_for_value(tries=5, delay=3): '''Retries a function or method until it returns True. delay sets the initial delay in seconds. @@ -748,18 +484,6 @@ def wrapper(*args, **kwargs): wrapper.has_run = False return wrapper -def run_cmd_on_server(issue_cmd, server_ip, username, - password, pty=True, as_sudo=False): - with hide('everything'): - with settings( - host_string='%s@%s' % (username, server_ip), password=password, - warn_only=True, abort_on_prompts=False): - if as_sudo: - output = sudo('%s' % (issue_cmd), pty=pty) - else: - output = run('%s' % (issue_cmd), pty=pty) - return output -# end run_cmd_on_server class Lock: @@ -807,28 +531,6 @@ def read_config_option(config, section, option, default_option): # end read_config_option -def copy_file_to_server(host, src, dest, filename, force=False): - - fname = "%s/%s" % (dest, filename) - with settings(host_string='%s@%s' % (host['username'], - host['ip']), password=host['password'], - warn_only=True, abort_on_prompts=False): - if not exists(fname) or force: - time.sleep(random.randint(1, 10)) - put(src, dest) -# end copy_file_to_server - -def get_host_domain_name(host): - output = None - with settings(hide('everything'), host_string='%s@%s' % (host['username'], - host['ip']), password=host['password'], - warn_only=True, abort_on_prompts=False): - output = run('hostname -d') - - return output -# end get_host_domain_name - - def get_random_vxlan_id(): return random.randint(1, 16777215) @@ -958,23 +660,3 @@ def wrapper(self, *func_args, **func_kwargs): return decorator -def get_build_sku(openstack_node_ip, openstack_node_password='c0ntrail123', user='root'): - build_sku = get_os_env("SKU") - if build_sku is not None: - return str(build_sku).lower() - else: - host_str='%s@%s' % (user, openstack_node_ip) - cmd = 'nova-manage version' - tries = 10 - while not build_sku and tries: - try: - with hide('everything'), settings(host_string=host_str, - user=user, - password=openstack_node_password): - output = sudo(cmd) - build_sku = sku_dict[re.findall("[0-9]+.[0-9]+",output)[0]] - except NetworkError, e: - time.sleep(1) - pass - tries -= 1 - return build_sku diff --git a/tools/common.sh b/tools/common.sh new file mode 100644 index 000000000..6e5c18e46 --- /dev/null +++ b/tools/common.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +function prepare { + if [ -n "$config_file" ]; then + config_file=`readlink -f "$config_file"` + export TEST_CONFIG_DIR=`dirname "$config_file"` + export TEST_CONFIG_FILE=`basename "$config_file"` + fi + + if [ ! -f "$config_file" ]; then + python tools/configure.py $(readlink -f .) -p $contrail_fab_path + fi + + #Start ssh-agent if not there + if [ -z "$SSH_AUTH_SOCK" ] ; then + eval `ssh-agent -s` + fi + if [ -z "$SSH_AUTH_SOCK" ] ; then + echo "Error: SSH agent failed to start" + exit 1 + fi + ssh-add +}