From 038a535e49970a7b16440ce2a356d8a08a0115ce Mon Sep 17 00:00:00 2001 From: Roman Imankulov Date: Fri, 4 Jan 2013 14:43:00 +0300 Subject: [PATCH 1/3] Added support for python 3.3 --- openssh_wrapper.py | 290 +++++++++++++++++++++++++++++++++------------ 1 file changed, 213 insertions(+), 77 deletions(-) diff --git a/openssh_wrapper.py b/openssh_wrapper.py index 34d391e..c6a0dcc 100644 --- a/openssh_wrapper.py +++ b/openssh_wrapper.py @@ -2,15 +2,67 @@ """ This is a wrapper around the openssh binaries ssh and scp. """ +import io import re import os +import sys import subprocess import signal import pipes import tempfile import shutil -__all__ = 'SSHConnection SSHResult SSHError'.split() +__all__ = 'SSHConnection SSHResult SSHError b u b_list u_list'.split() + +if sys.version[0] == 2: + text = unicode + bytes = str +else: # PY3K + text = str + + +def b(string): + """ + convert string (unicode, str or bytes) to binary representation + """ + if isinstance(string, bytes): + return string + return string.encode('utf-8') + + +def u(string): + """ + convert string (unicode, str or bytes) to textual representation + """ + if isinstance(string, text): + return string + return string.decode('utf-8') + + +def b_list(items): + """ + convert all items of the list to binary representation + """ + return [b(item) for item in items] + + +def u_list(items): + """ + convert all items of the list to textual representation + """ + return [u(item) for item in items] + + +def b_quote(cmd_chunks): + """ + Given a list of commands (unicode or text strings), return the safe list, + suitable to be passed to subprocess + """ + quoted = [] + for chunk in cmd_chunks: + # pipes.quote works with text representation only + quoted.append(b(pipes.quote(u(chunk)))) + return b(' ').join(quoted) class SSHConnection(object): @@ -22,32 +74,33 @@ class SSHConnection(object): def __init__(self, server, login=None, port=None, configfile=None, identity_file=None, ssh_agent_socket=None, timeout=60): """ - Create new object to establish SSH connection to remote - servers. - - Arguments: - - - `server`: server name or IP address to send commands to (required). - - `login`: user login (by default, current login) - - `confgfile`: local configuration file (by default ~/.ssh/config is used) - - `identity_file`: identity file (by default ~/.ssh/id_rsa) - - `ssh_agent_socket`: address of the socket to connect to ssh agent, - if you want to use one. ``SSH_AUTH_SOCK`` environment variable is - used if None is supplied. - - `timeout`: connect timeout. If you plan to execute long lasting - commands, adjust this variable accordingly. Default value of 60 - seconds is usually a good choice. - - By the way, `man ssh_config` is highly recommended amendment to this - command. + Create new object to establish SSH connection to remote servers + + :param server: server name or IP address to send commands to (required) + :param login: user login (by default current login is used) + :param port: SSH port number. Optional. + :param configfile: local configuration file (by default ~/.ssh/config is used) + :param identity_file: address of the socket to connect to ssh agent, + if you want to use it. ``SSH_AUTH_SOCK`` environment variable is + used if None is supplied. + :param ssh_agent_socket: address of the socket to connect to ssh agent + :param timeout: connect timeout. If you plan to execute long + lasting commands, adjust this variable accordingly. Default value of + 60 seconds is usually a good choice. + + :raise SSHError: if server name or login contain illegal symbols, or + some of the files, provided to the constructor, don't exist. + + .. note:: `man ssh_config` is highly recommended amendment to this + command. """ - self.server = str(server) + self.server = b(server) self.port = port self.timeout = timeout self.check_server(server) if login: self.check_login(login) - self.login = str(login) + self.login = b(login) else: self.login = None if configfile: @@ -67,10 +120,10 @@ def __init__(self, server, login=None, port=None, configfile=None, def check_server(self, server): """ Check the server string for illegal characters. - Returns: - Nothing - Raises: - SSHError + + :param server: a string with server name + :return: None + :raise: SSHError """ if not re.compile(r'^[a-zA-Z0-9.\-_]+$').match(server): raise SSHError('Server name contains illegal symbols') @@ -78,25 +131,34 @@ def check_server(self, server): def check_login(self, login): """ Check the login string for illegal characters. - Returns: - Nothing - Raises: - SSHError + + :param login: a string with user login + :return: None + :raise: SSHError """ if not re.compile(r'^[a-zA-Z0-9.\-_]+$').match(login): raise SSHError('User login contains illegal symbols') def run(self, command, interpreter='/bin/bash', forward_ssh_agent=False): """ - Execute ``command`` using ``interpreter`` + Execute the command using the interpreter provided Consider this roughly as:: echo "command" | ssh root@server "/bin/interpreter" - Raise SSHError if server is unreachable - Hint: Try interpreter='/usr/bin/python' + + :param command: string/unicode object or byte sequence with the command + or set of commands to execute + :param interpreter: name of the interpreter (by default "/bin/bash" is used) + :param forward_ssh_agent: turn this flag to `True`, if you want to use + and forward SSH agent + + :return: SSH result instance + :rtype: SSHResult + + :raise: SSHError, if server is unreachable, or timeout has reached. """ ssh_command = self.ssh_command(interpreter, forward_ssh_agent) pipe = subprocess.Popen(ssh_command, @@ -107,10 +169,11 @@ def run(self, command, interpreter='/bin/bash', forward_ssh_agent=False): except ValueError: # signal only works in main thread pass signal.alarm(self.timeout) - out = err = "" + out = b('') + err = b('') try: - out, err = pipe.communicate(command) - except IOError, exc: + out, err = pipe.communicate(b(command)) + except IOError as exc: # pipe.terminate() # only in python 2.6 allowed os.kill(pipe.pid, signal.SIGTERM) signal.alarm(0) # disable alarm @@ -125,16 +188,27 @@ def run(self, command, interpreter='/bin/bash', forward_ssh_agent=False): def scp(self, files, target, mode=None, owner=None): """ Copy files identified by their names to remote location - :param files: files or file-like objects to copy + .. note:: if you want your file objects to have meaningful names, + make sure they have `name` attribute. + + :param files: list of file names or file-like objects to copy. Before + copying the files will be interpreted the following way: if the element + is a string, it is considered as a file name, if it's a file-like object, + then it will be copied to a temporary directory, and then copied from + there to a remote location using "scp" command. + :param target: target file or directory to copy data to. Target file - makes sense only if the number of files to copy equals - to one. + makes sense only if the number of files to copy equals to one. + :param mode: optional parameter to define mode for every uploaded file - (must be a string in the form understandable by chmod) + (must be a string in the form understandable by chmod, like "0644") + :param owner: optional parameter to define user and group for every - uploaded file (must be a string in the form - understandable by chown). Makes sence only if you open - your connection as root. + uploaded file (must be a string in the form understandable by chown). + Makes sense only if you open your connection as root. + + :return: None + :raise: SSHError """ filenames, tmpdir = self.convert_files_to_filenames(files) @@ -148,10 +222,10 @@ def cleanup_tmp_dir(): stderr=subprocess.PIPE, env=self.get_env()) signal.signal(signal.SIGALRM, _timeout_handler) signal.alarm(self.timeout) - err = "" + err = b('') try: _, err = pipe.communicate() - except IOError, exc: + except IOError as exc: # pipe.terminate() # only in python 2.6 allowed os.kill(pipe.pid, signal.SIGTERM) signal.alarm(0) # disable alarm @@ -167,14 +241,14 @@ def cleanup_tmp_dir(): targets = self.get_scp_targets(filenames, target) if mode: cmd_chunks = ['chmod', mode] + targets - cmd = ' '.join([pipes.quote(chunk) for chunk in cmd_chunks]) + cmd = b_quote(cmd_chunks) result = self.run(cmd) if result.returncode: cleanup_tmp_dir() raise SSHError(result.stderr.strip()) if owner: cmd_chunks = ['chown', owner] + targets - cmd = ' '.join([pipes.quote(chunk) for chunk in cmd_chunks]) + cmd = b_quote(cmd_chunks) result = self.run(cmd) if result.returncode: cleanup_tmp_dir() @@ -183,12 +257,23 @@ def cleanup_tmp_dir(): def convert_files_to_filenames(self, files): """ - Check for every file in list and save it locally to send to remote side, if needed + Helper function which is invoked by scp. + + You don't usually need to execute this function manually. + Check for every file in list and save it locally to send to + remote side, if needed. + + :param files: list of strings or file-alike objects to be converted + to filenames + + :return: tuple (filenames, tmpdir), where filenames is a list of absolute + filenames, and tmpdir is a name of temp directory which must be removed + afterwards. If tmpdir is None, nothing should be removed. """ filenames = [] tmpdir = None for file_obj in files: - if isinstance(file_obj, basestring): + if isinstance(file_obj, (text, bytes)): filenames.append(file_obj) else: if not tmpdir: @@ -196,12 +281,12 @@ def convert_files_to_filenames(self, files): if hasattr(file_obj, 'name'): basename = os.path.basename(file_obj.name) tmpname = os.path.join(tmpdir, basename) - fd = open(tmpname, 'w') - fd.write(file_obj.read()) + fd = io.open(tmpname, 'wb') + fd.write(b(file_obj.read())) fd.close() else: tmpfd, tmpname = tempfile.mkstemp(dir=tmpdir) - os.write(tmpfd, file_obj.read()) + os.write(tmpfd, b(file_obj.read())) os.close(tmpfd) filenames.append(tmpname) return filenames, tmpdir @@ -210,12 +295,20 @@ def get_scp_targets(self, filenames, target): """ Given a list of filenames and a target name return the full list of targets + :param filenames: list of filenames to copy (basenames) + :param target: target file or directory + Internal command which is used to perform chmod and chown. - For example, get_scp_targets(['foo.txt', 'bar.txt'], '/etc') returns ['/etc/foo.txt', '/etc/bar.txt'], - whereas get_scp_targets(['foo.txt', ], '/etc/passwd') returns ['/etc/passwd', ] + Example:: + + >>> ssh_connection.get_scp_targets(['foo.txt', 'bar.txt'], '/etc') + ['/etc/foo.txt', '/etc/bar.txt'] + + >>> get_scp_targets(['foo.txt', ], '/etc/passwd') + ['/etc/passwd'] """ - result = self.run('test -d %s' % pipes.quote(target)) + result = self.run(b('test -d %s' % pipes.quote(u(target)))) is_directory = result.returncode == 0 if is_directory: ret = [] @@ -226,9 +319,12 @@ def get_scp_targets(self, filenames, target): return [target, ] def ssh_command(self, interpreter, forward_ssh_agent): - """ Build the command string to connect to the server - and start the given interpreter. """ - interpreter = str(interpreter) + """ + Build the command string to connect to the server and start the interpreter. + + Internal function + """ + interpreter = b(interpreter) cmd = ['/usr/bin/ssh', ] if self.login: cmd += ['-l', self.login] @@ -242,16 +338,18 @@ def ssh_command(self, interpreter, forward_ssh_agent): cmd += ['-p', str(self.port)] cmd.append(self.server) cmd.append(interpreter) - return cmd + return b_list(cmd) def scp_command(self, files, target): - """ Build the command string to transfer the files - identifiend by the given filenames. - Include target(s) if specified. """ + """ + Build the command string to transfer the files identified by filenames. + + Include target(s) if specified. Internal function + """ cmd = ['/usr/bin/scp', '-q', '-r'] - files = map(str, files) + files = b_list(files) if self.login: - remotename = '%s@%s' % (self.login, self.server) + remotename = '%s@%s' % (u(self.login), u(self.server)) else: remotename = self.server if self.configfile: @@ -261,18 +359,20 @@ def scp_command(self, files, target): if self.port: cmd += ['-P', self.port] - if isinstance(files, basestring): + if isinstance(files, (text, bytes)): raise ValueError('"files" argument have to be iterable (list or tuple)') if len(files) < 1: raise ValueError('You should name at least one file to copy') cmd += files cmd.append('%s:%s' % (remotename, target)) - return cmd + return b_list(cmd) def get_env(self): - """ Retrieve environment variables and replace SSH_AUTH_SOCK - if ssh_agent_socket was specified on object creation. """ + """ + Retrieve environment variables and replace SSH_AUTH_SOCK + if ssh_agent_socket was specified on object creation. + """ env = os.environ.copy() if self.ssh_agent_socket: env['SSH_AUTH_SOCK'] = self.ssh_agent_socket @@ -285,8 +385,18 @@ def _timeout_handler(signum, frame): class SSHResult(object): - """ Command execution status. Has ``command``, ``stdout``, ``stderr`` and - ``returncode`` fields """ + """ + Command execution status. + """ + #: command which has been executed remotely + command = None + #: command execution stdout (no charset applied, binary object) + stdout = None + #: command execution stderr (no charset applied, binary object) + stderr = None + #: command return code (integer, 0 means "success" usually) + returncode = None + def __init__(self, command, stdout, stderr, returncode): """ Create a new object to hold output and a return code @@ -297,17 +407,43 @@ def __init__(self, command, stdout, stderr, returncode): self.returncode = returncode def __str__(self): - """ Get acscii representation from unicode representation. """ - return unicode(self).decode('utf-8', 'ignore') + """ + Get string representation of the result. + + Effectively, returns stdout + """ + if sys.version_info[0] == 2: + # get ASCII representation. + return self.stdout + else: + # get string representation + return self.stdout.decode('utf-8', 'ignore') + + def __repr__(self): + """ + Get the verbose interpretation of the object + + For python2.x it's the raw string objects, whereas python3.x + contains the unicode representation (str) + """ + if sys.version_info[0] == 2: + # get ASCII representation. + return self.repr_binary() + else: + # get string representation + return self.repr_text() - def __unicode__(self): + def repr_binary(self): """ Build simple unicode representation from all member values. """ ret = [] - ret.append(u'command: %s' % unicode(self.command)) - ret.append(u'stdout: %s' % unicode(self.stdout)) - ret.append(u'stderr: %s' % unicode(self.stderr)) - ret.append(u'returncode: %s' % unicode(self.returncode)) - return u'\n'.join(ret) + ret += [b('command: '), self.command, b('\n')] + ret += [b('stdout: '), self.stdout, b('\n')] + ret += [b('stderr: '), self.stderr, b('\n')] + ret += [b('returncode: '), b(text(self.returncode))] + return b('').join(ret) + + def repr_text(self): + return self.repr_binary().encode('utf-8', 'ignore') class SSHError(Exception): From 88620d46ef7998a21ac2250bc685744c17b8054c Mon Sep 17 00:00:00 2001 From: Roman Imankulov Date: Fri, 4 Jan 2013 14:43:33 +0300 Subject: [PATCH 2/3] Changes in module testing - Tests fixed to support python 3.x - Added support for tox.ini --- .gitignore | 2 ++ MANIFEST.in | 2 ++ setup.py | 4 +++- ssh_config.test | 1 + tests.py | 56 ++++++++++++++++++++++++++++--------------------- tox.ini | 7 +++++++ 6 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 MANIFEST.in create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 3ffbb60..96122a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc .*.swp /*egg-info +/MANIFEST +/.tox diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7e106dc --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include tests.py +include README.rst diff --git a/setup.py b/setup.py index 1c1d446..a2c7127 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,9 @@ def read(fname): 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', ), ) diff --git a/ssh_config.test b/ssh_config.test index 3f2e593..13bb3e8 100644 --- a/ssh_config.test +++ b/ssh_config.test @@ -1,2 +1,3 @@ Host * PasswordAuthentication no + BatchMode yes diff --git a/tests.py b/tests.py index 1e774d6..b27d02d 100644 --- a/tests.py +++ b/tests.py @@ -1,27 +1,35 @@ # -*- coding: utf-8 -*- +import io import os +import pytest from openssh_wrapper import * -from nose.tools import * + test_file = os.path.join(os.path.dirname(__file__), 'tests.py') + +def eq_(arg1, arg2): + assert arg1 == arg2 + + class TestSSHCommandNames(object): - def setUp(self): + def setup_method(self, meth): self.c = SSHConnection('localhost', login='root', configfile='ssh_config.test') def test_ssh_command(self): eq_(self.c.ssh_command('/bin/bash', False), - ['/usr/bin/ssh', '-l', 'root', '-F', 'ssh_config.test', 'localhost', '/bin/bash']) + b_list(['/usr/bin/ssh', '-l', 'root', '-F', 'ssh_config.test', 'localhost', '/bin/bash'])) def test_scp_command(self): eq_(self.c.scp_command(('/tmp/1.txt', ), target='/tmp/2.txt'), - ['/usr/bin/scp', '-q', '-r', '-F', 'ssh_config.test', '/tmp/1.txt', 'root@localhost:/tmp/2.txt']) + b_list(['/usr/bin/scp', '-q', '-r', '-F', 'ssh_config.test', '/tmp/1.txt', 'root@localhost:/tmp/2.txt'])) def test_scp_multiple_files(self): eq_(self.c.scp_command(('/tmp/1.txt', '2.txt'), target='/home/username/'), - ['/usr/bin/scp', '-q', '-r', '-F', 'ssh_config.test', '/tmp/1.txt', '2.txt', 'root@localhost:/home/username/']) + b_list(['/usr/bin/scp', '-q', '-r', '-F', 'ssh_config.test', '/tmp/1.txt', '2.txt', + 'root@localhost:/home/username/'])) def test_scp_targets(self): targets = self.c.get_scp_targets(['foo.txt', 'bar.txt'], '/etc') @@ -31,46 +39,46 @@ def test_scp_targets(self): def test_simple_command(self): result = self.c.run('whoami') - eq_(result.stdout, 'root') - eq_(result.stderr, '') + eq_(result.stdout, b('root')) + eq_(result.stderr, b('')) eq_(result.returncode, 0) def test_python_command(self): result = self.c.run('print "Hello world"', interpreter='/usr/bin/python') - eq_(result.stdout, 'Hello world') - eq_(result.stderr, '') + eq_(result.stdout, b('Hello world')) + eq_(result.stderr, b('')) eq_(result.returncode, 0) -@raises(SSHError) # ssh connect timeout def test_timeout(): c = SSHConnection('example.com', login='root', timeout=1) - c.run('whoami') + with pytest.raises(SSHError): # ssh connect timeout + c.run('whoami') -@raises(SSHError) # Permission denied (publickey) def test_permission_denied(): c = SSHConnection('localhost', login='www-data', configfile='ssh_config.test') - c.run('whoami') + with pytest.raises(SSHError): # Permission denied (publickey) + c.run('whoami') class TestSCP(object): - def setUp(self): + def setup_method(self, meth): self.c = SSHConnection('localhost', login='root') self.c.run('rm -f /tmp/*.py /tmp/test*.txt') def test_scp(self): self.c.scp((test_file, ), target='/tmp') - ok_(os.path.isfile('/tmp/tests.py')) + assert os.path.isfile('/tmp/tests.py') - @raises(SSHError) def test_scp_to_nonexistent_dir(self): - self.c.scp((test_file, ), target='/abc/def/') + with pytest.raises(SSHError): + self.c.scp((test_file, ), target='/abc/def/') def test_mode(self): self.c.scp((test_file, ), target='/tmp', mode='0666') - mode = os.stat('/tmp/tests.py').st_mode & 0777 - eq_(mode, 0666) + mode = os.stat('/tmp/tests.py').st_mode & 0o777 + eq_(mode, 0o666) def test_owner(self): import pwd, grp @@ -82,13 +90,13 @@ def test_owner(self): eq_(stat.st_gid, gid) def test_file_descriptors(self): - from StringIO import StringIO # name is set explicitly as target - fd1 = StringIO('test') + fd1 = io.BytesIO(b('test')) self.c.scp((fd1, ), target='/tmp/test1.txt', mode='0644') - eq_(open('/tmp/test1.txt').read(), 'test') + assert io.open('/tmp/test1.txt', 'rt').read() == 'test' + # name is set explicitly in the name option - fd2 = StringIO('test') + fd2 = io.BytesIO(b('test')) fd2.name = 'test2.txt' self.c.scp((fd2, ), target='/tmp', mode='0644') - eq_(open('/tmp/test2.txt').read(), 'test') + assert io.open('/tmp/test2.txt', 'rt').read() == 'test' diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..382ba77 --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py26, py27, py33 + +[testenv] +deps = + pytest +commands = py.test tests.py {posargs} From 77e02c71e824856291ca04da678168480c863c9f Mon Sep 17 00:00:00 2001 From: Roman Imankulov Date: Fri, 4 Jan 2013 14:45:42 +0300 Subject: [PATCH 3/3] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a2c7127..57785bb 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def read(fname): setup( name='openssh-wrapper', - version='0.3.2', + version='0.4', description='OpenSSH python wrapper', author='NetAngels team', author_email='info@netangels.ru',