# coding: utf-8
# Copyright (C) 1994-2018 Altair Engineering, Inc.
# For more information, contact Altair at www.altair.com.
#
# This file is part of the PBS Professional ("PBS Pro") software.
#
# Open Source License Information:
#
# PBS Pro is free software. You can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
#
# Commercial License Information:
#
# For a copy of the commercial license terms and conditions,
# go to: (http://www.pbspro.com/UserArea/agreement.html)
# or contact the Altair Legal Department.
#
# Altair’s dual-license business model allows companies, individuals, and
# organizations to create proprietary derivative works of PBS Pro and
# distribute them - whether embedded or bundled with other software -
# under a commercial license agreement.
#
# Use of Altair’s trademarks, including but not limited to "PBS™",
# "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
# trademark licensing policies.
from subprocess import PIPE, Popen
import os
import sys
import re
import stat
import socket
import logging
import traceback
import copy
import tempfile
import pwd
import grp
import platform
DFLT_RSYNC_CMD = ['rsync', '-e', 'ssh', '--progress', '--partial', '-ravz']
DFLT_COPY_CMD = ['scp', '-p']
DFLT_RSH_CMD = ['ssh']
DFLT_SUDO_CMD = ['sudo', '-H']
logging.DEBUG2 = logging.DEBUG - 1
logging.INFOCLI = logging.INFO - 1
logging.INFOCLI2 = logging.INFOCLI - 1
class PbsConfigError(Exception):
"""
Initialize PBS configuration error
"""
def __init__(self, message=None, rv=None, rc=None, msg=None):
self.message = message
self.rv = rv
self.rc = rc
self.msg = msg
def __str__(self):
return ('rc=' + str(self.rc) + ', rv=' + str(self.rv) +
',msg=' + str(self.msg))
def __repr__(self):
return (self.__class__.__name__ + '(rc=' + str(self.rc) + ', rv=' +
str(self.rv) + ', msg=' + str(self.msg) + ')')
class PtlUtilError(Exception):
"""
Initialize PTL Util error
"""
def __init__(self, message=None, rv=None, rc=None, msg=None):
self.message = message
self.rv = rv
self.rc = rc
self.msg = msg
def __str__(self):
return ('rc=' + str(self.rc) + ', rv=' + str(self.rv) +
',msg=' + str(self.msg))
def __repr__(self):
return (self.__class__.__name__ + '(rc=' + str(self.rc) + ', rv=' +
str(self.rv) + ', msg=' + str(self.msg) + ')')
class DshUtils(object):
"""
PBS shell utilities
A set of tools to run commands, copy files, get process
information and parse a PBS configuration on an arbitrary host
"""
logger = logging.getLogger(__name__)
_h2osinfo = {} # host to OS info cache
_h2p = {} # host to platform cache
_h2pu = {} # host to uname cache
_h2c = {} # host to pbs_conf file cache
_h2l = {} # host to islocal cache
_h2which = {} # host to which cache
rsh_cmd = DFLT_RSH_CMD
sudo_cmd = DFLT_SUDO_CMD
copy_cmd = DFLT_COPY_CMD
def __init__(self):
self._current_user = None
logging.addLevelName('INFOCLI', logging.INFOCLI)
setattr(self.logger, 'infocli',
lambda *args: self.logger.log(logging.INFOCLI, *args))
logging.addLevelName('DEBUG2', logging.DEBUG2)
setattr(self.logger, 'debug2',
lambda *args: self.logger.log(logging.DEBUG2, *args))
logging.addLevelName('INFOCLI2', logging.INFOCLI2)
setattr(self.logger, 'infocli2',
lambda *args: self.logger.log(logging.INFOCLI2, *args))
self.mom_conf_map = {'PBS_MOM_SERVICE_PORT': '-M',
'PBS_MANAGER_SERVICE_PORT': '-R',
'PBS_HOME': '-d',
'PBS_BATCH_SERVICE_PORT': '-S',
}
self.server_conf_map = {'PBS_MOM_SERVICE_PORT': '-M',
'PBS_MANAGER_SERVICE_PORT': '-R',
'PBS_HOME': '-d',
'PBS_BATCH_SERVICE_PORT': '-p',
'PBS_SCHEDULER_SERVICE_PORT': '-S',
}
self.sched_conf_map = {'PBS_HOME': '-d',
'PBS_BATCH_SERVICE_PORT': '-p',
'PBS_SCHEDULER_SERVICE_PORT': '-S',
}
self._tempdir = {}
self.platform = self.get_platform()
def get_platform(self, hostname=None, pyexec=None):
"""
Get a local or remote platform info, essentially the value of
Python's sys.platform, in case of Cray it will return a string
as "cray" for actual Cray cluster and "craysim" for Cray ALPS
simulator
:param hostname: The hostname to query for platform info
:type hostname: str or None
:param pyexec: A path to a Python interpreter to use to query
a remote host for platform info
:type pyexec: str or None
For efficiency the value is cached and retrieved from the
cache upon subsequent request
"""
splatform = sys.platform
found_already = False
if hostname is None:
hostname = socket.gethostname()
if hostname in self._h2p:
return self._h2p[hostname]
if self.isfile(hostname=hostname, path='/etc/xthostname',
level=logging.DEBUG2):
if self.isfile(hostname=hostname, path='/proc/cray_xt/cname',
level=logging.DEBUG2):
splatform = 'cray'
else:
splatform = 'craysim'
found_already = True
if not self.is_localhost(hostname) and not found_already:
if pyexec is None:
pyexec = self.which(hostname, 'python', level=logging.DEBUG2)
cmd = [pyexec, '-c', '"import sys; print sys.platform"']
ret = self.run_cmd(hostname, cmd=cmd)
if ret['rc'] != 0 or len(ret['out']) == 0:
_msg = 'Unable to retrieve platform info,'
_msg += 'defaulting to local platform'
self.logger.warning(_msg)
splatform = sys.platform
else:
splatform = ret['out'][0]
self._h2p[hostname] = splatform
return splatform
def get_uname(self, hostname=None, pyexec=None):
"""
Get a local or remote platform info in uname format, essentially
the value of Python's platform.uname
:param hostname: The hostname to query for platform info
:type hostname: str or None
:param pyexec: A path to a Python interpreter to use to query
a remote host for platform info
:type pyexec: str or None
For efficiency the value is cached and retrieved from the
cache upon subsequent request
"""
uplatform = ' '.join(platform.uname())
if hostname is None:
hostname = socket.gethostname()
if hostname in self._h2pu:
return self._h2pu[hostname]
if not self.is_localhost(hostname):
if pyexec is None:
pyexec = self.which(hostname, 'python', level=logging.DEBUG2)
_cmdstr = '"import platform;'
_cmdstr += 'print \' \'.join(platform.uname())"'
cmd = [pyexec, '-c', _cmdstr]
ret = self.run_cmd(hostname, cmd=cmd)
if ret['rc'] != 0 or len(ret['out']) == 0:
_msg = 'Unable to retrieve platform info,'
_msg += 'defaulting to local platform'
self.logger.warning(_msg)
else:
uplatform = ret['out'][0]
self._h2pu[hostname] = uplatform
return uplatform
def get_os_info(self, hostname=None, pyexec=None):
"""
Get a local or remote OS info
:param hostname: The hostname to query for platform info
:type hostname: str or None
:param pyexec: A path to a Python interpreter to use to query
a remote host for platform info
:type pyexec: str or None
:returns: a 'str' object containing os info
"""
local_info = platform.platform()
if hostname is None or self.is_localhost(hostname):
return local_info
if hostname in self._h2osinfo:
return self._h2osinfo[hostname]
if pyexec is None:
pyexec = self.which(hostname, 'python', level=logging.DEBUG2)
cmd = [pyexec, '-c',
'"import platform; print platform.platform()"']
ret = self.run_cmd(hostname, cmd=cmd)
if ret['rc'] != 0 or len(ret['out']) == 0:
self.logger.warning("Unable to retrieve OS info, defaulting "
"to local")
ret_info = local_info
else:
ret_info = ret['out'][0]
self._h2osinfo[hostname] = ret_info
return ret_info
def _parse_file(self, hostname, file):
"""
helper function to parse a file containing entries of the
form ``=`` into a Python dictionary format
"""
if hostname is None:
hostname = socket.gethostname()
try:
rv = self.cat(hostname, file, level=logging.DEBUG2, logerr=False)
if rv['rc'] != 0:
return {}
props = {}
for l in rv['out']:
if l.find('=') != -1 and l[0] != '#':
c = l.split('=')
props[c[0]] = c[1].strip()
except:
self.logger.error('error parsing file ' + str(file))
self.logger.error(traceback.print_exc())
return {}
return props
def _set_file(self, hostname, fin, fout, append, variables, sudo=False):
"""
Create a file out of a set of dictionaries, possibly parsed
from an input file. @see _parse_file.
:param hostname: the name of the host on which to operate.
Defaults to localhost
:type hostname: str
:param fin: the input file to read from
:type fin: str
:param fout: the output file to write to
:type fout: str
:param append: If true, append to the output file.
:type append: bool
:param variables: The ``key/value`` pairs to write to fout
:type variables: dictionary
:param sudo: copy file to destination through sudo
:type sudo: boolean
:return dictionary of items set
:raises PbsConfigError:
"""
if hostname is None:
hostname = socket.gethostname()
if append:
conf = self._parse_file(hostname, fin)
else:
conf = {}
conf = dict(conf.items() + variables.items())
if os.path.isfile(fout):
fout_stat = os.stat(fout)
user = fout_stat.st_uid
group = fout_stat.st_gid
else:
user = None
group = None
try:
fn = self.create_temp_file()
self.chmod(path=fn, mode=0644)
with open(fn, 'w') as fd:
for k, v in conf.items():
fd.write(str(k) + '=' + str(v) + '\n')
rv = self.run_copy(hostname, fn, fout, uid=user, gid=group,
level=logging.DEBUG2, sudo=sudo)
if rv['rc'] != 0:
raise PbsConfigError
except:
raise PbsConfigError(rc=1, rv=None,
msg='error writing to file ' + str(fout))
finally:
if os.path.isfile(fn):
self.rm(path=fn)
return conf
def get_pbs_conf_file(self, hostname=None):
"""
Get the path of the pbs conf file. Defaults back to
``/etc/pbs.conf`` if unsuccessful
:param hostname: Hostname of the machine
:type hostname: str or None
:returns: Path to pbs conf file
"""
dflt_conf = '/etc/pbs.conf'
if hostname is None:
hostname = socket.gethostname()
if hostname in self._h2c:
return self._h2c[hostname]
if self.is_localhost(hostname):
if 'PBS_CONF_FILE' in os.environ:
dflt_conf = os.environ['PBS_CONF_FILE']
else:
pc = ('"import os;print [False, os.environ[\'PBS_CONF_FILE\']]'
'[\'PBS_CONF_FILE\' in os.environ]"')
cmd = ['python', '-c', pc]
ret = self.run_cmd(hostname, cmd, logerr=False)
if ((ret['rc'] != 0) and (len(ret['out']) > 0) and
(ret['out'][0] != 'False')):
dflt_conf = ret['out'][0]
self._h2c[hostname] = dflt_conf
return dflt_conf
def parse_pbs_config(self, hostname=None, file=None):
"""
initialize ``pbs_conf`` dictionary by parsing pbs config file
:param file: PBS conf file
:type file: str or None
"""
if file is None:
file = self.get_pbs_conf_file(hostname)
return self._parse_file(hostname, file)
def set_pbs_config(self, hostname=None, fin=None, fout=None,
append=True, confs=None):
"""
Set ``environment/configuration`` variables in a
``pbs.conf`` file
:param hostname: the name of the host on which to operate
:type hostname: str or None
:param fin: the input pbs.conf file
:type fin: str or None
:param fout: the name of the output pbs.conf file, defaults
to ``/etc/pbs.conf``
:type fout: str or None
:param append: whether to append to fout or not, defaults
to True
:type append: boolean
:param confs: The ``key/value`` pairs to create
:type confs: Dictionary or None
"""
if fin is None:
fin = self.get_pbs_conf_file(hostname)
if fout is None and fin is not None:
fout = fin
if confs is not None:
self.logger.info('Set ' + str(confs) + ' in ' + fout)
else:
confs = {}
return self._set_file(hostname, fin, fout, append, confs, sudo=True)
def unset_pbs_config(self, hostname=None, fin=None, fout=None,
confs=None):
"""
Unset ``environment/configuration`` variables in a pbs.conf
file
:param hostname: the name of the host on which to operate
:type hostname: str or None
:param fin: the input pbs.conf file
:type fin: str or None
:param fout: the name of the output pbs.conf file, defaults
to ``/etc/pbs.conf``
:type fout: str or None
:param confs: The configuration keys to unset
:type confs: List or str or dict or None
"""
if fin is None:
fin = self.get_pbs_conf_file(hostname)
if fout is None and fin is not None:
fout = fin
if confs is None:
confs = []
elif isinstance(confs, str):
confs = confs.split(',')
elif isinstance(confs, dict):
confs = confs.keys()
tounset = []
cur_confs = self.parse_pbs_config(hostname, fin)
for k in confs:
if k in cur_confs:
tounset.append(k)
del cur_confs[k]
if tounset:
self.logger.info('Unset ' + ",".join(tounset) + ' from ' + fout)
return self._set_file(hostname, fin, fout, append=False,
variables=cur_confs, sudo=True)
def get_pbs_server_name(self, pbs_conf=None):
"""
Return the name of the server which may be different than
``PBS_SERVER``,in order, this method looks at
``PBS_PRIMARY``, ``PBS_SERVER_HOST_NAME``, and
``PBS_LEAF_NAME``, and ``PBS_SERVER``
"""
if pbs_conf is None:
pbs_conf = self.parse_pbs_config()
if 'PBS_PRIMARY' in pbs_conf:
return pbs_conf['PBS_PRIMARY']
elif 'PBS_SERVER_HOST_NAME' in pbs_conf:
return pbs_conf['PBS_SERVER_HOST_NAME']
elif 'PBS_LEAF_NAME' in pbs_conf:
return pbs_conf['PBS_LEAF_NAME']
return pbs_conf['PBS_SERVER']
def parse_pbs_environment(self, hostname=None,
file='/var/spool/pbs/pbs_environment'):
"""
Initialize pbs_conf dictionary by parsing pbs config file
"""
return self._parse_file(hostname, file)
def set_pbs_environment(self, hostname=None,
fin='/var/spool/pbs/pbs_environment', fout=None,
append=True, environ=None):
"""
Set the PBS environment
:param environ: variables to set
:type environ: dict or None
:param hostname: Hostname of the machine
:type hostname: str or None
:param fin: pbs_environment input file
:type fin: str
:param fout: pbs_environment output file
:type fout: str or None
:param append: whether to append to fout or not, defaults
defaults to true
:type append: bool
"""
if fout is None and fin is not None:
fout = fin
if environ is None:
environ = {}
return self._set_file(hostname, fin, fout, append, environ, sudo=True)
def unset_pbs_environment(self, hostname=None,
fin='/var/spool/pbs/pbs_environment', fout=None,
environ=None):
"""
Unset environment variables in a pbs_environment file
:param hostname: the name of the host on which to operate
:type hostname: str or None
:param fin: the input pbs_environment file
:type fin: str
:param fout: the name of the output pbs.conf file, defaults
to ``/var/spool/pbs/pbs_environment``
:type fout: str or None
:param environ: The environment keys to unset
:type environ: List or str or dict or None
"""
if fout is None and fin is not None:
fout = fin
if environ is None:
environ = []
elif isinstance(environ, str):
environ = environ.split(',')
elif isinstance(environ, dict):
environ = environ.keys()
tounset = []
cur_environ = self.parse_pbs_environment(hostname, fin)
for k in environ:
if k in cur_environ:
tounset.append(k)
del cur_environ[k]
if tounset:
self.logger.info('Unset ' + ",".join(tounset) + ' from ' + fout)
return self._set_file(hostname, fin, fout, append=False,
variables=cur_environ, sudo=True)
def parse_rhosts(self, hostname=None, user=None):
"""
Parse remote host
:param hostname: Hostname of the machine
:type hostname: str or None
:param user: User name
:type user: str or None
"""
if hostname is None:
hostname = socket.gethostname()
if user is None:
user = os.getuid()
try:
# currently assumes identical file system layout on every host
if isinstance(user, int):
home = pwd.getpwuid(user).pw_dir
else:
home = pwd.getpwnam(user).pw_dir
rhost = os.path.join(home, '.rhosts')
rv = self.cat(hostname, rhost, level=logging.DEBUG2, runas=user,
logerr=False)
if rv['rc'] != 0:
return {}
props = {}
for l in rv['out']:
if l[0] != '#':
k, v = l.split()
v = v.strip()
if k in self.props:
if isinstance(self.props[k], list):
self.props[k].append(v)
else:
self.props[k] = [self.props[k], v]
else:
self.props[k] = v
except:
self.logger.error('error parsing .rhost')
self.logger.error(traceback.print_exc())
return {}
return props
def set_rhosts(self, hostname=None, user=None, entry={}, append=True):
"""
Set the remote host attributes
:param entry: remote hostname user dictionary
:type entry: Dictionary
:param append: If true append key value else not
:type append: boolean
"""
if hostname is None:
hostname = socket.gethostname()
if user is None:
user = os.getuid()
if append:
conf = self.parse_rhosts(hostname, user)
for k, v in entry.items():
if k in conf:
if isinstance(conf[k], list):
if isinstance(v, list):
conf[k].extend(v)
else:
conf[k].append(v)
else:
if isinstance(v, list):
conf[k] = [conf[k]] + v
else:
conf[k] = [conf[k], v]
else:
conf[k] = v
else:
conf = entry
try:
# currently assumes identical file system layout on every host
if isinstance(user, int):
_user = pwd.getpwuid(user)
home = _user.pw_dir
uid = _user.pw_uid
else:
# user might be PbsUser object
_user = pwd.getpwnam(str(user))
home = _user.pw_dir
uid = _user.pw_uid
rhost = os.path.join(home, '.rhosts')
fn = self.create_temp_file(hostname)
self.chmod(hostname, fn, mode=0755)
with open(fn, 'w') as fd:
fd.write('#!/bin/bash\n')
fd.write('cd %s\n' % (home))
fd.write('%s -rf %s\n' % (self.which(hostname, 'rm',
level=logging.DEBUG2),
rhost))
fd.write('touch %s\n' % (rhost))
for k, v in conf.items():
if isinstance(v, list):
for eachprop in v:
l = 'echo "%s %s" >> %s\n' % (str(k),
str(eachprop),
rhost)
fd.write(l)
else:
l = 'echo "%s %s" >> %s\n' % (str(k), str(v), rhost)
fd.write(l)
fd.write('%s 0600 %s\n' % (self.which(hostname, 'chmod',
level=logging.DEBUG2),
rhost))
ret = self.run_cmd(hostname, cmd=fn, runas=uid)
self.rm(hostname, path=fn)
if ret['rc'] != 0:
raise Exception(ret['out'] + ret['err'])
except Exception, e:
raise PbsConfigError(rc=1, rv=None, msg='error writing .rhosts ' +
str(e))
return conf
def map_pbs_conf_to_cmd(self, cmd_map={}, pconf={}):
"""
Map PBS configuration parameter to command
:param cmd_map: command mapping
:type cmd_map: Dictionary
:param pconf: PBS conf parameter dictionary
:type pconf: Dictionary
"""
cmd = []
for k, v in pconf.items():
if k in cmd_map:
cmd += [cmd_map[k], str(v)]
return cmd
def get_current_user(self):
"""
helper function to return the name of the current user
"""
if self._current_user is not None:
return self._current_user
self._current_user = pwd.getpwuid(os.getuid())[0]
return self._current_user
def check_user_exists(self, username=None, hostname=None):
"""
Check if user exist or not
:param username: Username to check
:type username: str or None
:param hostname: Machine hostname
:type hostname: str or None
:returns: True if exist else return False
"""
if hostname is None:
hostname = socket.gethostname()
ret = self.run_cmd(hostname, ['id', username])
if ret['rc'] == 0:
return True
return False
def check_group_membership(self, username=None, uid=None, grpname=None,
gid=None):
"""
Checks whether a user, passed in as username or uid, is a
member of a group, passed in as group name or group id.
:param username: The username to inquire about
:type username: str or None
:param uid: The uid of the user to inquire about (alternative
to username)
:param grpname: The groupname to check for user membership
:type grpname: str or None
:param gid: The group id to check for user membership
(alternative to grpname)
"""
if username is None and uid is None:
self.logger.warning('A username or uid was expected')
return True
if grpname is None and gid is None:
self.logger.warning('A grpname or gid was expected')
return True
if grpname:
try:
_g = grp.getgrnam(grpname)
if username and username in _g.gr_mem:
return True
elif uid is not None:
_u = pwd.getpwuid(uid)
if _u.pwname in _g.gr_mem:
return True
except:
self.logger.error('Unknown user')
return False
def group_memberships(self, group_list=[]):
"""
Returns all group memberships as a dictionary of group names
and associated memberships
"""
groups = {}
if not group_list:
return groups
users_list = [u.pw_name for u in pwd.getpwall()]
glist = {}
for u in users_list:
info = self.get_id_info(u)
if not info['pgroup'] in glist.keys():
glist[info['pgroup']] = [info['name']]
else:
glist[info['pgroup']].append(info['name'])
for g in info['groups']:
if g not in glist.keys():
glist[g] = []
if not info['name'] in glist[g]:
glist[g].append(info['name'])
for g in group_list:
if g in glist.keys():
groups[g] = glist[g]
else:
try:
i = grp.getgrnam(g)
groups[g] = i.gr_mem
except KeyError:
pass
return groups
def get_id_info(self, user):
"""
Return user info in dic format
obtained by ``"id -a "`` command for given user
:param user: The username to inquire about
:type user: str
:returns: dic format:
{
"uid": ,
"gid": ,
"name": ,
"pgroup": ,
"groups":
}
"""
info = {'uid': None, 'gid': None, 'name': None, 'pgroup': None,
'groups': None}
ret = self.run_cmd(cmd=['id', '-a', str(user)], logerr=True)
if ret['rc'] == 0:
p = re.compile(r'(?P\d+)\((?P[\w\s."\'-]+)\)')
map_list = re.findall(p, ret['out'][0])
info['uid'] = int(map_list[0][0])
info['name'] = map_list[0][1].strip()
info['gid'] = int(map_list[1][0])
info['pgroup'] = map_list[1][1].strip()
groups = []
if len(map_list) > 2:
for g in map_list[2:]:
groups.append(g[1].strip().strip('"').strip("'"))
info['groups'] = groups
return info
def get_tempdir(self, hostname=None):
"""
:returns: The temporary directory on the given host
Default host is localhost.
"""
# return the cached value whenever possible
if hostname is None:
hostname = socket.gethostname()
if hostname in self._tempdir:
return self._tempdir[hostname]
if self.is_localhost(hostname):
self._tempdir[hostname] = tempfile.gettempdir()
else:
cmd = ['python', '-c',
'"import tempfile;print tempfile.gettempdir()"']
ret = self.run_cmd(hostname, cmd, level=logging.DEBUG)
if ret['rc'] == 0:
self._tempdir[hostname] = ret['out'][0].strip()
else:
# Optimistically fall back to /tmp.
self._tempdir[hostname] = '/tmp'
return self._tempdir[hostname]
def run_cmd(self, hosts=None, cmd=None, sudo=False, stdin=None,
stdout=PIPE, stderr=PIPE, input=None, cwd=None, env=None,
runas=None, logerr=True, as_script=False, wait_on_script=True,
level=logging.INFOCLI2):
"""
Run a command on a host or list of hosts.
:param hosts: the name of hosts on which to run the command,
can be a comma-separated string or a list.
Defaults to localhost
:type hosts: str or None
:param cmd: the command to run
:type cmd: str or None
:param sudo: whether to run the command as root or not.
Defaults to False.
:type sudo: boolean
:param stdin: custom stdin. Defaults to PIPE
:param stdout: custom stdout. Defaults to PIPE
:param stderr: custom stderr. Defaults to PIPE
:param input: input to pass to the pipe on target host,
e.g. PBS answer file
:param cwd: working directory on local host from which
command is run
:param env: environment variables to set on local host
:param runas: run command as given user. Defaults to calling
user
:param logerr: whether to log error messages or not. Defaults
to True
:type logerr: boolean
:param as_script: if True, run the command in a script
created as a temporary file that gets
deleted after being run. This is used
mainly to circumvent some implementations
of sudo that prevent passing environment
variables through sudo.
:type as_script: boolean
:param wait_on_script: If True (default) waits on process
launched as script to return.
:type wait_on_script: boolean
:returns: error, output, return code as a dictionary:
``{'out':...,'err':...,'rc':...}``
"""
rshcmd = []
sudocmd = []
if level is None:
level = self.logger.level
_user = self.get_current_user()
# runas may be a PbsUser object, ensure it is a string for the
# remainder of the function
if runas is not None:
if isinstance(runas, int):
runas = pwd.getpwuid(runas).pw_name
elif not isinstance(runas, str):
# must be as PbsUser object
runas = str(runas)
if isinstance(cmd, str):
cmd = cmd.split()
if hosts is None:
hosts = socket.gethostname()
if isinstance(hosts, str):
hosts = hosts.split(',')
if not isinstance(hosts, list):
err_msg = 'target hostnames must be a comma-separated ' + \
'string or list'
self.logger.error(err_msg)
return {'out': '', 'err': err_msg, 'rc': 1}
ret = {'out': '', 'err': '', 'rc': 0}
for hostname in hosts:
islocal = self.is_localhost(hostname)
if islocal is None:
# an error occurred processing that name, move on
# the error is logged in is_localhost.
ret['err'] = 'error getting host by name in run_cmd'
ret['rc'] = 1
continue
if not islocal:
rshcmd = self.rsh_cmd + [hostname]
if sudo or ((runas is not None) and (runas != _user)):
sudocmd = copy.copy(self.sudo_cmd)
if runas is not None:
sudocmd += ['-u', runas]
# Initialize information to return
ret = {'out': None, 'err': None, 'rc': None}
rc = rshcmd + sudocmd + cmd
if as_script:
_script = self.create_temp_file()
script_body = ['#!/bin/bash']
if cwd is not None:
script_body += ['cd "%s"' % (cwd)]
cwd = None
if isinstance(cmd, str):
script_body += [cmd]
elif isinstance(cmd, list):
script_body += [" ".join(cmd)]
with open(_script, 'w') as f:
f.write('\n'.join(script_body))
os.chmod(_script, 0755)
if not islocal:
# TODO: get a valid remote temporary file rather than
# assume that the remote host has a similar file
# system layout
self.run_copy(hostname, _script, _script, level=level)
os.remove(_script)
runcmd = rshcmd + sudocmd + [_script]
else:
runcmd = rc
_msg = hostname.split('.')[0] + ': '
_runcmd = map(lambda x: '\'\'' if x == '' else str(x), runcmd)
_msg += ' '.join(_runcmd)
_msg = [_msg]
if as_script:
_msg += ['Contents of ' + _script + ':']
_msg += ['-' * 40, '\n'.join(script_body), '-' * 40]
self.logger.log(level, '\n'.join(_msg))
if input:
self.logger.log(level, input)
try:
p = Popen(runcmd, bufsize=-1, stdin=stdin, stdout=stdout,
stderr=stderr, cwd=cwd, env=env)
except Exception, e:
self.logger.error("Error running command " + str(runcmd))
if as_script:
self.logger.error('Script contents: \n' +
'\n'.join(script_body))
self.logger.debug(str(e))
raise
if as_script and not wait_on_script:
o = p.stdout.readline()
e = p.stderr.readline()
ret['rc'] = 0
else:
(o, e) = p.communicate(input)
ret['rc'] = p.returncode
if as_script:
# must pass as_script=False otherwise it will loop infinite
self.rm(hostname, path=_script, as_script=False,
level=level)
# handle the case where stdout is not a PIPE
if o is not None:
ret['out'] = o.splitlines()
else:
ret['out'] = []
# Some output can be very verbose, for example listing many lines
# of a log file, those messages are typically channeled through
# at level DEBUG2, since we don't to pollute the output with too
# verbose an information, we log at most at level DEBUG
if level < logging.DEBUG:
self.logger.log(level, 'out: ' + str(ret['out']))
else:
self.logger.debug('out: ' + str(ret['out']))
if e is not None:
ret['err'] = e.splitlines()
else:
ret['err'] = []
if ret['err'] and logerr:
self.logger.error('err: ' + str(ret['err']))
else:
self.logger.debug('err: ' + str(ret['err']))
self.logger.debug('rc: ' + str(ret['rc']))
return ret
def run_copy(self, hosts=None, src=None, dest=None, sudo=False, uid=None,
gid=None, mode=None, env=None, logerr=True,
recursive=False, runas=None, preserve_permission=True,
level=logging.INFOCLI2):
"""
copy a file or directory to specified target hosts.
:param hosts: the host(s) to which to copy the data. Can be
a comma-separated string or a list
:type hosts: str or None
:param src: the path to the file or directory to copy. If
src is remote,it must be prefixed by the
hostname. ``e.g. remote1:/path,remote2:/path``
:type src: str or None
:param dest: the destination path.
:type dest: str or None
:param sudo: whether to copy as root or not. Defaults to
False
:type sudo: boolean
:param uid: optionally change ownership of dest to the
specified user id,referenced by uid number or
username
:param gid: optionally change ownership of dest to the
specified group ``name/id``
:param mode: optinoally set mode bits of dest
:param env: environment variables to set on the calling host
:param logerr: whether to log error messages or not.
Defaults to True.
:param recursive: whether to copy a directory (when true) or
a file.Defaults to False.
:type recursive: boolean
:param runas: run command as user
:type runas: str or None
:param preserve_permission: Preserve file permission while
copying file (cp cmd with -p flag)
Defaults to True
:type preserve_permission:boolean
:param level: logging level, defaults to DEBUG
:type level: int
:returns: {'out':, 'err': , 'rc':}
upon and None if no source file specified
"""
if src is None:
self.logger.warning('no source file specified')
return None
if hosts is None:
hosts = socket.gethostname()
if isinstance(hosts, str):
hosts = hosts.split(',')
if not isinstance(hosts, list):
self.logger.error('destination must be a string or a list')
return 1
if dest is None:
dest = src
# If PTL_SUDO_CMD were to be unset we should assume no sudo
if sudo is True and not self.sudo_cmd:
sudo = False
for targethost in hosts:
islocal = self.is_localhost(targethost)
if sudo and not islocal:
# to avoid a file copy as root, we copy it as current user
# and move it remotely to the desired path/name.
# First, get a remote temporary filename
cmd = ['python', '-c',
'"import tempfile;print ' +
'tempfile.mkstemp(\'PtlPbstmpcopy\')[1]"']
# save original destination
sudo_save_dest = dest
# Make the target of the copy the temporary file
dest = self.run_cmd(targethost, cmd,
level=level,
logerr=logerr)['out'][0]
cmd = []
else:
# if not using sudo or target is local, initialize the
# command to run accordingly
sudo_save_dest = None
if sudo:
cmd = copy.copy(self.sudo_cmd)
else:
cmd = []
# Remote copy if target host is remote or if source file/dir is
# remote.
if ((not islocal) or (':' in src)):
copy_cmd = copy.deepcopy(self.copy_cmd)
if not preserve_permission:
copy_cmd.remove('-p')
if copy_cmd[0][0] != '/':
copy_cmd[0] = self.which(targethost, copy_cmd[0],
level=level)
cmd += copy_cmd
if recursive:
cmd += ['-r']
cmd += [src]
if islocal:
cmd += [dest]
else:
cmd += [targethost + ':' + dest]
else:
cmd += [self.which(targethost, 'cp', level=level)]
if preserve_permission:
cmd += ['-p']
if recursive:
cmd += ['-r']
cmd += [src]
cmd = cmd + [dest]
ret = self.run_cmd(socket.gethostname(), cmd, env=env, runas=runas,
logerr=logerr, level=level)
if ret['rc'] != 0:
self.logger.error(ret['err'])
elif sudo_save_dest:
cmd = [self.which(targethost, 'cp', level=level)]
cmd += [dest, sudo_save_dest]
ret = self.run_cmd(targethost, cmd=cmd, sudo=True, level=level)
self.rm(targethost, path=dest, level=level)
dest = sudo_save_dest
if ret['rc'] != 0:
self.logger.error(ret['err'])
if mode is not None:
self.chmod(targethost, path=dest, mode=mode, sudo=sudo,
recursive=recursive, runas=runas)
if ((uid is not None and uid != self.get_current_user()) or
gid is not None):
self.chown(targethost, path=dest, uid=uid, gid=gid, sudo=True,
recursive=False)
return ret
def run_ptl_cmd(self, hostname, cmd, sudo=False, stdin=None, stdout=PIPE,
stderr=PIPE, input=None, cwd=None, env=None, runas=None,
logerr=True, as_script=False, wait_on_script=True,
level=logging.INFOCLI2):
"""
Wrapper method of run_cmd to run PTL command
"""
# Add absolute path of command also add log level to command
self.logger.infocli('running command "%s" on %s' % (' '.join(cmd),
hostname))
_cmd = [self.which(exe=cmd[0], level=level)]
_cmd += ['-l', logging.getLevelName(self.logger.parent.level)]
_cmd += cmd[1:]
cmd = _cmd
self.logger.debug(' '.join(cmd))
dest = None
if ('PYTHONPATH' in os.environ.keys() and
not self.is_localhost(hostname)):
body = ['#!/bin/bash']
body += ['PYTHONPATH=%s exec %s' % (os.environ['PYTHONPATH'],
' '.join(cmd))]
fn = self.create_temp_file(body='\n'.join(body))
tmpdir = self.get_tempdir(hostname)
dest = os.path.join(tmpdir, os.path.basename(fn))
oldc = self.copy_cmd[:]
self.set_copy_cmd('scp -p')
self.run_copy(hostname, fn, dest, mode=0755, level=level)
self.set_copy_cmd(' '.join(oldc))
self.rm(None, path=fn, force=True, logerr=False)
cmd = dest
ret = self.run_cmd(hostname, cmd, sudo, stdin, stdout, stderr, input,
cwd, env, runas, logerr, as_script, wait_on_script,
level)
if dest is not None:
self.rm(hostname, path=dest, force=True, logerr=False)
# TODO: check why output is coming to ret['err']
if ret['rc'] == 0:
ret['out'] = ret['err']
ret['err'] = []
return ret
@classmethod
def set_sudo_cmd(cls, cmd):
"""
set the sudo command
"""
cls.logger.infocli('setting sudo command to ' + cmd)
cls.sudo_cmd = cmd.split()
@classmethod
def set_copy_cmd(cls, cmd):
"""
set the copy command
"""
cls.logger.infocli('setting copy command to ' + cmd)
cls.copy_cmd = cmd.split()
@classmethod
def set_rsh_cmd(cls, cmd):
"""
set the remote shell command
"""
cls.logger.infocli('setting remote shell command to ' + cmd)
cls.rsh_cmd = cmd.split()
def is_localhost(self, host=None):
"""
:param host: Hostname of machine
:type host: str or None
:returns: true if specified host (by name) is the localhost
all aliases matching the hostname are searched
"""
if host is None:
return True
if host in self._h2l:
return self._h2l[host]
try:
(hostname, aliaslist, iplist) = socket.gethostbyname_ex(host)
except:
self.logger.error('error getting host by name: ' + host)
print traceback.print_stack()
return None
localhost = socket.gethostname()
if localhost == hostname or localhost in aliaslist:
self._h2l[host] = True
try:
ipaddr = socket.gethostbyname(localhost)
except:
self.logger.error('could not resolve local host name')
return False
if ipaddr in iplist:
self._h2l[host] = True
return True
self._h2l[host] = False
return False
def isdir(self, hostname=None, path=None, sudo=False, runas=None,
level=logging.INFOCLI2):
"""
:param hostname: The name of the host on which to check for
directory
:type hostname: str or None
:param path: The path to the directory to check
:type path: str or None
:param sudo: Whether to run the command as a privileged user
:type sudo: boolean
:param runas: run command as user
:type runas: str or None
:param level: Logging level
:returns: True if directory pointed to by path exists and
False otherwise
"""
if path is None:
return False
if (self.is_localhost(hostname) and (not sudo) and (runas is None)):
return os.path.isdir(path)
else:
# Constraints on the build system prevent running commands as
# a privileged user through python, fall back to ls
dirname = os.path.dirname(path)
basename = os.path.basename(path)
cmd = ['ls', '-l', dirname]
self.logger.log(level, "grep'ing for " + basename + " in " +
dirname)
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, runas=runas,
logerr=False, level=level)
if ret['rc'] != 0:
return False
else:
for l in ret['out']:
if basename == l[-len(basename):] and l.startswith('d'):
return True
return False
def isfile(self, hostname=None, path=None, sudo=False, runas=None,
level=logging.INFOCLI2):
"""
:param hostname: The name of the host on which to check for
file
:type hostname: str or None
:param path: The path to the file to check
:type path: str or None
:param sudo: Whether to run the command as a privileged user
:type sudo: boolean
:param runas: run command as user
:type runas: str or None
:param level: Logging level
:returns: True if file pointed to by path exists, and False
otherwise
"""
if path is None:
return False
if (self.is_localhost(hostname) and (not sudo) and (runas is None)):
return os.path.isfile(path)
else:
# Constraints on the build system prevent running commands as
# a privileged user through python, fall back to ls
cmd = ['ls', '-l', path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, runas=runas,
logerr=False, level=level)
if ret['rc'] != 0:
return False
elif ret['out']:
if not ret['out'][0].startswith('d'):
return True
return False
def getmtime(self, hostname=None, path=None, sudo=False, runas=None,
level=logging.INFOCLI2):
"""
:param hostname: The name of the host on which file exists
:type hostname: str or None
:param path: The path to the file to get mtime
:type path: str or None
:param sudo: Whether to run the command as a privileged user
:type sudo: boolean
:param runas: run command as user
:type runas: str or None
:param level: Logging level
:returns: Modified time of given file
"""
if path is None:
return None
if (self.is_localhost(hostname) and (not sudo) and (runas is None)):
return os.path.getmtime(path)
else:
py_cmd = 'import os; print os.path.getmtime(\'%s\')' % (path)
if not self.is_localhost(hostname):
py_cmd = '\"' + py_cmd + '\"'
cmd = [self.which(hostname, 'python', level=level), '-c', py_cmd]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, runas=runas,
logerr=False, level=level)
if ((ret['rc'] == 0) and (len(ret['out']) == 1) and
(isinstance(eval(ret['out'][0].strip()), (int, float)))):
return eval(ret['out'][0].strip())
return None
def listdir(self, hostname=None, path=None, sudo=False, runas=None,
fullpath=True, level=logging.INFOCLI2):
"""
:param hostname: The name of the host on which to list for
directory
:type hostname: str or None
:param path: The path to directory to list
:type path: str or None
:param sudo: Whether to chmod as root or not. Defaults to
False
:type sudo: bool
:param runas: run command as user
:type runas: str or None
:param fullpath: Return full paths?
:type fullpath: bool
:param level: Logging level.
:type level: int
:returns: A list containing the names of the entries in
the directory
"""
if path is None:
return None
if (self.is_localhost(hostname) and (not sudo) and (runas is None)):
try:
files = os.listdir(path)
except OSError:
return None
else:
ret = self.run_cmd(hostname, cmd=['ls', path], sudo=sudo,
runas=runas, logerr=False, level=level)
if ret['rc'] == 0:
files = ret['out']
else:
return None
if fullpath is True:
return map(lambda p: os.path.join(path, p.strip()), files)
else:
return map(lambda p: p.strip(), files)
def chmod(self, hostname=None, path=None, mode=None, sudo=False,
runas=None, recursive=False, logerr=True,
level=logging.INFOCLI2):
"""
Generic function of chmod with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param path: the path to the file or directory to chmod
:type path: str or None
:param mode: mode to apply as octal number like 0777,
0666 etc.
:param sudo: whether to chmod as root or not. Defaults
to False
:type sudo: boolean
:param runas: run command as user
:type runas: str or None
:param recursive: whether to chmod a directory (when true)
or a file.Defaults to False.
:type recursive: boolean
:param logerr: whether to log error messages or not. Defaults
to True.
:type logerr: boolean
:param level: logging level, defaults to INFOCLI2
:returns: True on success otherwise False
"""
if (path is None) or (mode is None):
return False
cmd = [self.which(hostname, 'chmod', level=level)]
if recursive:
cmd += ['-R']
cmd += [oct(mode), path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, logerr=logerr,
runas=runas, level=level)
if ret['rc'] == 0:
return True
return False
def chown(self, hostname=None, path=None, uid=None, gid=None, sudo=False,
recursive=False, runas=None, logerr=True,
level=logging.INFOCLI2):
"""
Generic function of chown with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param path: the path to the file or directory to chown
:type path: str or None
:param uid: uid to apply (must be either user name or
uid or -1)
:param gid: gid to apply (must be either group name or
gid or -1)
:param sudo: whether to chown as root or not. Defaults
to False
:type sudo: boolean
:param recursive: whether to chmod a directory (when true)
or a file.Defaults to False.
:type recursive: boolean
:param runas: run command as user
:type runas: str or None
:param logerr: whether to log error messages or not. Defaults
to True.
:type logerr: boolean
:param level: logging level, defaults to INFOCLI2
:returns: True on success otherwise False
"""
if path is None or (uid is None and gid is None):
return False
_u = ''
if isinstance(uid, int)and uid != -1:
_u = pwd.getpwuid(uid).pw_name
elif (isinstance(uid, str) and (uid != '-1')):
_u = uid
else:
# must be as PbsUser object
if str(uid) != '-1':
_u = str(uid)
if _u == '':
return False
cmd = [self.which(hostname, 'chown', level=level)]
if recursive:
cmd += ['-R']
cmd += [_u, path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, logerr=logerr,
runas=runas, level=level)
if ret['rc'] == 0:
if gid is not None:
rv = self.chgrp(hostname, path, gid=gid, sudo=sudo,
level=level, recursive=recursive, runas=runas,
logerr=logerr)
if not rv:
return False
return True
return False
def chgrp(self, hostname=None, path=None, gid=None, sudo=False,
recursive=False, runas=None, logerr=True,
level=logging.INFOCLI2):
"""
Generic function of chgrp with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param path: the path to the file or directory to chown
:type path: str or None
:param gid: gid to apply (must be either group name or
gid or -1)
:param sudo: whether to chgrp as root or not. Defaults
to False
:type sudo: boolean
:param recursive: whether to chmod a directory (when true)
or a file.Defaults to False.
:type recursive: boolean
:param runas: run command as user
:type runas: str or None
:param logerr: whether to log error messages or not. Defaults
to True.
:type logerr: boolean
:param level: logging level, defaults to INFOCLI2
:returns: True on success otherwise False
"""
if path is None or gid is None:
return False
_g = ''
if isinstance(gid, int) and gid != -1:
_g = grp.getgrgid(gid).gr_name
elif (isinstance(gid, str) and (gid != '-1')):
_g = gid
else:
# must be as PbsGroup object
if str(gid) != '-1':
_g = str(gid)
if _g == '':
return False
cmd = [self.which(hostname, 'chgrp', level=level)]
if recursive:
cmd += ['-R']
cmd += [_g, path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, logerr=logerr,
runas=runas, level=level)
if ret['rc'] == 0:
return True
return False
def which(self, hostname=None, exe=None, level=logging.INFOCLI2):
"""
Generic function of which with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param exe: executable to locate (can be full path also)
(if exe is full path then only basename will
be used to locate)
:type exe: str or None
:param level: logging level, defaults to INFOCLI2
"""
if exe is None:
return None
if hostname is None:
hostname = socket.gethostname()
oexe = exe
exe = os.path.basename(exe)
if hostname in self._h2which.keys():
if exe in self._h2which[hostname]:
return self._h2which[hostname][exe]
sudo_wrappers_dir = '/opt/tools/wrappers'
_exe = os.path.join(sudo_wrappers_dir, exe)
if os.path.isfile(_exe) and os.access(_exe, os.X_OK):
if hostname not in self._h2which.keys():
self._h2which.setdefault(hostname, {exe: _exe})
else:
self._h2which[hostname].setdefault(exe, _exe)
return _exe
cmd = ['which', exe]
ret = self.run_cmd(hostname, cmd=cmd, logerr=False,
level=level)
if ((ret['rc'] == 0) and (len(ret['out']) == 1) and
os.path.isabs(ret['out'][0].strip())):
path = ret['out'][0].strip()
if hostname not in self._h2which.keys():
self._h2which.setdefault(hostname, {exe: path})
else:
self._h2which[hostname].setdefault(exe, path)
return path
else:
return oexe
def rm(self, hostname=None, path=None, sudo=False, runas=None,
recursive=False, force=False, cwd=None, logerr=True,
as_script=False, level=logging.INFOCLI2):
"""
Generic function of rm with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param path: the path to the files or directories to remove
for more than one files or directories pass as
list
:type path: str or None
:param sudo: whether to remove files or directories as root
or not.Defaults to False
:type sudo: boolean
:param runas: remove files or directories as given user
Defaults to calling user
:param recursive: remove files or directories and their
contents recursively
:type recursive: boolean
:param force: force remove files or directories
:type force: boolean
:param cwd: working directory on local host from which
command is run
:param logerr: whether to log error messages or not.
Defaults to True.
:type logerr: boolean
:param as_script: if True, run the rm in a script created
as a temporary file that gets deleted after
being run. This is used mainly to handle
wildcard in path list. Defaults to False.
:type as_script: boolean
:param level: logging level, defaults to INFOCLI2
:returns: True on success otherwise False
"""
if (path is None) or (len(path) == 0):
return True
cmd = [self.which(hostname, 'rm', level=level)]
if recursive and force:
cmd += ['-rf']
else:
if recursive:
cmd += ['-r']
if force:
cmd += ['-f']
if isinstance(path, list):
for p in path:
if p == '/':
msg = 'encountered a dangerous package path ' + p
self.logger.error(msg)
return False
cmd += path
else:
if path == '/':
msg = 'encountered a dangerous package path ' + path
self.logger.error(msg)
return False
cmd += [path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, logerr=logerr,
runas=runas, cwd=cwd, level=level,
as_script=as_script)
if ret['rc'] != 0:
return False
return True
def mkdir(self, hostname=None, path=None, mode=None, sudo=False,
runas=None, parents=True, cwd=None, logerr=True,
as_script=False, level=logging.INFOCLI2):
"""
Generic function of mkdir with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param path: the path to the directories to create
for more than one directories pass as list
:type path: str or None
:param mode: mode to use while creating directories
(must be octal like 0777)
:param sudo: whether to create directories as root or not.
Defaults to False
:type sudo: boolean
:param runas: create directories as given user. Defaults to
calling user
:param parents: create parent directories as needed. Defaults
to True
:type parents: boolean
:param cwd: working directory on local host from which
command is run
:type cwd: str or None
:param logerr: whether to log error messages or not. Defaults
to True.
:type logerr: boolean
:param as_script: if True, run the command in a script
created as a temporary file that gets
deleted after being run. This is used
mainly to handle wildcard in path list.
Defaults to False.
:type as_script: boolean
:param level: logging level, defaults to INFOCLI2
:returns: True on success otherwise False
"""
if (path is None) or (len(path) == 0):
return True
cmd = [self.which(hostname, 'mkdir', level=level)]
if parents:
cmd += ['-p']
if mode is not None:
cmd += ['-m', oct(mode)]
if isinstance(path, list):
cmd += path
else:
cmd += [path]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, logerr=logerr,
runas=runas, cwd=cwd, level=level,
as_script=as_script)
if ret['rc'] != 0:
return False
return True
def cat(self, hostname=None, filename=None, sudo=False, runas=None,
logerr=True, level=logging.INFOCLI2):
"""
Generic function of cat with remote host support
:param hostname: hostname (default current host)
:type hostname: str or None
:param filename: the path to the filename to cat
:type filename: str or None
:param sudo: whether to create directories as root or not.
Defaults to False
:type sudo: boolean
:param runas: create directories as given user. Defaults
to calling user
:type runas: str or None
:param logerr: whether to log error messages or not. Defaults
to True.
:type logerr: boolean
:returns: output of run_cmd
"""
cmd = [self.which(hostname, 'cat', level=level), filename]
return self.run_cmd(hostname, cmd=cmd, sudo=sudo,
runas=runas, logerr=logerr, level=level)
def cmp(self, hostname=None, fileA=None, fileB=None, sudo=False,
runas=None, logerr=True):
"""
Compare two files and return 0 if they are identical or
non-zero if not
:param hostname: the name of the host to operate on
:type hostname: str or None
:param fileA: the first file to compare
:type fileA: str or None
:param fileB: the file to compare fileA to
:type fileB: str or None
:param sudo: run the command as a privileged user
:type sudo: boolean
:param runas: run the cmp command as given user
:type runas: str or None
:param logerr: whether to log error messages or not.
Defaults to True.
:type logerr: boolean
"""
if fileA is None and fileB is None:
return 0
if fileA is None or fileB is None:
return 1
cmd = ['cmp', fileA, fileB]
ret = self.run_cmd(hostname, cmd=cmd, sudo=sudo, runas=runas,
logerr=logerr)
return ret['rc']
def useradd(self, name, uid=None, gid=None, shell='/bin/bash',
create_home_dir=True, home_dir=None, groups=None, logerr=True,
level=logging.INFOCLI2):
"""
Add the user
:param name: User name
:type name: str
:param shell: shell to use
:param create_home_dir: If true then create home directory
:type create_home_dir: boolean
:param home_dir: path to home directory
:type home_dir: str or None
:param groups: User groups
"""
self.logger.info('adding user ' + str(name))
cmd = ['useradd']
cmd += ['-K', 'UMASK=0022']
if uid is not None:
cmd += ['-u', str(uid)]
if shell is not None:
cmd += ['-s', shell]
if gid is not None:
cmd += ['-g', str(gid)]
if create_home_dir:
cmd += ['-m']
if home_dir is not None:
cmd += ['-d', home_dir]
if ((groups is not None) and (len(groups) > 0)):
cmd += ['-G', ','.join(map(lambda g: str(g), groups))]
cmd += [str(name)]
ret = self.run_cmd(cmd=cmd, logerr=logerr, sudo=True, level=level)
if ((ret['rc'] != 0) and logerr):
raise PtlUtilError(rc=ret['rc'], rv=False, msg=ret['err'])
def userdel(self, name, del_home=True, force=True, logerr=True,
level=logging.INFOCLI2):
"""
Delete the user
:param del_home: If true then delete user home
:type del_home: boolean
:param force: If true then delete forcefully
:type force: boolean
"""
cmd = ['userdel']
if del_home:
cmd += ['-r']
if force:
cmd += ['-f']
cmd += [str(name)]
self.logger.info('deleting user ' + str(name))
ret = self.run_cmd(cmd=cmd, sudo=True, logerr=False, level=level)
if ((ret['rc'] != 0) and logerr):
raise PtlUtilError(rc=ret['rc'], rv=False, msg=ret['err'])
def groupadd(self, name, gid=None, logerr=True, level=logging.INFOCLI2):
"""
Add a group
"""
self.logger.info('adding group ' + str(name))
cmd = ['groupadd']
if gid is not None:
cmd += ['-g', str(gid)]
cmd += [str(name)]
ret = self.run_cmd(cmd=cmd, sudo=True, logerr=False, level=level)
if ((ret['rc'] != 0) and logerr):
raise PtlUtilError(rc=ret['rc'], rv=False, msg=ret['err'])
def groupdel(self, name, logerr=True, level=logging.INFOCLI2):
self.logger.info('deleting group ' + str(name))
cmd = ['groupdel', str(name)]
ret = self.run_cmd(cmd=cmd, sudo=True, logerr=logerr, level=level)
if ((ret['rc'] != 0) and logerr):
raise PtlUtilError(rc=ret['rc'], rv=False, msg=ret['err'])
def create_temp_file(self, hostname=None, suffix='', prefix='PtlPbs',
dirname=None, text=False, asuser=None, body=None,
level=logging.INFOCLI2):
"""
Create a temp file by calling tempfile.mkstemp
:param hostname: the hostname on which to query tempdir from
:type hostname: str or None
:param suffix: the file name will end with this suffix
:type suffix: str
:param prefix: the file name will begin with this prefix
:type prefix: str
:param dirname: the file will be created in this directory
:type dirname: str or None
:param text: the file is opened in text mode is this is true
else in binary mode
:type text: boolean
:param asuser: Optional username or uid of temp file owner
:type asuser: str or None
:param body: Optional content to write to the temporary file
:type body: str or None
:param level: logging level, defaults to INFOCLI2
:type level: int
"""
# create a temp file as current user
(fd, tmpfile) = tempfile.mkstemp(suffix, prefix, dirname, text)
# write user provided contents to file
if body is not None:
if isinstance(body, list):
os.write(fd, "\n".join(body))
else:
os.write(fd, body)
os.close(fd)
# if temp file to be created on remote host
if not self.is_localhost(hostname):
if asuser is not None:
# by default mkstemp creates file with 0600 permission
# to create file as different user first change the file
# permission to 0644 so that other user has read permission
self.chmod(hostname, tmpfile, mode=0644)
# copy temp file created on local host to remote host
# as different user
self.run_copy(hostname, tmpfile, tmpfile, runas=asuser,
preserve_permission=False, level=level)
else:
# copy temp file created on localhost to remote as current user
self.run_copy(hostname, tmpfile, tmpfile,
preserve_permission=False, level=level)
# remove local temp file
os.unlink(tmpfile)
if asuser is not None:
# by default mkstemp creates file with 0600 permission
# to create file as different user first change the file
# permission to 0644 so that other user has read permission
self.chmod(hostname, tmpfile, mode=0644)
# since we need to create as differnt user than current user
# create a temp file just to get temp file name with absolute path
(_, tmpfile2) = tempfile.mkstemp(suffix, prefix, dirname, text)
# remove the newly created temp file
os.unlink(tmpfile2)
# copy the orginal temp as new temp file
self.run_copy(hostname, tmpfile, tmpfile2, runas=asuser,
preserve_permission=False, level=level)
# remove original temp file
os.unlink(tmpfile)
return tmpfile2
return tmpfile
def mkdtemp(self, hostname=None, suffix='', prefix='PtlPbs', dir=None,
uid=None, gid=None, mode=None, level=logging.INFOCLI2):
"""
Create a temp dir by calling ``tempfile.mkdtemp``
:param hostname: the hostname on which to query tempdir from
:type hostname: str or None
:param suffix: the directory name will end with this suffix
:type suffix: str
:param prefix: the directory name will begin with this prefix
:type prefix: str
:param dir: the directory will be created in this directory
:type dir: str or None
:param uid: Optional username or uid of temp directory owner
:param gid: Optional group name or gid of temp directory
group owner
:param mode: Optional mode bits to assign to the temporary
directory
:param level: logging level, defaults to INFOCLI2
"""
if not self.is_localhost(hostname):
tmp_args = []
if suffix:
tmp_args += ['suffix=\'' + suffix + '\'']
if prefix:
tmp_args += ['prefix=\'' + prefix + '\'']
if dir is not None:
tmp_args += ['dir=\'' + str(dir) + '\'']
args = ",".join(tmp_args)
ret = self.run_cmd(hostname,
['python', '-c', '"import tempfile; ' +
'print tempfile.mkdtemp(' + args + ')"'],
level=level)
if ret['rc'] == 0 and ret['out']:
fn = ret['out'][0]
else:
fn = tempfile.mkdtemp(suffix, prefix, dir)
if mode is not None:
self.chmod(hostname, fn, mode=mode, recursive=True, level=level,
sudo=True)
if ((uid is not None) or (gid is not None)):
self.chown(hostname, fn, uid=uid, gid=gid, recursive=True,
sudo=True)
return fn
def parse_strace(self, lines):
"""
strace parsing. Just the regular expressions for now
"""
timestamp_pat = r'(^(\d{2}:\d{2}:\d{2})(.\d+){0,1} |^(\d+.\d+) ){0,1}'
exec_pat = r'execve\(("[^"]+"), \[([^]]+)\], [^,]+ = (\d+)$'
timestamp_exec_re = re.compile(timestamp_pat + exec_pat)
for line in lines:
m = timestamp_exec_re.match(line)
if m:
print line