CRAB/python/crab_util.py

###########################################################################
#
#   C O N V E N I E N C E    F U N C T I O N S
#
###########################################################################

import string, sys, os, time
import ConfigParser, re, popen2, select, fcntl

import common
from crab_exceptions import CrabException
from ServerConfig import *

###########################################################################
def parseOptions(argv):
    """
    Parses command-line options.
    Returns a dictionary with specified options as keys:
    -opt1             --> 'opt1' : None
    -opt2 val         --> 'opt2' : 'val'
    -opt3=val         --> 'opt3' : 'val'
    Usually called as
    options = parseOptions(sys.argv[1:])
    """
    options = {}
    argc = len(argv)
    i = 0
    while ( i < argc ):
        if argv[i][0] != '-':
            i = i + 1
            continue
        eq = string.find(argv[i], '=')
        if eq > 0 :
            opt = argv[i][:eq]
            val = argv[i][eq+1:]
            pass
        else:
            opt = argv[i]
            val = None
            if ( i+1 < argc and argv[i+1][0] != '-' ):
                i = i + 1
                val = argv[i]
                pass
            pass
        options[opt] = val
        i = i + 1
        pass
    return options

def loadConfig(file, config):
    """
    returns a dictionary with keys of the form
    <section>.<option> and the corresponding values
    """
    #config={}
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        # print 'Section',sec
        for opt in cp.options(sec):
            #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
            config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
    return config

###########################################################################
def isInt(str):
    """ Is the given string an integer ?"""
    try: int(str)
    except ValueError: return 0
    return 1

###########################################################################
def isBool(str):
    """ Is the given string 0 or 1 ?"""
    if (str in ('0','1')): return 1
    return 0

###########################################################################
def parseRange(range):
    """
    Takes as the input a string with two integers separated by
    the minus sign and returns the tuple with these numbers:
    'n1-n2' -> (n1, n2)
    'n1'    -> (n1, n1)
    """
    start = None
    end   = None
    minus = string.find(range, '-')
    if ( minus < 0 ):
        if isInt(range):
            start = int(range)
            end   = start
            pass
        pass
    else:
        if isInt(range[:minus]) and isInt(range[minus+1:]):
            start = int(range[:minus])
            end   = int(range[minus+1:])
            pass
        pass
    return (start, end)

###########################################################################
def parseRange2(range):
    """
    Takes as the input a string in the form of a comma-separated
    numbers and ranges
    and returns a list with all specified numbers:
    'n1'          -> [n1]
    'n1-n2'       -> [n1, n1+1, ..., n2]
    'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
    """
    result = []
    if not range: return result

    comma = string.find(range, ',')
    if comma == -1: left = range
    else:           left = range[:comma]

    (n1, n2) = parseRange(left)
    while ( n1 <= n2 ):
        try:
            result.append(n1)
            n1 += 1
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    if comma != -1:
        try:
            result.extend(parseRange2(range[comma+1:]))
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    return result

###########################################################################
def findLastWorkDir(dir_prefix, where = None):

    if not where: where = os.getcwd() + '/'
    # dir_prefix usually has the form 'crab_0_'
    pattern = re.compile(dir_prefix)

    file_list = [f for f in os.listdir(where) if os.path.isdir(f) and pattern.match(f)]

    if len(file_list) == 0: return None

    file_list.sort()

    wdir = where + file_list[-1]
    return wdir

###########################################################################
def importName(module_name, name):
    """
    Import a named object from a Python module,
    i.e., it is an equivalent of 'from module_name import name'.
    """
    module = __import__(module_name, globals(), locals(), [name])
    return vars(module)[name]

###########################################################################
def readable(fd):
    return bool(select.select([fd], [], [], 0))

###########################################################################
def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

###########################################################################
def runCommand(cmd, printout=0, timeout=-1):
    """
    Run command 'cmd'.
    Returns command stdoutput+stderror string on success,
    or None if an error occurred.
    Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
    """

    if printout:
        common.logger.message(cmd)
    else:
        common.logger.debug(10,cmd)
        common.logger.write(cmd)
        pass

    child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = []
    errdata = []
    outeof = erreof = 0

    if timeout > 0 :
        maxwaittime = time.time() + timeout

    err = -1
    while (timeout == -1 or time.time() < maxwaittime):
        ready = select.select([outfd,errfd],[],[]) # wait for input
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '': outeof = 1
            outdata.append(outchunk)
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '': erreof = 1
            errdata.append(errchunk)
        if outeof and erreof:
            err = child.wait()
            break
        select.select([],[],[],.01) # give a little time for buffers to fill
    if err == -1:
        # kill the pid
        common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
        os.kill (child.pid, 9)
        err = child.wait()

    cmd_out = string.join(outdata,"")
    cmd_err = string.join(errdata,"")

    if err:
        common.logger.message('`'+cmd+'`\n   failed with exit code '
                           +`err`+'='+`(err&0xff)`+'(signal)+'
                           +`(err>>8)`+'(status)')
        common.logger.message(cmd_out)
        common.logger.message(cmd_err)
        return None

#    cmd_out = string.join(outdata,"")
#    cmd_err = string.join(errdata,"")
    cmd_out = cmd_out + cmd_err
    if printout:
        common.logger.message(cmd_out)
    else:
        common.logger.debug(10,cmd_out)
        common.logger.write(cmd_out)
        pass
    #print "<"+cmd_out+">"
    return cmd_out

####################################
def makeCksum(filename) :
    """
    make check sum using filename and content of file
    """

    from zlib import crc32
    hashString = filename

    inFile = open(filename, 'r')
    hashString += inFile.read()
    inFile.close()

    cksum = str(crc32(hashString))
    return cksum


def spanRanges(jobArray):
    """
    take array of job numbers and concatenate 1,2,3 to 1-3
    return string
    """

    output = ""
    jobArray.sort()

    previous = jobArray[0]-1
    for job in jobArray:
        if previous+1 == job:
            previous = job
            if len(output) > 0 :
                if output[-1] != "-":
                    output += "-"
            else :
                output += str(previous)
        else:
            output += str(previous) + "," + str(job)
            #output += "," + str(job)
            previous = job
    if len(jobArray) > 1 :
        output += str(previous)

    return output

def displayReport(self, header, lines, xml=''):

    counter = 0
    printline = ''
    printline+= header
    print printline
    print '---------------------------------------------------------------------------------------------------'

    for i in range(len(lines)):
        if counter != 0 and counter%10 == 0 :
            print '---------------------------------------------------------------------------------------------------'
        print lines[i]
        counter += 1
    if xml != '' :
        fileName = common.work_space.shareDir() + xml
        task = common._db.getTask()
        taskXML = common._db.serializeTask(task)
        common.logger.debug(5, taskXML)
        f = open(fileName, 'w')
        f.write(taskXML)
        f.close()
        pass

def CliServerParams(self):
    """
    Init client-server interactions
    """
    self.srvCfg = {}
    ## First I have to check if the decision has been already taken...
    task = common._db.getTask()
    if task['serverName']!=None:
        self.cfg_params['CRAB.server_name']=task['serverName']
    if self.cfg_params.has_key('CRAB.use_server'):
        print 'CRAB.use_server',self.cfg_params.get('CRAB.use_server')
        serverName=self.cfg_params.get('CRAB.server_name','default')
        print serverName
        if self.cfg_params.has_key('CRAB.server_name'):
            serverName=self.cfg_params['CRAB.server_name']
        else:
            serverName='default'
        self.srvCfg = ServerConfig(serverName).config()
        # save the serverName for future use
        opsToBeSaved={}
        opsToBeSaved['serverName']=self.srvCfg['serverGenericName']
        common._db.updateTask_(opsToBeSaved)
    elif self.cfg_params.has_key('CRAB.server_name'):
        self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()
    else:
        msg = 'No server selected or port specified.\n'
        msg += 'Please specify a server in the crab cfg file'
        raise CrabException(msg)
        return

    self.server_admin = str(self.srvCfg['serverAdmin'])
    self.server_dn = str(self.srvCfg['serverDN'])

    self.server_name = str(self.srvCfg['serverName'])
    self.server_port = int(self.srvCfg['serverPort'])

    self.storage_name = str(self.srvCfg['storageName'])
    self.storage_path = str(self.srvCfg['storagePath'])
    self.storage_proto = str(self.srvCfg['storageProtocol'])
    self.storage_port = str(self.srvCfg['storagePort'])

def bulkControl(self,list):
    """
    Check the BULK size and  reduce collection ...if needed
    """
    max_size = 400
    sub_bulk = []
    if len(list) > int(max_size):
        n_sub_bulk = int( int(len(list) ) / int(max_size) )
        for n in xrange(n_sub_bulk):
            first =n*int(max_size)
            last = (n+1)*int(max_size)
            sub_bulk.append(list[first:last])
        if len(list[last:-1]) < 50:
            for pp in list[last:-1]:
                sub_bulk[n_sub_bulk-1].append(pp)
        else:
            sub_bulk.append(list[last:])
    else:
        sub_bulk.append(list)

    return sub_bulk
###########################################################################

def getUserName():
    """
    extract user name from either SiteDB or Unix
    """
    if common.scheduler.name().upper() in ['LSF', 'CAF']:
       common.logger.debug(10,"Using as username the Unix user name")
       UserName=UnixUserName()
    else :
       UserName=gethnUserNameFromSiteDB() 

    return UserName

def UnixUserName():
    """
    extract username from whoami
    """
    try:
        UserName = runCommand("whoami")
        UserName = string.strip(UserName)
    except:
        msg = "Error. Problem with whoami command"
        raise CrabException(msg)
    return UserName

def getDN():
    """
    extract DN from user proxy's identity
    """
    try:
        userdn = runCommand("voms-proxy-info -identity")
        userdn = string.strip(userdn)
        #search for a / to avoid picking up warning messages
        userdn = userdn[userdn.find('/'):]
    except:
        msg = "Error. Problem with voms-proxy-info -identity command"
        raise CrabException(msg)
    return userdn.split('\n')[0]

def gethnUserNameFromSiteDB():
    """
    extract user name from SiteDB
    """
    from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
    hnUserName = None
    userdn = getDN()
    dict={ 'cacheduration' : 24 } 
    mySiteDB = SiteDBJSON(dict)
    msg_ = "there is no user name associated to DN %s in SiteDB. You need to register in SiteDB with the instructions at https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB" % userdn
    try:
        hnUserName = mySiteDB.dnUserName(dn=userdn)
    except:
        msg = "Error. Problem extracting user name from SiteDB"
        msg += "\n Check that you are registered in SiteDB, see https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB\n"
        msg += 'or %s'%msg_
        raise CrabException(msg)
    if not hnUserName:
        msg = "Error. %s"%msg_
        raise CrabException(msg)
    return hnUserName

###################################################################33

def numberFile(file, txt):
    """
    append _'txt' before last extension of a file
    """
    txt=str(txt)
    p = string.split(file,".")
    # take away last extension
    name = p[0]
    for x in p[1:-1]:
        name=name+"."+x
    # add "_txt"
    if len(p)>1:
        ext = p[len(p)-1]
        result = name + '_' + txt + "." + ext
    else:
        result = name + '_' + txt

    return result

def readTXTfile(self,inFileName):
    """
    read file and return a list with the content
    """
    out_list=[]
    if os.path.exists(inFileName):
        f = open(inFileName, 'r')
        for line in  f.readlines():
            out_list.append(string.strip(line))
        f.close()
    else:
        msg = ' file '+str(inFileName)+' not found.'
        raise CrabException(msg)
    return out_list

def writeTXTfile(self, outFileName, args):
    """
    write a file with the given content ( args )
    """
    outFile = open(outFileName,"a")
    outFile.write(str(args))
    outFile.close()
    return

def readableList(self,rawList):
    """
    Turn a list of numbers into a string like 1-5,7,9,12-20
    """
    if not rawList:
        return ''

    listString = str(rawList[0])
    endRange = ''
    for i in range(1,len(rawList)):
        if rawList[i] == rawList[i-1]+1:
            endRange = str(rawList[i])
        else:
            if endRange:
                listString += '-' + endRange + ',' + str(rawList[i])
                endRange = ''
            else:
                listString += ',' + str(rawList[i])
    if endRange:
        listString += '-' + endRange
        endRange = ''

    return listString

def getLocalDomain(self):
    """
    Get local domain name
    """
    import socket
    tmp=socket.gethostname()
    dot=string.find(tmp,'.')
    if (dot==-1):
        msg='Unkown domain name. Cannot use local scheduler'
        raise CrabException(msg)
    localDomainName = string.split(tmp,'.',1)[-1]
    return localDomainName

#######################################################
# Brian Bockelman bbockelm@cse.unl.edu
# Module to check the avaialble disk space on a specified directory.
#
import os
import statvfs

def has_freespace(dir_name, needed_space_kilobytes):
     enough_unix_quota = False
     enough_quota = False
     enough_partition = False
     enough_mount = False
     try:
         enough_mount = check_mount(dir_name, need_space_kilobytes)
     except:
         enough_mount = True
     try:
         enough_quota = check_quota(dir_name, needed_space_kilobytes)
     except:
         raise
         enough_quota = True
     try:
         enough_partition = check_partition(dir_name,
             needed_space_kilobytes)
     except:
         enough_partition = True
     try:
         enough_unix_quota = check_unix_quota(dir_name,
             needed_space_kilobytes)
     except:
         enough_unix_quota = True
     return enough_mount and enough_quota and enough_partition \
         and enough_unix_quota

def check_mount(dir_name, needed_space_kilobytes):
     try:
         vfs = os.statvfs(dir_name)
     except:
         raise Exception("Unable to query VFS for %s." % dir_name)
     dev_free = vfs[statvfs.F_FRSIZE] * vfs[statvfs.F_BAVAIL]
     return dev_free/1024 > needed_space_kilobytes

def check_quota(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("/usr/bin/fs lq %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the file system quota!")
     try:
         results = results.split('\n')[1].split()
         quota, used = results[1:3]
         avail = int(quota) - int(used)
         return avail > needed_space_kilobytes
     except:
         return Exception("Unable to parse AFS output.")

def check_partition(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("/usr/bin/fs diskfree %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the file system quota!")
     try:
         results = results.split('\n')[1].split()
         avail = results[3]
         return int(avail) > needed_space_kilobytes
     except:
         raise Exception("Unable to parse AFS output.")

def check_unix_quota(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("df %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the filesystem with df.")
     fs = results.split('\n')[1].split()[0]
     _, fd, _ = os.popen3("quota -Q -u -g")
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the quotas.")
     has_info = False
     for line in results.splitlines():
         info = line.split()
         if info[0] in ['Filesystem', 'Disk']:
             continue
         if len(info) == 1:
             filesystem = info[0]
             has_info = False
         if len(info) == 6:
             used, limit = info[0], info[2]
             has_info = True
         if len(info) == 7:
             filesystem, used, limit = info[0], info[1], info[3]
             has_info = True
         if has_info:
            if filesystem != fs:
                continue
            avail = int(limit) - int(used)
            if avail < needed_space_kilobytes:
                return False
     return True

def getGZSize(gzipfile):
    # return the uncompressed size of a gzipped file
    import struct
    f = open(gzipfile, "rb")
    if f.read(2) != "\x1f\x8b":
        raise IOError("not a gzip file")
    f.seek(-4, 2)
    return struct.unpack("<i", f.read())[0]

def showWebMon(server_name):
    taskName = common._db.queryTask('name')
    msg = ''
    msg +='You can also follow the status of this task on :\n'
    msg +='\tCMS Dashboard: http://dashb-cms-job-task.cern.ch/taskmon.html#task=%s\n'%(taskName)
    if server_name != '' :
        msg += '\tServer page: http://%s:8888/logginfo\n'%server_name
    msg += '\tYour task name is: %s \n'%taskName
    return msg


####################################
if __name__ == '__main__':
    print 'sys.argv[1] =',sys.argv[1]
    list = parseRange2(sys.argv[1])
    print list
    cksum = makeCksum("crab_util.py")
    print cksum

Revision:	1.77
Committed:	Tue Apr 28 17:33:40 2009 UTC (16 years ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_6_0_pre2, CRAB_2_6_0_pre1
Changes since 1.76:	+3 -0 lines
Log Message:	check client version in available server list
#	User	Rev	Content
1	nsmirnov	1.1	###########################################################################
2			#
3			# C O N V E N I E N C E F U N C T I O N S
4			#
5			###########################################################################
6
7	slacapra	1.12	import string, sys, os, time
8	slacapra	1.16	import ConfigParser, re, popen2, select, fcntl
9	nsmirnov	1.1
10	nsmirnov	1.2	import common
11	slacapra	1.30	from crab_exceptions import CrabException
12	spiga	1.42	from ServerConfig import *
13	nsmirnov	1.1
14			###########################################################################
15			def parseOptions(argv):
16			"""
17			Parses command-line options.
18			Returns a dictionary with specified options as keys:
19			-opt1 --> 'opt1' : None
20			-opt2 val --> 'opt2' : 'val'
21			-opt3=val --> 'opt3' : 'val'
22			Usually called as
23			options = parseOptions(sys.argv[1:])
24			"""
25			options = {}
26			argc = len(argv)
27			i = 0
28			while ( i < argc ):
29			if argv[i][0] != '-':
30			i = i + 1
31			continue
32			eq = string.find(argv[i], '=')
33			if eq > 0 :
34			opt = argv[i][:eq]
35			val = argv[i][eq+1:]
36			pass
37			else:
38			opt = argv[i]
39			val = None
40			if ( i+1 < argc and argv[i+1][0] != '-' ):
41			i = i + 1
42			val = argv[i]
43			pass
44			pass
45			options[opt] = val
46			i = i + 1
47			pass
48			return options
49
50	slacapra	1.47	def loadConfig(file, config):
51	nsmirnov	1.1	"""
52			returns a dictionary with keys of the form
53			<section>.<option> and the corresponding values
54			"""
55	slacapra	1.47	#config={}
56	nsmirnov	1.1	cp = ConfigParser.ConfigParser()
57			cp.read(file)
58			for sec in cp.sections():
59			# print 'Section',sec
60			for opt in cp.options(sec):
61			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
62			config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
63			return config
64
65			###########################################################################
66			def isInt(str):
67			""" Is the given string an integer ?"""
68			try: int(str)
69			except ValueError: return 0
70			return 1
71
72			###########################################################################
73			def isBool(str):
74			""" Is the given string 0 or 1 ?"""
75			if (str in ('0','1')): return 1
76			return 0
77
78			###########################################################################
79	nsmirnov	1.3	def parseRange(range):
80			"""
81			Takes as the input a string with two integers separated by
82			the minus sign and returns the tuple with these numbers:
83			'n1-n2' -> (n1, n2)
84			'n1' -> (n1, n1)
85			"""
86			start = None
87			end = None
88			minus = string.find(range, '-')
89			if ( minus < 0 ):
90			if isInt(range):
91			start = int(range)
92			end = start
93			pass
94			pass
95			else:
96			if isInt(range[:minus]) and isInt(range[minus+1:]):
97			start = int(range[:minus])
98			end = int(range[minus+1:])
99			pass
100			pass
101			return (start, end)
102
103			###########################################################################
104	nsmirnov	1.4	def parseRange2(range):
105			"""
106			Takes as the input a string in the form of a comma-separated
107			numbers and ranges
108			and returns a list with all specified numbers:
109			'n1' -> [n1]
110			'n1-n2' -> [n1, n1+1, ..., n2]
111			'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
112			"""
113	slacapra	1.30	result = []
114			if not range: return result
115	nsmirnov	1.4
116			comma = string.find(range, ',')
117			if comma == -1: left = range
118			else: left = range[:comma]
119
120			(n1, n2) = parseRange(left)
121			while ( n1 <= n2 ):
122	slacapra	1.11	try:
123	slacapra	1.30	result.append(n1)
124	slacapra	1.11	n1 += 1
125			pass
126			except:
127			msg = 'Syntax error in range <'+range+'>'
128			raise CrabException(msg)
129	nsmirnov	1.4
130			if comma != -1:
131	slacapra	1.11	try:
132	slacapra	1.30	result.extend(parseRange2(range[comma+1:]))
133	slacapra	1.11	pass
134			except:
135			msg = 'Syntax error in range <'+range+'>'
136			raise CrabException(msg)
137	nsmirnov	1.4
138	slacapra	1.30	return result
139	nsmirnov	1.4
140			###########################################################################
141	nsmirnov	1.1	def findLastWorkDir(dir_prefix, where = None):
142
143			if not where: where = os.getcwd() + '/'
144			# dir_prefix usually has the form 'crab_0_'
145			pattern = re.compile(dir_prefix)
146
147	slacapra	1.76	file_list = [f for f in os.listdir(where) if os.path.isdir(f) and pattern.match(f)]
148	nsmirnov	1.1
149			if len(file_list) == 0: return None
150
151			file_list.sort()
152
153	slacapra	1.76	wdir = where + file_list[-1]
154	nsmirnov	1.1	return wdir
155
156			###########################################################################
157			def importName(module_name, name):
158			"""
159			Import a named object from a Python module,
160			i.e., it is an equivalent of 'from module_name import name'.
161			"""
162			module = __import__(module_name, globals(), locals(), [name])
163			return vars(module)[name]
164
165			###########################################################################
166	slacapra	1.16	def readable(fd):
167	ewv	1.44	return bool(select.select([fd], [], [], 0))
168	slacapra	1.16
169			###########################################################################
170			def makeNonBlocking(fd):
171			fl = fcntl.fcntl(fd, fcntl.F_GETFL)
172			try:
173			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.O_NDELAY)
174			except AttributeError:
175			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.FNDELAY)
176
177			###########################################################################
178	slacapra	1.11	def runCommand(cmd, printout=0, timeout=-1):
179	nsmirnov	1.1	"""
180			Run command 'cmd'.
181			Returns command stdoutput+stderror string on success,
182			or None if an error occurred.
183	slacapra	1.16	Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
184	nsmirnov	1.1	"""
185	fanzago	1.15
186	slacapra	1.11	if printout:
187			common.logger.message(cmd)
188			else:
189	slacapra	1.13	common.logger.debug(10,cmd)
190	slacapra	1.11	common.logger.write(cmd)
191	slacapra	1.29	pass
192	slacapra	1.11
193	slacapra	1.16	child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
194			child.tochild.close() # don't need to talk to child
195	ewv	1.44	outfile = child.fromchild
196	slacapra	1.16	outfd = outfile.fileno()
197			errfile = child.childerr
198			errfd = errfile.fileno()
199			makeNonBlocking(outfd) # don't deadlock!
200			makeNonBlocking(errfd)
201			outdata = []
202			errdata = []
203			outeof = erreof = 0
204
205			if timeout > 0 :
206			maxwaittime = time.time() + timeout
207
208			err = -1
209			while (timeout == -1 or time.time() < maxwaittime):
210			ready = select.select([outfd,errfd],[],[]) # wait for input
211			if outfd in ready[0]:
212			outchunk = outfile.read()
213			if outchunk == '': outeof = 1
214			outdata.append(outchunk)
215			if errfd in ready[0]:
216			errchunk = errfile.read()
217			if errchunk == '': erreof = 1
218			errdata.append(errchunk)
219			if outeof and erreof:
220			err = child.wait()
221			break
222	slacapra	1.21	select.select([],[],[],.01) # give a little time for buffers to fill
223	slacapra	1.16	if err == -1:
224			# kill the pid
225	slacapra	1.18	common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
226	slacapra	1.16	os.kill (child.pid, 9)
227	slacapra	1.11	err = child.wait()
228	slacapra	1.16
229	fanzago	1.17	cmd_out = string.join(outdata,"")
230			cmd_err = string.join(errdata,"")
231
232	slacapra	1.11	if err:
233			common.logger.message('`'+cmd+'`\n failed with exit code '
234			+`err`+'='+`(err&0xff)`+'(signal)+'
235			+`(err>>8)`+'(status)')
236			common.logger.message(cmd_out)
237			common.logger.message(cmd_err)
238	nsmirnov	1.1	return None
239
240	fanzago	1.17	# cmd_out = string.join(outdata,"")
241			# cmd_err = string.join(errdata,"")
242	nsmirnov	1.1	cmd_out = cmd_out + cmd_err
243	slacapra	1.11	if printout:
244			common.logger.message(cmd_out)
245			else:
246	slacapra	1.13	common.logger.debug(10,cmd_out)
247	slacapra	1.11	common.logger.write(cmd_out)
248	slacapra	1.29	pass
249	slacapra	1.16	#print "<"+cmd_out+">"
250	nsmirnov	1.1	return cmd_out
251	nsmirnov	1.4
252	slacapra	1.11	####################################
253	gutsche	1.20	def makeCksum(filename) :
254			"""
255	ewv	1.44	make check sum using filename and content of file
256	gutsche	1.20	"""
257
258	ewv	1.44	from zlib import crc32
259			hashString = filename
260	gutsche	1.20
261	ewv	1.44	inFile = open(filename, 'r')
262			hashString += inFile.read()
263			inFile.close()
264	gutsche	1.20
265	ewv	1.44	cksum = str(crc32(hashString))
266	gutsche	1.20	return cksum
267
268	ewv	1.52
269	gutsche	1.32	def spanRanges(jobArray):
270			"""
271			take array of job numbers and concatenate 1,2,3 to 1-3
272			return string
273			"""
274
275			output = ""
276	mcinquil	1.35	jobArray.sort()
277	ewv	1.44
278	gutsche	1.32	previous = jobArray[0]-1
279			for job in jobArray:
280			if previous+1 == job:
281			previous = job
282			if len(output) > 0 :
283			if output[-1] != "-":
284			output += "-"
285			else :
286			output += str(previous)
287			else:
288			output += str(previous) + "," + str(job)
289	mcinquil	1.35	#output += "," + str(job)
290	gutsche	1.32	previous = job
291			if len(jobArray) > 1 :
292			output += str(previous)
293
294			return output
295
296	spiga	1.40	def displayReport(self, header, lines, xml=''):
297	ewv	1.44
298	spiga	1.49	counter = 0
299			printline = ''
300			printline+= header
301			print printline
302			print '---------------------------------------------------------------------------------------------------'
303
304			for i in range(len(lines)):
305			if counter != 0 and counter%10 == 0 :
306			print '---------------------------------------------------------------------------------------------------'
307			print lines[i]
308			counter += 1
309			if xml != '' :
310	spiga	1.40	fileName = common.work_space.shareDir() + xml
311			task = common._db.getTask()
312			taskXML = common._db.serializeTask(task)
313			common.logger.debug(5, taskXML)
314			f = open(fileName, 'w')
315			f.write(taskXML)
316			f.close()
317			pass
318	spiga	1.39
319	spiga	1.42	def CliServerParams(self):
320	slacapra	1.45	"""
321			Init client-server interactions
322			"""
323			self.srvCfg = {}
324	slacapra	1.63	## First I have to check if the decision has been already taken...
325			task = common._db.getTask()
326			if task['serverName']!=None:
327			self.cfg_params['CRAB.server_name']=task['serverName']
328	slacapra	1.61	if self.cfg_params.has_key('CRAB.use_server'):
329	slacapra	1.77	print 'CRAB.use_server',self.cfg_params.get('CRAB.use_server')
330			serverName=self.cfg_params.get('CRAB.server_name','default')
331			print serverName
332	slacapra	1.62	if self.cfg_params.has_key('CRAB.server_name'):
333			serverName=self.cfg_params['CRAB.server_name']
334			else:
335			serverName='default'
336			self.srvCfg = ServerConfig(serverName).config()
337	slacapra	1.63	# save the serverName for future use
338			opsToBeSaved={}
339			opsToBeSaved['serverName']=self.srvCfg['serverGenericName']
340			common._db.updateTask_(opsToBeSaved)
341	slacapra	1.61	elif self.cfg_params.has_key('CRAB.server_name'):
342	slacapra	1.45	self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()
343	slacapra	1.61	else:
344			msg = 'No server selected or port specified.\n'
345			msg += 'Please specify a server in the crab cfg file'
346			raise CrabException(msg)
347			return
348	slacapra	1.45
349	slacapra	1.61	self.server_admin = str(self.srvCfg['serverAdmin'])
350			self.server_dn = str(self.srvCfg['serverDN'])
351	spiga	1.58
352	slacapra	1.61	self.server_name = str(self.srvCfg['serverName'])
353			self.server_port = int(self.srvCfg['serverPort'])
354	slacapra	1.45
355	slacapra	1.61	self.storage_name = str(self.srvCfg['storageName'])
356			self.storage_path = str(self.srvCfg['storagePath'])
357			self.storage_proto = str(self.srvCfg['storageProtocol'])
358			self.storage_port = str(self.srvCfg['storagePort'])
359	spiga	1.39
360	ewv	1.44	def bulkControl(self,list):
361	slacapra	1.45	"""
362			Check the BULK size and reduce collection ...if needed
363			"""
364			max_size = 400
365			sub_bulk = []
366			if len(list) > int(max_size):
367			n_sub_bulk = int( int(len(list) ) / int(max_size) )
368	mcinquil	1.53	for n in xrange(n_sub_bulk):
369	slacapra	1.45	first =n*int(max_size)
370			last = (n+1)*int(max_size)
371			sub_bulk.append(list[first:last])
372			if len(list[last:-1]) < 50:
373			for pp in list[last:-1]:
374			sub_bulk[n_sub_bulk-1].append(pp)
375	spiga	1.43	else:
376	mcinquil	1.53	sub_bulk.append(list[last:])
377	slacapra	1.45	else:
378			sub_bulk.append(list)
379
380			return sub_bulk
381	spiga	1.66	###########################################################################
382	slacapra	1.45
383	spiga	1.69	def getUserName():
384	spiga	1.66	"""
385			extract user name from either SiteDB or Unix
386			"""
387			if common.scheduler.name().upper() in ['LSF', 'CAF']:
388			common.logger.debug(10,"Using as username the Unix user name")
389	spiga	1.70	UserName=UnixUserName()
390	spiga	1.66	else :
391	spiga	1.74	UserName=gethnUserNameFromSiteDB()
392	spiga	1.66
393			return UserName
394	spiga	1.58
395	spiga	1.69	def UnixUserName():
396	spiga	1.58	"""
397			extract username from whoami
398			"""
399			try:
400			UserName = runCommand("whoami")
401			UserName = string.strip(UserName)
402			except:
403			msg = "Error. Problem with whoami command"
404			raise CrabException(msg)
405			return UserName
406	spiga	1.66
407	spiga	1.69	def getDN():
408	spiga	1.66	"""
409			extract DN from user proxy's identity
410			"""
411			try:
412			userdn = runCommand("voms-proxy-info -identity")
413			userdn = string.strip(userdn)
414			#search for a / to avoid picking up warning messages
415			userdn = userdn[userdn.find('/'):]
416			except:
417			msg = "Error. Problem with voms-proxy-info -identity command"
418			raise CrabException(msg)
419			return userdn.split('\n')[0]
420
421	spiga	1.69	def gethnUserNameFromSiteDB():
422	spiga	1.66	"""
423			extract user name from SiteDB
424			"""
425			from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
426			hnUserName = None
427	spiga	1.69	userdn = getDN()
428	spiga	1.75	dict={ 'cacheduration' : 24 }
429			mySiteDB = SiteDBJSON(dict)
430	spiga	1.73	msg_ = "there is no user name associated to DN %s in SiteDB. You need to register in SiteDB with the instructions at https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB" % userdn
431	spiga	1.66	try:
432			hnUserName = mySiteDB.dnUserName(dn=userdn)
433			except:
434			msg = "Error. Problem extracting user name from SiteDB"
435	spiga	1.73	msg += "\n Check that you are registered in SiteDB, see https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB\n"
436			msg += 'or %s'%msg_
437	spiga	1.66	raise CrabException(msg)
438			if not hnUserName:
439	spiga	1.73	msg = "Error. %s"%msg_
440	spiga	1.66	raise CrabException(msg)
441			return hnUserName
442
443			###################################################################33
444
445	slacapra	1.45	def numberFile(file, txt):
446			"""
447			append _'txt' before last extension of a file
448			"""
449			txt=str(txt)
450			p = string.split(file,".")
451			# take away last extension
452			name = p[0]
453			for x in p[1:-1]:
454			name=name+"."+x
455			# add "_txt"
456			if len(p)>1:
457			ext = p[len(p)-1]
458			result = name + '_' + txt + "." + ext
459			else:
460			result = name + '_' + txt
461	spiga	1.43
462	slacapra	1.45	return result
463	spiga	1.46
464			def readTXTfile(self,inFileName):
465			"""
466			read file and return a list with the content
467			"""
468			out_list=[]
469			if os.path.exists(inFileName):
470			f = open(inFileName, 'r')
471			for line in f.readlines():
472	ewv	1.52	out_list.append(string.strip(line))
473	spiga	1.46	f.close()
474			else:
475			msg = ' file '+str(inFileName)+' not found.'
476	ewv	1.52	raise CrabException(msg)
477	spiga	1.46	return out_list
478
479			def writeTXTfile(self, outFileName, args):
480			"""
481			write a file with the given content ( args )
482			"""
483			outFile = open(outFileName,"a")
484			outFile.write(str(args))
485			outFile.close()
486			return
487
488	spiga	1.54	def readableList(self,rawList):
489	ewv	1.64	"""
490			Turn a list of numbers into a string like 1-5,7,9,12-20
491			"""
492			if not rawList:
493			return ''
494
495	slacapra	1.56	listString = str(rawList[0])
496			endRange = ''
497			for i in range(1,len(rawList)):
498			if rawList[i] == rawList[i-1]+1:
499			endRange = str(rawList[i])
500			else:
501			if endRange:
502			listString += '-' + endRange + ',' + str(rawList[i])
503			endRange = ''
504			else:
505			listString += ',' + str(rawList[i])
506			if endRange:
507			listString += '-' + endRange
508			endRange = ''
509	ewv	1.64
510	slacapra	1.56	return listString
511	spiga	1.54
512	spiga	1.51	def getLocalDomain(self):
513	ewv	1.52	"""
514	slacapra	1.56	Get local domain name
515	ewv	1.52	"""
516	spiga	1.51	import socket
517			tmp=socket.gethostname()
518			dot=string.find(tmp,'.')
519			if (dot==-1):
520			msg='Unkown domain name. Cannot use local scheduler'
521			raise CrabException(msg)
522			localDomainName = string.split(tmp,'.',1)[-1]
523	ewv	1.52	return localDomainName
524	spiga	1.51
525	slacapra	1.56	#######################################################
526			# Brian Bockelman bbockelm@cse.unl.edu
527			# Module to check the avaialble disk space on a specified directory.
528			#
529			import os
530			import statvfs
531
532			def has_freespace(dir_name, needed_space_kilobytes):
533			enough_unix_quota = False
534			enough_quota = False
535			enough_partition = False
536			enough_mount = False
537			try:
538			enough_mount = check_mount(dir_name, need_space_kilobytes)
539			except:
540			enough_mount = True
541			try:
542			enough_quota = check_quota(dir_name, needed_space_kilobytes)
543			except:
544			raise
545			enough_quota = True
546			try:
547			enough_partition = check_partition(dir_name,
548			needed_space_kilobytes)
549			except:
550			enough_partition = True
551			try:
552			enough_unix_quota = check_unix_quota(dir_name,
553			needed_space_kilobytes)
554			except:
555			enough_unix_quota = True
556			return enough_mount and enough_quota and enough_partition \
557			and enough_unix_quota
558
559			def check_mount(dir_name, needed_space_kilobytes):
560			try:
561			vfs = os.statvfs(dir_name)
562			except:
563			raise Exception("Unable to query VFS for %s." % dir_name)
564			dev_free = vfs[statvfs.F_FRSIZE] * vfs[statvfs.F_BAVAIL]
565			return dev_free/1024 > needed_space_kilobytes
566
567			def check_quota(dir_name, needed_space_kilobytes):
568			_, fd, _ = os.popen3("/usr/bin/fs lq %s" % dir_name)
569			results = fd.read()
570			if fd.close():
571			raise Exception("Unable to query the file system quota!")
572			try:
573			results = results.split('\n')[1].split()
574			quota, used = results[1:3]
575			avail = int(quota) - int(used)
576			return avail > needed_space_kilobytes
577			except:
578			return Exception("Unable to parse AFS output.")
579
580			def check_partition(dir_name, needed_space_kilobytes):
581			_, fd, _ = os.popen3("/usr/bin/fs diskfree %s" % dir_name)
582			results = fd.read()
583			if fd.close():
584			raise Exception("Unable to query the file system quota!")
585			try:
586			results = results.split('\n')[1].split()
587			avail = results[3]
588			return int(avail) > needed_space_kilobytes
589			except:
590			raise Exception("Unable to parse AFS output.")
591
592			def check_unix_quota(dir_name, needed_space_kilobytes):
593			_, fd, _ = os.popen3("df %s" % dir_name)
594			results = fd.read()
595			if fd.close():
596			raise Exception("Unable to query the filesystem with df.")
597			fs = results.split('\n')[1].split()[0]
598			_, fd, _ = os.popen3("quota -Q -u -g")
599			results = fd.read()
600			if fd.close():
601			raise Exception("Unable to query the quotas.")
602			has_info = False
603			for line in results.splitlines():
604			info = line.split()
605			if info[0] in ['Filesystem', 'Disk']:
606			continue
607			if len(info) == 1:
608			filesystem = info[0]
609			has_info = False
610			if len(info) == 6:
611			used, limit = info[0], info[2]
612			has_info = True
613			if len(info) == 7:
614			filesystem, used, limit = info[0], info[1], info[3]
615			has_info = True
616			if has_info:
617			if filesystem != fs:
618			continue
619			avail = int(limit) - int(used)
620			if avail < needed_space_kilobytes:
621			return False
622			return True
623	spiga	1.54
624	slacapra	1.57	def getGZSize(gzipfile):
625			# return the uncompressed size of a gzipped file
626			import struct
627			f = open(gzipfile, "rb")
628			if f.read(2) != "\x1f\x8b":
629			raise IOError("not a gzip file")
630			f.seek(-4, 2)
631	ewv	1.64	return struct.unpack("<i", f.read())[0]
632	slacapra	1.57
633	spiga	1.60	def showWebMon(server_name):
634	spiga	1.72	taskName = common._db.queryTask('name')
635	spiga	1.60	msg = ''
636	spiga	1.71	msg +='You can also follow the status of this task on :\n'
637	spiga	1.72	msg +='\tCMS Dashboard: http://dashb-cms-job-task.cern.ch/taskmon.html#task=%s\n'%(taskName)
638	ewv	1.64	if server_name != '' :
639	spiga	1.71	msg += '\tServer page: http://%s:8888/logginfo\n'%server_name
640	spiga	1.72	msg += '\tYour task name is: %s \n'%taskName
641	spiga	1.60	return msg
642
643
644	gutsche	1.20	####################################
645	nsmirnov	1.4	if __name__ == '__main__':
646			print 'sys.argv[1] =',sys.argv[1]
647			list = parseRange2(sys.argv[1])
648			print list
649	slacapra	1.29	cksum = makeCksum("crab_util.py")
650			print cksum
651	ewv	1.44