CRAB/python/crab_util.py

###########################################################################
#
#   C O N V E N I E N C E    F U N C T I O N S
#
###########################################################################

import string, sys, os, time
import ConfigParser, re, popen2, select, fcntl

import common
from crab_exceptions import CrabException
from ServerConfig import *

###########################################################################
def parseOptions(argv):
    """
    Parses command-line options.
    Returns a dictionary with specified options as keys:
    -opt1             --> 'opt1' : None
    -opt2 val         --> 'opt2' : 'val'
    -opt3=val         --> 'opt3' : 'val'
    Usually called as
    options = parseOptions(sys.argv[1:])
    """
    options = {}
    argc = len(argv)
    i = 0
    while ( i < argc ):
        if argv[i][0] != '-':
            i = i + 1
            continue
        eq = string.find(argv[i], '=')
        if eq > 0 :
            opt = argv[i][:eq]
            val = argv[i][eq+1:]
            pass
        else:
            opt = argv[i]
            val = None
            if ( i+1 < argc and argv[i+1][0] != '-' ):
                i = i + 1
                val = argv[i]
                pass
            pass
        options[opt] = val
        i = i + 1
        pass
    return options

def loadConfig(file, config):
    """
    returns a dictionary with keys of the form
    <section>.<option> and the corresponding values
    """
    #config={}
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        # print 'Section',sec
        for opt in cp.options(sec):
            #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
            config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
    return config

###########################################################################
def isInt(str):
    """ Is the given string an integer ?"""
    try: int(str)
    except ValueError: return 0
    return 1

###########################################################################
def isBool(str):
    """ Is the given string 0 or 1 ?"""
    if (str in ('0','1')): return 1
    return 0

###########################################################################
def parseRange(range):
    """
    Takes as the input a string with two integers separated by
    the minus sign and returns the tuple with these numbers:
    'n1-n2' -> (n1, n2)
    'n1'    -> (n1, n1)
    """
    start = None
    end   = None
    minus = string.find(range, '-')
    if ( minus < 0 ):
        if isInt(range):
            start = int(range)
            end   = start
            pass
        pass
    else:
        if isInt(range[:minus]) and isInt(range[minus+1:]):
            start = int(range[:minus])
            end   = int(range[minus+1:])
            pass
        pass
    return (start, end)

###########################################################################
def parseRange2(range):
    """
    Takes as the input a string in the form of a comma-separated
    numbers and ranges
    and returns a list with all specified numbers:
    'n1'          -> [n1]
    'n1-n2'       -> [n1, n1+1, ..., n2]
    'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
    """
    result = []
    if not range: return result

    comma = string.find(range, ',')
    if comma == -1: left = range
    else:           left = range[:comma]

    (n1, n2) = parseRange(left)
    while ( n1 <= n2 ):
        try:
            result.append(n1)
            n1 += 1
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    if comma != -1:
        try:
            result.extend(parseRange2(range[comma+1:]))
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    return result

###########################################################################
def findLastWorkDir(dir_prefix, where = None):

    if not where: where = os.getcwd() + '/'
    # dir_prefix usually has the form 'crab_0_'
    pattern = re.compile(dir_prefix)

    file_list = [f for f in os.listdir(where) if os.path.isdir(f) and pattern.match(f)]

    if len(file_list) == 0: return None

    file_list.sort()

    wdir = where + file_list[-1]
    return wdir

###########################################################################
def importName(module_name, name):
    """
    Import a named object from a Python module,
    i.e., it is an equivalent of 'from module_name import name'.
    """
    module = __import__(module_name, globals(), locals(), [name])
    return vars(module)[name]

###########################################################################
def readable(fd):
    return bool(select.select([fd], [], [], 0))

###########################################################################
def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

###########################################################################
def runCommand(cmd, printout=0, timeout=-1):
    """
    Run command 'cmd'.
    Returns command stdoutput+stderror string on success,
    or None if an error occurred.
    Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
    """

    if printout:
        common.logger.message(cmd)
    else:
        common.logger.debug(10,cmd)
        common.logger.write(cmd)
        pass

    child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = []
    errdata = []
    outeof = erreof = 0

    if timeout > 0 :
        maxwaittime = time.time() + timeout

    err = -1
    while (timeout == -1 or time.time() < maxwaittime):
        ready = select.select([outfd,errfd],[],[]) # wait for input
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '': outeof = 1
            outdata.append(outchunk)
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '': erreof = 1
            errdata.append(errchunk)
        if outeof and erreof:
            err = child.wait()
            break
        select.select([],[],[],.01) # give a little time for buffers to fill
    if err == -1:
        # kill the pid
        common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
        os.kill (child.pid, 9)
        err = child.wait()

    cmd_out = string.join(outdata,"")
    cmd_err = string.join(errdata,"")

    if err:
        common.logger.message('`'+cmd+'`\n   failed with exit code '
                           +`err`+'='+`(err&0xff)`+'(signal)+'
                           +`(err>>8)`+'(status)')
        common.logger.message(cmd_out)
        common.logger.message(cmd_err)
        return None

#    cmd_out = string.join(outdata,"")
#    cmd_err = string.join(errdata,"")
    cmd_out = cmd_out + cmd_err
    if printout:
        common.logger.message(cmd_out)
    else:
        common.logger.debug(10,cmd_out)
        common.logger.write(cmd_out)
        pass
    #print "<"+cmd_out+">"
    return cmd_out

####################################
def makeCksum(filename) :
    """
    make check sum using filename and content of file
    """

    from zlib import crc32
    hashString = filename

    inFile = open(filename, 'r')
    hashString += inFile.read()
    inFile.close()

    cksum = str(crc32(hashString))
    return cksum


def spanRanges(jobArray):
    """
    take array of job numbers and concatenate 1,2,3 to 1-3
    return string
    """

    output = ""
    jobArray.sort()

    previous = jobArray[0]-1
    for job in jobArray:
        if previous+1 == job:
            previous = job
            if len(output) > 0 :
                if output[-1] != "-":
                    output += "-"
            else :
                output += str(previous)
        else:
            output += str(previous) + "," + str(job)
            #output += "," + str(job)
            previous = job
    if len(jobArray) > 1 :
        output += str(previous)

    return output

def displayReport(self, header, lines, xml=''):

    counter = 0
    printline = ''
    printline+= header
    print printline
    print '---------------------------------------------------------------------------------------------------'

    for i in range(len(lines)):
        if counter != 0 and counter%10 == 0 :
            print '---------------------------------------------------------------------------------------------------'
        print lines[i]
        counter += 1
    if xml != '' :
        fileName = common.work_space.shareDir() + xml
        task = common._db.getTask()
        taskXML = common._db.serializeTask(task)
        common.logger.debug(5, taskXML)
        f = open(fileName, 'w')
        f.write(taskXML)
        f.close()
        pass

def CliServerParams(self):
    """
    Init client-server interactions
    """
    self.srvCfg = {}
    ## First I have to check if the decision has been already taken...
    task = common._db.getTask()
    if task['serverName']!=None:
        self.cfg_params['CRAB.server_name']=task['serverName']
    if self.cfg_params.has_key('CRAB.use_server'):
        if self.cfg_params.has_key('CRAB.server_name'):
            serverName=self.cfg_params['CRAB.server_name']
        else:
            serverName='default'
        self.srvCfg = ServerConfig(serverName).config()
        # save the serverName for future use
        opsToBeSaved={}
        opsToBeSaved['serverName']=self.srvCfg['serverGenericName']
        common._db.updateTask_(opsToBeSaved)
    elif self.cfg_params.has_key('CRAB.server_name'):
        self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()
    else:
        msg = 'No server selected or port specified.\n'
        msg += 'Please specify a server in the crab cfg file'
        raise CrabException(msg)
        return

    self.server_admin = str(self.srvCfg['serverAdmin'])
    self.server_dn = str(self.srvCfg['serverDN'])

    self.server_name = str(self.srvCfg['serverName'])
    self.server_port = int(self.srvCfg['serverPort'])

    self.storage_name = str(self.srvCfg['storageName'])
    self.storage_path = str(self.srvCfg['storagePath'])
    self.storage_proto = str(self.srvCfg['storageProtocol'])
    self.storage_port = str(self.srvCfg['storagePort'])

def bulkControl(self,list):
    """
    Check the BULK size and  reduce collection ...if needed
    """
    max_size = 400
    sub_bulk = []
    if len(list) > int(max_size):
        n_sub_bulk = int( int(len(list) ) / int(max_size) )
        for n in xrange(n_sub_bulk):
            first =n*int(max_size)
            last = (n+1)*int(max_size)
            sub_bulk.append(list[first:last])
        if len(list[last:-1]) < 50:
            for pp in list[last:-1]:
                sub_bulk[n_sub_bulk-1].append(pp)
        else:
            sub_bulk.append(list[last:])
    else:
        sub_bulk.append(list)

    return sub_bulk
###########################################################################

def getUserName():
    """
    extract user name from either SiteDB or Unix
    """
    if common.scheduler.name().upper() in ['LSF', 'CAF']:
       common.logger.debug(10,"Using as username the Unix user name")
       UserName=UnixUserName()
    else :
       UserName=gethnUserNameFromSiteDB() 

    return UserName

def UnixUserName():
    """
    extract username from whoami
    """
    try:
        UserName = runCommand("whoami")
        UserName = string.strip(UserName)
    except:
        msg = "Error. Problem with whoami command"
        raise CrabException(msg)
    return UserName

def getDN():
    """
    extract DN from user proxy's identity
    """
    try:
        userdn = runCommand("voms-proxy-info -identity")
        userdn = string.strip(userdn)
        #search for a / to avoid picking up warning messages
        userdn = userdn[userdn.find('/'):]
    except:
        msg = "Error. Problem with voms-proxy-info -identity command"
        raise CrabException(msg)
    return userdn.split('\n')[0]

def gethnUserNameFromSiteDB():
    """
    extract user name from SiteDB
    """
    from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
    hnUserName = None
    userdn = getDN()
    dict={ 'cacheduration' : 24 } 
    mySiteDB = SiteDBJSON(dict)
    msg_ = "there is no user name associated to DN %s in SiteDB. You need to register in SiteDB with the instructions at https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB" % userdn
    try:
        hnUserName = mySiteDB.dnUserName(dn=userdn)
    except:
        msg = "Error. Problem extracting user name from SiteDB"
        msg += "\n Check that you are registered in SiteDB, see https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB\n"
        msg += 'or %s'%msg_
        raise CrabException(msg)
    if not hnUserName:
        msg = "Error. %s"%msg_
        raise CrabException(msg)
    return hnUserName

###################################################################33

def numberFile(file, txt):
    """
    append _'txt' before last extension of a file
    """
    txt=str(txt)
    p = string.split(file,".")
    # take away last extension
    name = p[0]
    for x in p[1:-1]:
        name=name+"."+x
    # add "_txt"
    if len(p)>1:
        ext = p[len(p)-1]
        result = name + '_' + txt + "." + ext
    else:
        result = name + '_' + txt

    return result

def readTXTfile(self,inFileName):
    """
    read file and return a list with the content
    """
    out_list=[]
    if os.path.exists(inFileName):
        f = open(inFileName, 'r')
        for line in  f.readlines():
            out_list.append(string.strip(line))
        f.close()
    else:
        msg = ' file '+str(inFileName)+' not found.'
        raise CrabException(msg)
    return out_list

def writeTXTfile(self, outFileName, args):
    """
    write a file with the given content ( args )
    """
    outFile = open(outFileName,"a")
    outFile.write(str(args))
    outFile.close()
    return

def readableList(self,rawList):
    """
    Turn a list of numbers into a string like 1-5,7,9,12-20
    """
    if not rawList:
        return ''

    listString = str(rawList[0])
    endRange = ''
    for i in range(1,len(rawList)):
        if rawList[i] == rawList[i-1]+1:
            endRange = str(rawList[i])
        else:
            if endRange:
                listString += '-' + endRange + ',' + str(rawList[i])
                endRange = ''
            else:
                listString += ',' + str(rawList[i])
    if endRange:
        listString += '-' + endRange
        endRange = ''

    return listString

def getLocalDomain(self):
    """
    Get local domain name
    """
    import socket
    tmp=socket.gethostname()
    dot=string.find(tmp,'.')
    if (dot==-1):
        msg='Unkown domain name. Cannot use local scheduler'
        raise CrabException(msg)
    localDomainName = string.split(tmp,'.',1)[-1]
    return localDomainName

#######################################################
# Brian Bockelman bbockelm@cse.unl.edu
# Module to check the avaialble disk space on a specified directory.
#
import os
import statvfs

def has_freespace(dir_name, needed_space_kilobytes):
     enough_unix_quota = False
     enough_quota = False
     enough_partition = False
     enough_mount = False
     try:
         enough_mount = check_mount(dir_name, need_space_kilobytes)
     except:
         enough_mount = True
     try:
         enough_quota = check_quota(dir_name, needed_space_kilobytes)
     except:
         raise
         enough_quota = True
     try:
         enough_partition = check_partition(dir_name,
             needed_space_kilobytes)
     except:
         enough_partition = True
     try:
         enough_unix_quota = check_unix_quota(dir_name,
             needed_space_kilobytes)
     except:
         enough_unix_quota = True
     return enough_mount and enough_quota and enough_partition \
         and enough_unix_quota

def check_mount(dir_name, needed_space_kilobytes):
     try:
         vfs = os.statvfs(dir_name)
     except:
         raise Exception("Unable to query VFS for %s." % dir_name)
     dev_free = vfs[statvfs.F_FRSIZE] * vfs[statvfs.F_BAVAIL]
     return dev_free/1024 > needed_space_kilobytes

def check_quota(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("/usr/bin/fs lq %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the file system quota!")
     try:
         results = results.split('\n')[1].split()
         quota, used = results[1:3]
         avail = int(quota) - int(used)
         return avail > needed_space_kilobytes
     except:
         return Exception("Unable to parse AFS output.")

def check_partition(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("/usr/bin/fs diskfree %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the file system quota!")
     try:
         results = results.split('\n')[1].split()
         avail = results[3]
         return int(avail) > needed_space_kilobytes
     except:
         raise Exception("Unable to parse AFS output.")

def check_unix_quota(dir_name, needed_space_kilobytes):
     _, fd, _ = os.popen3("df %s" % dir_name)
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the filesystem with df.")
     fs = results.split('\n')[1].split()[0]
     _, fd, _ = os.popen3("quota -Q -u -g")
     results = fd.read()
     if fd.close():
         raise Exception("Unable to query the quotas.")
     has_info = False
     for line in results.splitlines():
         info = line.split()
         if info[0] in ['Filesystem', 'Disk']:
             continue
         if len(info) == 1:
             filesystem = info[0]
             has_info = False
         if len(info) == 6:
             used, limit = info[0], info[2]
             has_info = True
         if len(info) == 7:
             filesystem, used, limit = info[0], info[1], info[3]
             has_info = True
         if has_info:
            if filesystem != fs:
                continue
            avail = int(limit) - int(used)
            if avail < needed_space_kilobytes:
                return False
     return True

def getGZSize(gzipfile):
    # return the uncompressed size of a gzipped file
    import struct
    f = open(gzipfile, "rb")
    if f.read(2) != "\x1f\x8b":
        raise IOError("not a gzip file")
    f.seek(-4, 2)
    return struct.unpack("<i", f.read())[0]

def showWebMon(server_name):
    taskName = common._db.queryTask('name')
    msg = ''
    msg +='You can also follow the status of this task on :\n'
    msg +='\tCMS Dashboard: http://dashb-cms-job-task.cern.ch/taskmon.html#task=%s\n'%(taskName)
    if server_name != '' :
        msg += '\tServer page: http://%s:8888/logginfo\n'%server_name
    msg += '\tYour task name is: %s \n'%taskName
    return msg


####################################
if __name__ == '__main__':
    print 'sys.argv[1] =',sys.argv[1]
    list = parseRange2(sys.argv[1])
    print list
    cksum = makeCksum("crab_util.py")
    print cksum

Revision:	1.76
Committed:	Thu Apr 23 19:09:38 2009 UTC (16 years ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
Changes since 1.75:	+2 -7 lines
Log Message:	search only for directories when looking for last working directory + cleanup of code
#	User	Rev	Content
1	nsmirnov	1.1	###########################################################################
2			#
3			# C O N V E N I E N C E F U N C T I O N S
4			#
5			###########################################################################
6
7	slacapra	1.12	import string, sys, os, time
8	slacapra	1.16	import ConfigParser, re, popen2, select, fcntl
9	nsmirnov	1.1
10	nsmirnov	1.2	import common
11	slacapra	1.30	from crab_exceptions import CrabException
12	spiga	1.42	from ServerConfig import *
13	nsmirnov	1.1
14			###########################################################################
15			def parseOptions(argv):
16			"""
17			Parses command-line options.
18			Returns a dictionary with specified options as keys:
19			-opt1 --> 'opt1' : None
20			-opt2 val --> 'opt2' : 'val'
21			-opt3=val --> 'opt3' : 'val'
22			Usually called as
23			options = parseOptions(sys.argv[1:])
24			"""
25			options = {}
26			argc = len(argv)
27			i = 0
28			while ( i < argc ):
29			if argv[i][0] != '-':
30			i = i + 1
31			continue
32			eq = string.find(argv[i], '=')
33			if eq > 0 :
34			opt = argv[i][:eq]
35			val = argv[i][eq+1:]
36			pass
37			else:
38			opt = argv[i]
39			val = None
40			if ( i+1 < argc and argv[i+1][0] != '-' ):
41			i = i + 1
42			val = argv[i]
43			pass
44			pass
45			options[opt] = val
46			i = i + 1
47			pass
48			return options
49
50	slacapra	1.47	def loadConfig(file, config):
51	nsmirnov	1.1	"""
52			returns a dictionary with keys of the form
53			<section>.<option> and the corresponding values
54			"""
55	slacapra	1.47	#config={}
56	nsmirnov	1.1	cp = ConfigParser.ConfigParser()
57			cp.read(file)
58			for sec in cp.sections():
59			# print 'Section',sec
60			for opt in cp.options(sec):
61			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
62			config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
63			return config
64
65			###########################################################################
66			def isInt(str):
67			""" Is the given string an integer ?"""
68			try: int(str)
69			except ValueError: return 0
70			return 1
71
72			###########################################################################
73			def isBool(str):
74			""" Is the given string 0 or 1 ?"""
75			if (str in ('0','1')): return 1
76			return 0
77
78			###########################################################################
79	nsmirnov	1.3	def parseRange(range):
80			"""
81			Takes as the input a string with two integers separated by
82			the minus sign and returns the tuple with these numbers:
83			'n1-n2' -> (n1, n2)
84			'n1' -> (n1, n1)
85			"""
86			start = None
87			end = None
88			minus = string.find(range, '-')
89			if ( minus < 0 ):
90			if isInt(range):
91			start = int(range)
92			end = start
93			pass
94			pass
95			else:
96			if isInt(range[:minus]) and isInt(range[minus+1:]):
97			start = int(range[:minus])
98			end = int(range[minus+1:])
99			pass
100			pass
101			return (start, end)
102
103			###########################################################################
104	nsmirnov	1.4	def parseRange2(range):
105			"""
106			Takes as the input a string in the form of a comma-separated
107			numbers and ranges
108			and returns a list with all specified numbers:
109			'n1' -> [n1]
110			'n1-n2' -> [n1, n1+1, ..., n2]
111			'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
112			"""
113	slacapra	1.30	result = []
114			if not range: return result
115	nsmirnov	1.4
116			comma = string.find(range, ',')
117			if comma == -1: left = range
118			else: left = range[:comma]
119
120			(n1, n2) = parseRange(left)
121			while ( n1 <= n2 ):
122	slacapra	1.11	try:
123	slacapra	1.30	result.append(n1)
124	slacapra	1.11	n1 += 1
125			pass
126			except:
127			msg = 'Syntax error in range <'+range+'>'
128			raise CrabException(msg)
129	nsmirnov	1.4
130			if comma != -1:
131	slacapra	1.11	try:
132	slacapra	1.30	result.extend(parseRange2(range[comma+1:]))
133	slacapra	1.11	pass
134			except:
135			msg = 'Syntax error in range <'+range+'>'
136			raise CrabException(msg)
137	nsmirnov	1.4
138	slacapra	1.30	return result
139	nsmirnov	1.4
140			###########################################################################
141	nsmirnov	1.1	def findLastWorkDir(dir_prefix, where = None):
142
143			if not where: where = os.getcwd() + '/'
144			# dir_prefix usually has the form 'crab_0_'
145			pattern = re.compile(dir_prefix)
146
147	slacapra	1.76	file_list = [f for f in os.listdir(where) if os.path.isdir(f) and pattern.match(f)]
148	nsmirnov	1.1
149			if len(file_list) == 0: return None
150
151			file_list.sort()
152
153	slacapra	1.76	wdir = where + file_list[-1]
154	nsmirnov	1.1	return wdir
155
156			###########################################################################
157			def importName(module_name, name):
158			"""
159			Import a named object from a Python module,
160			i.e., it is an equivalent of 'from module_name import name'.
161			"""
162			module = __import__(module_name, globals(), locals(), [name])
163			return vars(module)[name]
164
165			###########################################################################
166	slacapra	1.16	def readable(fd):
167	ewv	1.44	return bool(select.select([fd], [], [], 0))
168	slacapra	1.16
169			###########################################################################
170			def makeNonBlocking(fd):
171			fl = fcntl.fcntl(fd, fcntl.F_GETFL)
172			try:
173			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.O_NDELAY)
174			except AttributeError:
175			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.FNDELAY)
176
177			###########################################################################
178	slacapra	1.11	def runCommand(cmd, printout=0, timeout=-1):
179	nsmirnov	1.1	"""
180			Run command 'cmd'.
181			Returns command stdoutput+stderror string on success,
182			or None if an error occurred.
183	slacapra	1.16	Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
184	nsmirnov	1.1	"""
185	fanzago	1.15
186	slacapra	1.11	if printout:
187			common.logger.message(cmd)
188			else:
189	slacapra	1.13	common.logger.debug(10,cmd)
190	slacapra	1.11	common.logger.write(cmd)
191	slacapra	1.29	pass
192	slacapra	1.11
193	slacapra	1.16	child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
194			child.tochild.close() # don't need to talk to child
195	ewv	1.44	outfile = child.fromchild
196	slacapra	1.16	outfd = outfile.fileno()
197			errfile = child.childerr
198			errfd = errfile.fileno()
199			makeNonBlocking(outfd) # don't deadlock!
200			makeNonBlocking(errfd)
201			outdata = []
202			errdata = []
203			outeof = erreof = 0
204
205			if timeout > 0 :
206			maxwaittime = time.time() + timeout
207
208			err = -1
209			while (timeout == -1 or time.time() < maxwaittime):
210			ready = select.select([outfd,errfd],[],[]) # wait for input
211			if outfd in ready[0]:
212			outchunk = outfile.read()
213			if outchunk == '': outeof = 1
214			outdata.append(outchunk)
215			if errfd in ready[0]:
216			errchunk = errfile.read()
217			if errchunk == '': erreof = 1
218			errdata.append(errchunk)
219			if outeof and erreof:
220			err = child.wait()
221			break
222	slacapra	1.21	select.select([],[],[],.01) # give a little time for buffers to fill
223	slacapra	1.16	if err == -1:
224			# kill the pid
225	slacapra	1.18	common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
226	slacapra	1.16	os.kill (child.pid, 9)
227	slacapra	1.11	err = child.wait()
228	slacapra	1.16
229	fanzago	1.17	cmd_out = string.join(outdata,"")
230			cmd_err = string.join(errdata,"")
231
232	slacapra	1.11	if err:
233			common.logger.message('`'+cmd+'`\n failed with exit code '
234			+`err`+'='+`(err&0xff)`+'(signal)+'
235			+`(err>>8)`+'(status)')
236			common.logger.message(cmd_out)
237			common.logger.message(cmd_err)
238	nsmirnov	1.1	return None
239
240	fanzago	1.17	# cmd_out = string.join(outdata,"")
241			# cmd_err = string.join(errdata,"")
242	nsmirnov	1.1	cmd_out = cmd_out + cmd_err
243	slacapra	1.11	if printout:
244			common.logger.message(cmd_out)
245			else:
246	slacapra	1.13	common.logger.debug(10,cmd_out)
247	slacapra	1.11	common.logger.write(cmd_out)
248	slacapra	1.29	pass
249	slacapra	1.16	#print "<"+cmd_out+">"
250	nsmirnov	1.1	return cmd_out
251	nsmirnov	1.4
252	slacapra	1.11	####################################
253	gutsche	1.20	def makeCksum(filename) :
254			"""
255	ewv	1.44	make check sum using filename and content of file
256	gutsche	1.20	"""
257
258	ewv	1.44	from zlib import crc32
259			hashString = filename
260	gutsche	1.20
261	ewv	1.44	inFile = open(filename, 'r')
262			hashString += inFile.read()
263			inFile.close()
264	gutsche	1.20
265	ewv	1.44	cksum = str(crc32(hashString))
266	gutsche	1.20	return cksum
267
268	ewv	1.52
269	gutsche	1.32	def spanRanges(jobArray):
270			"""
271			take array of job numbers and concatenate 1,2,3 to 1-3
272			return string
273			"""
274
275			output = ""
276	mcinquil	1.35	jobArray.sort()
277	ewv	1.44
278	gutsche	1.32	previous = jobArray[0]-1
279			for job in jobArray:
280			if previous+1 == job:
281			previous = job
282			if len(output) > 0 :
283			if output[-1] != "-":
284			output += "-"
285			else :
286			output += str(previous)
287			else:
288			output += str(previous) + "," + str(job)
289	mcinquil	1.35	#output += "," + str(job)
290	gutsche	1.32	previous = job
291			if len(jobArray) > 1 :
292			output += str(previous)
293
294			return output
295
296	spiga	1.40	def displayReport(self, header, lines, xml=''):
297	ewv	1.44
298	spiga	1.49	counter = 0
299			printline = ''
300			printline+= header
301			print printline
302			print '---------------------------------------------------------------------------------------------------'
303
304			for i in range(len(lines)):
305			if counter != 0 and counter%10 == 0 :
306			print '---------------------------------------------------------------------------------------------------'
307			print lines[i]
308			counter += 1
309			if xml != '' :
310	spiga	1.40	fileName = common.work_space.shareDir() + xml
311			task = common._db.getTask()
312			taskXML = common._db.serializeTask(task)
313			common.logger.debug(5, taskXML)
314			f = open(fileName, 'w')
315			f.write(taskXML)
316			f.close()
317			pass
318	spiga	1.39
319	spiga	1.42	def CliServerParams(self):
320	slacapra	1.45	"""
321			Init client-server interactions
322			"""
323			self.srvCfg = {}
324	slacapra	1.63	## First I have to check if the decision has been already taken...
325			task = common._db.getTask()
326			if task['serverName']!=None:
327			self.cfg_params['CRAB.server_name']=task['serverName']
328	slacapra	1.61	if self.cfg_params.has_key('CRAB.use_server'):
329	slacapra	1.62	if self.cfg_params.has_key('CRAB.server_name'):
330			serverName=self.cfg_params['CRAB.server_name']
331			else:
332			serverName='default'
333			self.srvCfg = ServerConfig(serverName).config()
334	slacapra	1.63	# save the serverName for future use
335			opsToBeSaved={}
336			opsToBeSaved['serverName']=self.srvCfg['serverGenericName']
337			common._db.updateTask_(opsToBeSaved)
338	slacapra	1.61	elif self.cfg_params.has_key('CRAB.server_name'):
339	slacapra	1.45	self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()
340	slacapra	1.61	else:
341			msg = 'No server selected or port specified.\n'
342			msg += 'Please specify a server in the crab cfg file'
343			raise CrabException(msg)
344			return
345	slacapra	1.45
346	slacapra	1.61	self.server_admin = str(self.srvCfg['serverAdmin'])
347			self.server_dn = str(self.srvCfg['serverDN'])
348	spiga	1.58
349	slacapra	1.61	self.server_name = str(self.srvCfg['serverName'])
350			self.server_port = int(self.srvCfg['serverPort'])
351	slacapra	1.45
352	slacapra	1.61	self.storage_name = str(self.srvCfg['storageName'])
353			self.storage_path = str(self.srvCfg['storagePath'])
354			self.storage_proto = str(self.srvCfg['storageProtocol'])
355			self.storage_port = str(self.srvCfg['storagePort'])
356	spiga	1.39
357	ewv	1.44	def bulkControl(self,list):
358	slacapra	1.45	"""
359			Check the BULK size and reduce collection ...if needed
360			"""
361			max_size = 400
362			sub_bulk = []
363			if len(list) > int(max_size):
364			n_sub_bulk = int( int(len(list) ) / int(max_size) )
365	mcinquil	1.53	for n in xrange(n_sub_bulk):
366	slacapra	1.45	first =n*int(max_size)
367			last = (n+1)*int(max_size)
368			sub_bulk.append(list[first:last])
369			if len(list[last:-1]) < 50:
370			for pp in list[last:-1]:
371			sub_bulk[n_sub_bulk-1].append(pp)
372	spiga	1.43	else:
373	mcinquil	1.53	sub_bulk.append(list[last:])
374	slacapra	1.45	else:
375			sub_bulk.append(list)
376
377			return sub_bulk
378	spiga	1.66	###########################################################################
379	slacapra	1.45
380	spiga	1.69	def getUserName():
381	spiga	1.66	"""
382			extract user name from either SiteDB or Unix
383			"""
384			if common.scheduler.name().upper() in ['LSF', 'CAF']:
385			common.logger.debug(10,"Using as username the Unix user name")
386	spiga	1.70	UserName=UnixUserName()
387	spiga	1.66	else :
388	spiga	1.74	UserName=gethnUserNameFromSiteDB()
389	spiga	1.66
390			return UserName
391	spiga	1.58
392	spiga	1.69	def UnixUserName():
393	spiga	1.58	"""
394			extract username from whoami
395			"""
396			try:
397			UserName = runCommand("whoami")
398			UserName = string.strip(UserName)
399			except:
400			msg = "Error. Problem with whoami command"
401			raise CrabException(msg)
402			return UserName
403	spiga	1.66
404	spiga	1.69	def getDN():
405	spiga	1.66	"""
406			extract DN from user proxy's identity
407			"""
408			try:
409			userdn = runCommand("voms-proxy-info -identity")
410			userdn = string.strip(userdn)
411			#search for a / to avoid picking up warning messages
412			userdn = userdn[userdn.find('/'):]
413			except:
414			msg = "Error. Problem with voms-proxy-info -identity command"
415			raise CrabException(msg)
416			return userdn.split('\n')[0]
417
418	spiga	1.69	def gethnUserNameFromSiteDB():
419	spiga	1.66	"""
420			extract user name from SiteDB
421			"""
422			from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
423			hnUserName = None
424	spiga	1.69	userdn = getDN()
425	spiga	1.75	dict={ 'cacheduration' : 24 }
426			mySiteDB = SiteDBJSON(dict)
427	spiga	1.73	msg_ = "there is no user name associated to DN %s in SiteDB. You need to register in SiteDB with the instructions at https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB" % userdn
428	spiga	1.66	try:
429			hnUserName = mySiteDB.dnUserName(dn=userdn)
430			except:
431			msg = "Error. Problem extracting user name from SiteDB"
432	spiga	1.73	msg += "\n Check that you are registered in SiteDB, see https://twiki.cern.ch/twiki/bin/view/CMS/SiteDBForCRAB\n"
433			msg += 'or %s'%msg_
434	spiga	1.66	raise CrabException(msg)
435			if not hnUserName:
436	spiga	1.73	msg = "Error. %s"%msg_
437	spiga	1.66	raise CrabException(msg)
438			return hnUserName
439
440			###################################################################33
441
442	slacapra	1.45	def numberFile(file, txt):
443			"""
444			append _'txt' before last extension of a file
445			"""
446			txt=str(txt)
447			p = string.split(file,".")
448			# take away last extension
449			name = p[0]
450			for x in p[1:-1]:
451			name=name+"."+x
452			# add "_txt"
453			if len(p)>1:
454			ext = p[len(p)-1]
455			result = name + '_' + txt + "." + ext
456			else:
457			result = name + '_' + txt
458	spiga	1.43
459	slacapra	1.45	return result
460	spiga	1.46
461			def readTXTfile(self,inFileName):
462			"""
463			read file and return a list with the content
464			"""
465			out_list=[]
466			if os.path.exists(inFileName):
467			f = open(inFileName, 'r')
468			for line in f.readlines():
469	ewv	1.52	out_list.append(string.strip(line))
470	spiga	1.46	f.close()
471			else:
472			msg = ' file '+str(inFileName)+' not found.'
473	ewv	1.52	raise CrabException(msg)
474	spiga	1.46	return out_list
475
476			def writeTXTfile(self, outFileName, args):
477			"""
478			write a file with the given content ( args )
479			"""
480			outFile = open(outFileName,"a")
481			outFile.write(str(args))
482			outFile.close()
483			return
484
485	spiga	1.54	def readableList(self,rawList):
486	ewv	1.64	"""
487			Turn a list of numbers into a string like 1-5,7,9,12-20
488			"""
489			if not rawList:
490			return ''
491
492	slacapra	1.56	listString = str(rawList[0])
493			endRange = ''
494			for i in range(1,len(rawList)):
495			if rawList[i] == rawList[i-1]+1:
496			endRange = str(rawList[i])
497			else:
498			if endRange:
499			listString += '-' + endRange + ',' + str(rawList[i])
500			endRange = ''
501			else:
502			listString += ',' + str(rawList[i])
503			if endRange:
504			listString += '-' + endRange
505			endRange = ''
506	ewv	1.64
507	slacapra	1.56	return listString
508	spiga	1.54
509	spiga	1.51	def getLocalDomain(self):
510	ewv	1.52	"""
511	slacapra	1.56	Get local domain name
512	ewv	1.52	"""
513	spiga	1.51	import socket
514			tmp=socket.gethostname()
515			dot=string.find(tmp,'.')
516			if (dot==-1):
517			msg='Unkown domain name. Cannot use local scheduler'
518			raise CrabException(msg)
519			localDomainName = string.split(tmp,'.',1)[-1]
520	ewv	1.52	return localDomainName
521	spiga	1.51
522	slacapra	1.56	#######################################################
523			# Brian Bockelman bbockelm@cse.unl.edu
524			# Module to check the avaialble disk space on a specified directory.
525			#
526			import os
527			import statvfs
528
529			def has_freespace(dir_name, needed_space_kilobytes):
530			enough_unix_quota = False
531			enough_quota = False
532			enough_partition = False
533			enough_mount = False
534			try:
535			enough_mount = check_mount(dir_name, need_space_kilobytes)
536			except:
537			enough_mount = True
538			try:
539			enough_quota = check_quota(dir_name, needed_space_kilobytes)
540			except:
541			raise
542			enough_quota = True
543			try:
544			enough_partition = check_partition(dir_name,
545			needed_space_kilobytes)
546			except:
547			enough_partition = True
548			try:
549			enough_unix_quota = check_unix_quota(dir_name,
550			needed_space_kilobytes)
551			except:
552			enough_unix_quota = True
553			return enough_mount and enough_quota and enough_partition \
554			and enough_unix_quota
555
556			def check_mount(dir_name, needed_space_kilobytes):
557			try:
558			vfs = os.statvfs(dir_name)
559			except:
560			raise Exception("Unable to query VFS for %s." % dir_name)
561			dev_free = vfs[statvfs.F_FRSIZE] * vfs[statvfs.F_BAVAIL]
562			return dev_free/1024 > needed_space_kilobytes
563
564			def check_quota(dir_name, needed_space_kilobytes):
565			_, fd, _ = os.popen3("/usr/bin/fs lq %s" % dir_name)
566			results = fd.read()
567			if fd.close():
568			raise Exception("Unable to query the file system quota!")
569			try:
570			results = results.split('\n')[1].split()
571			quota, used = results[1:3]
572			avail = int(quota) - int(used)
573			return avail > needed_space_kilobytes
574			except:
575			return Exception("Unable to parse AFS output.")
576
577			def check_partition(dir_name, needed_space_kilobytes):
578			_, fd, _ = os.popen3("/usr/bin/fs diskfree %s" % dir_name)
579			results = fd.read()
580			if fd.close():
581			raise Exception("Unable to query the file system quota!")
582			try:
583			results = results.split('\n')[1].split()
584			avail = results[3]
585			return int(avail) > needed_space_kilobytes
586			except:
587			raise Exception("Unable to parse AFS output.")
588
589			def check_unix_quota(dir_name, needed_space_kilobytes):
590			_, fd, _ = os.popen3("df %s" % dir_name)
591			results = fd.read()
592			if fd.close():
593			raise Exception("Unable to query the filesystem with df.")
594			fs = results.split('\n')[1].split()[0]
595			_, fd, _ = os.popen3("quota -Q -u -g")
596			results = fd.read()
597			if fd.close():
598			raise Exception("Unable to query the quotas.")
599			has_info = False
600			for line in results.splitlines():
601			info = line.split()
602			if info[0] in ['Filesystem', 'Disk']:
603			continue
604			if len(info) == 1:
605			filesystem = info[0]
606			has_info = False
607			if len(info) == 6:
608			used, limit = info[0], info[2]
609			has_info = True
610			if len(info) == 7:
611			filesystem, used, limit = info[0], info[1], info[3]
612			has_info = True
613			if has_info:
614			if filesystem != fs:
615			continue
616			avail = int(limit) - int(used)
617			if avail < needed_space_kilobytes:
618			return False
619			return True
620	spiga	1.54
621	slacapra	1.57	def getGZSize(gzipfile):
622			# return the uncompressed size of a gzipped file
623			import struct
624			f = open(gzipfile, "rb")
625			if f.read(2) != "\x1f\x8b":
626			raise IOError("not a gzip file")
627			f.seek(-4, 2)
628	ewv	1.64	return struct.unpack("<i", f.read())[0]
629	slacapra	1.57
630	spiga	1.60	def showWebMon(server_name):
631	spiga	1.72	taskName = common._db.queryTask('name')
632	spiga	1.60	msg = ''
633	spiga	1.71	msg +='You can also follow the status of this task on :\n'
634	spiga	1.72	msg +='\tCMS Dashboard: http://dashb-cms-job-task.cern.ch/taskmon.html#task=%s\n'%(taskName)
635	ewv	1.64	if server_name != '' :
636	spiga	1.71	msg += '\tServer page: http://%s:8888/logginfo\n'%server_name
637	spiga	1.72	msg += '\tYour task name is: %s \n'%taskName
638	spiga	1.60	return msg
639
640
641	gutsche	1.20	####################################
642	nsmirnov	1.4	if __name__ == '__main__':
643			print 'sys.argv[1] =',sys.argv[1]
644			list = parseRange2(sys.argv[1])
645			print list
646	slacapra	1.29	cksum = makeCksum("crab_util.py")
647			print cksum
648	ewv	1.44