CRAB/python/crab_util.py

###########################################################################
#
#   C O N V E N I E N C E    F U N C T I O N S
#
###########################################################################

import string, sys, os, time
import ConfigParser, re, popen2, select, fcntl

import common
from crab_exceptions import CrabException

###########################################################################
def parseOptions(argv):
    """
    Parses command-line options.
    Returns a dictionary with specified options as keys:
    -opt1             --> 'opt1' : None
    -opt2 val         --> 'opt2' : 'val'
    -opt3=val         --> 'opt3' : 'val'
    Usually called as
    options = parseOptions(sys.argv[1:])
    """
    options = {}
    argc = len(argv)
    i = 0
    while ( i < argc ):
        if argv[i][0] != '-':
            i = i + 1
            continue
        eq = string.find(argv[i], '=')
        if eq > 0 :
            opt = argv[i][:eq]
            val = argv[i][eq+1:]
            pass
        else:
            opt = argv[i]
            val = None
            if ( i+1 < argc and argv[i+1][0] != '-' ):
                i = i + 1
                val = argv[i]
                pass
            pass
        options[opt] = val
        i = i + 1
        pass
    return options

###########################################################################
def loadConfig(file):
    """
    returns a dictionary with keys of the form
    <section>.<option> and the corresponding values
    """
    config={}
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        # print 'Section',sec
        for opt in cp.options(sec):
            #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
            config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
    # marco. Try to prevent user from switching off Monalisa reporting
    config['USER.activate_monalisa'] = 1
    # marco
    return config

###########################################################################
def isInt(str):
    """ Is the given string an integer ?"""
    try: int(str)
    except ValueError: return 0
    return 1

###########################################################################
def isBool(str):
    """ Is the given string 0 or 1 ?"""
    if (str in ('0','1')): return 1
    return 0

###########################################################################
def parseRange(range):
    """
    Takes as the input a string with two integers separated by
    the minus sign and returns the tuple with these numbers:
    'n1-n2' -> (n1, n2)
    'n1'    -> (n1, n1)
    """
    start = None
    end   = None
    minus = string.find(range, '-')
    if ( minus < 0 ):
        if isInt(range):
            start = int(range)
            end   = start
            pass
        pass
    else:
        if isInt(range[:minus]) and isInt(range[minus+1:]):
            start = int(range[:minus])
            end   = int(range[minus+1:])
            pass
        pass
    return (start, end)

###########################################################################
def parseRange2(range):
    """
    Takes as the input a string in the form of a comma-separated
    numbers and ranges
    and returns a list with all specified numbers:
    'n1'          -> [n1]
    'n1-n2'       -> [n1, n1+1, ..., n2]
    'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
    """
    result = []
    if not range: return result

    comma = string.find(range, ',')
    if comma == -1: left = range
    else:           left = range[:comma]

    (n1, n2) = parseRange(left)
    while ( n1 <= n2 ):
        try:
            result.append(n1)
            n1 += 1
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    if comma != -1:
        try:
            result.extend(parseRange2(range[comma+1:]))
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    return result

###########################################################################
def crabJobStatusToString(crab_status):
    """
    Convert one-letter crab job status into more readable string.
    """
    status={
           'H':'Hold',
           'U':'Ready',
           'I':'Scheduled',
           'X':'Canceled',
           'W':'Created',
           'R':'Running',
           'SC':'Checkpointed',
           'SS':'Scheduled',
           'SR':'Ready',
           'RE':'Ready',
           'SW':'Waiting',
           'SU':'Submitted',
           'S' :'Submitted (Boss)',
           'UN':'Undefined',
           'SK':'Cancelled',
           'SD':'Done (Success)',
           'SA':'Aborted',
           'DA':'Done (Aborted)',
           'SE':'Cleared',
           'OR':'Done (Success)',
           'A?':'Aborted',
           'K':'Killed',
           'E':'Cleared',
           'Z':'Cleared (Corrupt)',
           'NA':'Unknown',
           'I?':'Idle',
           'O?':'Done',
           'R?':'Running'
           }
#    if   crab_status == 'C': status = 'Created'
#    elif crab_status == 'D': status = 'Done'
#    elif crab_status == 'R': status = 'Submitted'#Should be running? ds  
#    elif crab_status == 'S': status = 'Submitted'
#    elif crab_status == 'K': status = 'Killed'
#    elif crab_status == 'X': status = 'None'
#    elif crab_status == 'Y': status = 'Output retrieved'
#    elif crab_status == 'A': status = 'Aborted'
#    elif crab_status == 'RC': status = 'ReCreated'
#    else: status = '???'
    return status[crab_status]

###########################################################################
def findLastWorkDir(dir_prefix, where = None):

    if not where: where = os.getcwd() + '/'
    # dir_prefix usually has the form 'crab_0_'
    pattern = re.compile(dir_prefix)

    file_list = []
    for fl in os.listdir(where):
        if pattern.match(fl):
            file_list.append(fl)
            pass
        pass

    if len(file_list) == 0: return None

    file_list.sort()

    wdir = where + file_list[len(file_list)-1]
    return wdir

###########################################################################
def importName(module_name, name):
    """
    Import a named object from a Python module,
    i.e., it is an equivalent of 'from module_name import name'.
    """
    module = __import__(module_name, globals(), locals(), [name])
    return vars(module)[name]


###########################################################################

### WARNING This Function become USELESS after Boss API implementation
def runBossCommand(cmd, printout=0, timeout=3600):
    """
    Cd to correct directory before running a boss command
    """
    cwd = os.getcwd()
    os.chdir(common.work_space.shareDir())
    out = runCommand(cmd, printout, timeout)
    os.chdir(cwd)
    return out

###########################################################################
def readable(fd):
    return bool(select.select([fd], [], [], 0)) 

###########################################################################
def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

###########################################################################
def runCommand(cmd, printout=0, timeout=-1):
    """
    Run command 'cmd'.
    Returns command stdoutput+stderror string on success,
    or None if an error occurred.
    Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
    """

    if printout:
        common.logger.message(cmd)
    else:
        common.logger.debug(10,cmd)
        common.logger.write(cmd)
        pass

    child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild 
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = []
    errdata = []
    outeof = erreof = 0

    if timeout > 0 :
        maxwaittime = time.time() + timeout

    err = -1
    while (timeout == -1 or time.time() < maxwaittime):
        ready = select.select([outfd,errfd],[],[]) # wait for input
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '': outeof = 1
            outdata.append(outchunk)
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '': erreof = 1
            errdata.append(errchunk)
        if outeof and erreof:
            err = child.wait()
            break
        select.select([],[],[],.01) # give a little time for buffers to fill
    if err == -1:
        # kill the pid
        common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
        os.kill (child.pid, 9)
        err = child.wait()

    cmd_out = string.join(outdata,"")
    cmd_err = string.join(errdata,"")

    if err:
        common.logger.message('`'+cmd+'`\n   failed with exit code '
                           +`err`+'='+`(err&0xff)`+'(signal)+'
                           +`(err>>8)`+'(status)')
        common.logger.message(cmd_out)
        common.logger.message(cmd_err)
        return None

#    cmd_out = string.join(outdata,"")
#    cmd_err = string.join(errdata,"")
    cmd_out = cmd_out + cmd_err
    if printout:
        common.logger.message(cmd_out)
    else:
        common.logger.debug(10,cmd_out)
        common.logger.write(cmd_out)
        pass
    #print "<"+cmd_out+">"
    return cmd_out

####################################
def makeCksum(filename) :
    """
    make chksum using filename and content of file
    """

    try:
        import tempfile
        tmpfile= tempfile.NamedTemporaryFile(mode='w')
        tmp_filename = tmpfile.name
    except:
        ## SL in case py2.2 is used, fall back to old solution
        tmp_filename = tempfile.mktemp()
        tmpfile= open(tmp_filename,'w')
        tobedeleted=1

    # add filename as first line
    tmpfile.write(filename+'\n')

    # fill input file in tmp file
    infile = open(filename)
    tmpfile.writelines(infile.readlines())
    tmpfile.flush()
    infile.close()

    cmd = 'cksum '+tmp_filename
    cmd_out = runCommand(cmd)

    cksum = cmd_out.split()[0]

    ## SL this is not needed if we use NamedTemporaryFile, which is not available in py2.2
    if (tobedeleted): os.remove(tmp_filename)

    return cksum

####################################
if __name__ == '__main__':
    print 'sys.argv[1] =',sys.argv[1]
    list = parseRange2(sys.argv[1])
    print list
    cksum = makeCksum("crab_util.py")
    print cksum
    
Revision:	1.30
Committed:	Wed Jan 17 18:17:59 2007 UTC (18 years, 3 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_5_0_pre4
Changes since 1.29:	+9 -9 lines
Log Message:	many minor fixes reported by pychecker, mostly unsude import and unused variables
#	User	Rev	Content
1	nsmirnov	1.1	###########################################################################
2			#
3			# C O N V E N I E N C E F U N C T I O N S
4			#
5			###########################################################################
6
7	slacapra	1.12	import string, sys, os, time
8	slacapra	1.16	import ConfigParser, re, popen2, select, fcntl
9	nsmirnov	1.1
10	nsmirnov	1.2	import common
11	slacapra	1.30	from crab_exceptions import CrabException
12	nsmirnov	1.1
13			###########################################################################
14			def parseOptions(argv):
15			"""
16			Parses command-line options.
17			Returns a dictionary with specified options as keys:
18			-opt1 --> 'opt1' : None
19			-opt2 val --> 'opt2' : 'val'
20			-opt3=val --> 'opt3' : 'val'
21			Usually called as
22			options = parseOptions(sys.argv[1:])
23			"""
24			options = {}
25			argc = len(argv)
26			i = 0
27			while ( i < argc ):
28			if argv[i][0] != '-':
29			i = i + 1
30			continue
31			eq = string.find(argv[i], '=')
32			if eq > 0 :
33			opt = argv[i][:eq]
34			val = argv[i][eq+1:]
35			pass
36			else:
37			opt = argv[i]
38			val = None
39			if ( i+1 < argc and argv[i+1][0] != '-' ):
40			i = i + 1
41			val = argv[i]
42			pass
43			pass
44			options[opt] = val
45			i = i + 1
46			pass
47			return options
48
49			###########################################################################
50			def loadConfig(file):
51			"""
52			returns a dictionary with keys of the form
53			<section>.<option> and the corresponding values
54			"""
55			config={}
56			cp = ConfigParser.ConfigParser()
57			cp.read(file)
58			for sec in cp.sections():
59			# print 'Section',sec
60			for opt in cp.options(sec):
61			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
62			config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
63	corvo	1.19	# marco. Try to prevent user from switching off Monalisa reporting
64			config['USER.activate_monalisa'] = 1
65			# marco
66	nsmirnov	1.1	return config
67
68			###########################################################################
69			def isInt(str):
70			""" Is the given string an integer ?"""
71			try: int(str)
72			except ValueError: return 0
73			return 1
74
75			###########################################################################
76			def isBool(str):
77			""" Is the given string 0 or 1 ?"""
78			if (str in ('0','1')): return 1
79			return 0
80
81			###########################################################################
82	nsmirnov	1.3	def parseRange(range):
83			"""
84			Takes as the input a string with two integers separated by
85			the minus sign and returns the tuple with these numbers:
86			'n1-n2' -> (n1, n2)
87			'n1' -> (n1, n1)
88			"""
89			start = None
90			end = None
91			minus = string.find(range, '-')
92			if ( minus < 0 ):
93			if isInt(range):
94			start = int(range)
95			end = start
96			pass
97			pass
98			else:
99			if isInt(range[:minus]) and isInt(range[minus+1:]):
100			start = int(range[:minus])
101			end = int(range[minus+1:])
102			pass
103			pass
104			return (start, end)
105
106			###########################################################################
107	nsmirnov	1.4	def parseRange2(range):
108			"""
109			Takes as the input a string in the form of a comma-separated
110			numbers and ranges
111			and returns a list with all specified numbers:
112			'n1' -> [n1]
113			'n1-n2' -> [n1, n1+1, ..., n2]
114			'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
115			"""
116	slacapra	1.30	result = []
117			if not range: return result
118	nsmirnov	1.4
119			comma = string.find(range, ',')
120			if comma == -1: left = range
121			else: left = range[:comma]
122
123			(n1, n2) = parseRange(left)
124			while ( n1 <= n2 ):
125	slacapra	1.11	try:
126	slacapra	1.30	result.append(n1)
127	slacapra	1.11	n1 += 1
128			pass
129			except:
130			msg = 'Syntax error in range <'+range+'>'
131			raise CrabException(msg)
132	nsmirnov	1.4
133			if comma != -1:
134	slacapra	1.11	try:
135	slacapra	1.30	result.extend(parseRange2(range[comma+1:]))
136	slacapra	1.11	pass
137			except:
138			msg = 'Syntax error in range <'+range+'>'
139			raise CrabException(msg)
140	nsmirnov	1.4
141	slacapra	1.30	return result
142	nsmirnov	1.4
143			###########################################################################
144	nsmirnov	1.3	def crabJobStatusToString(crab_status):
145			"""
146			Convert one-letter crab job status into more readable string.
147			"""
148	corvo	1.27	status={
149			'H':'Hold',
150			'U':'Ready',
151			'I':'Scheduled',
152			'X':'Canceled',
153			'W':'Created',
154			'R':'Running',
155			'SC':'Checkpointed',
156			'SS':'Scheduled',
157			'SR':'Ready',
158			'RE':'Ready',
159			'SW':'Waiting',
160			'SU':'Submitted',
161			'S' :'Submitted (Boss)',
162			'UN':'Undefined',
163			'SK':'Cancelled',
164			'SD':'Done (Success)',
165			'SA':'Aborted',
166			'DA':'Done (Aborted)',
167			'SE':'Cleared',
168			'OR':'Done (Success)',
169			'A?':'Aborted',
170			'K':'Killed',
171			'E':'Cleared',
172			'Z':'Cleared (Corrupt)',
173			'NA':'Unknown',
174			'I?':'Idle',
175			'O?':'Done',
176			'R?':'Running'
177			}
178			# if crab_status == 'C': status = 'Created'
179			# elif crab_status == 'D': status = 'Done'
180			# elif crab_status == 'R': status = 'Submitted'#Should be running? ds
181			# elif crab_status == 'S': status = 'Submitted'
182			# elif crab_status == 'K': status = 'Killed'
183			# elif crab_status == 'X': status = 'None'
184			# elif crab_status == 'Y': status = 'Output retrieved'
185			# elif crab_status == 'A': status = 'Aborted'
186			# elif crab_status == 'RC': status = 'ReCreated'
187			# else: status = '???'
188			return status[crab_status]
189	nsmirnov	1.3
190			###########################################################################
191	nsmirnov	1.1	def findLastWorkDir(dir_prefix, where = None):
192
193			if not where: where = os.getcwd() + '/'
194			# dir_prefix usually has the form 'crab_0_'
195			pattern = re.compile(dir_prefix)
196
197			file_list = []
198			for fl in os.listdir(where):
199			if pattern.match(fl):
200			file_list.append(fl)
201			pass
202			pass
203
204			if len(file_list) == 0: return None
205
206			file_list.sort()
207
208			wdir = where + file_list[len(file_list)-1]
209			return wdir
210
211			###########################################################################
212			def importName(module_name, name):
213			"""
214			Import a named object from a Python module,
215			i.e., it is an equivalent of 'from module_name import name'.
216			"""
217			module = __import__(module_name, globals(), locals(), [name])
218			return vars(module)[name]
219
220	spiga	1.9
221	slacapra	1.11	###########################################################################
222	spiga	1.28
223	slacapra	1.29	### WARNING This Function become USELESS after Boss API implementation
224	gutsche	1.24	def runBossCommand(cmd, printout=0, timeout=3600):
225	slacapra	1.11	"""
226			Cd to correct directory before running a boss command
227			"""
228			cwd = os.getcwd()
229			os.chdir(common.work_space.shareDir())
230			out = runCommand(cmd, printout, timeout)
231			os.chdir(cwd)
232			return out
233	spiga	1.9
234	nsmirnov	1.1	###########################################################################
235	slacapra	1.16	def readable(fd):
236			return bool(select.select([fd], [], [], 0))
237
238			###########################################################################
239			def makeNonBlocking(fd):
240			fl = fcntl.fcntl(fd, fcntl.F_GETFL)
241			try:
242			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.O_NDELAY)
243			except AttributeError:
244			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.FNDELAY)
245
246			###########################################################################
247	slacapra	1.11	def runCommand(cmd, printout=0, timeout=-1):
248	nsmirnov	1.1	"""
249			Run command 'cmd'.
250			Returns command stdoutput+stderror string on success,
251			or None if an error occurred.
252	slacapra	1.16	Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
253	nsmirnov	1.1	"""
254	fanzago	1.15
255	slacapra	1.11	if printout:
256			common.logger.message(cmd)
257			else:
258	slacapra	1.13	common.logger.debug(10,cmd)
259	slacapra	1.11	common.logger.write(cmd)
260	slacapra	1.29	pass
261	slacapra	1.11
262	slacapra	1.16	child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
263			child.tochild.close() # don't need to talk to child
264			outfile = child.fromchild
265			outfd = outfile.fileno()
266			errfile = child.childerr
267			errfd = errfile.fileno()
268			makeNonBlocking(outfd) # don't deadlock!
269			makeNonBlocking(errfd)
270			outdata = []
271			errdata = []
272			outeof = erreof = 0
273
274			if timeout > 0 :
275			maxwaittime = time.time() + timeout
276
277			err = -1
278			while (timeout == -1 or time.time() < maxwaittime):
279			ready = select.select([outfd,errfd],[],[]) # wait for input
280			if outfd in ready[0]:
281			outchunk = outfile.read()
282			if outchunk == '': outeof = 1
283			outdata.append(outchunk)
284			if errfd in ready[0]:
285			errchunk = errfile.read()
286			if errchunk == '': erreof = 1
287			errdata.append(errchunk)
288			if outeof and erreof:
289			err = child.wait()
290			break
291	slacapra	1.21	select.select([],[],[],.01) # give a little time for buffers to fill
292	slacapra	1.16	if err == -1:
293			# kill the pid
294	slacapra	1.18	common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
295	slacapra	1.16	os.kill (child.pid, 9)
296	slacapra	1.11	err = child.wait()
297	slacapra	1.16
298	fanzago	1.17	cmd_out = string.join(outdata,"")
299			cmd_err = string.join(errdata,"")
300
301	slacapra	1.11	if err:
302			common.logger.message('`'+cmd+'`\n failed with exit code '
303			+`err`+'='+`(err&0xff)`+'(signal)+'
304			+`(err>>8)`+'(status)')
305			common.logger.message(cmd_out)
306			common.logger.message(cmd_err)
307	nsmirnov	1.1	return None
308
309	fanzago	1.17	# cmd_out = string.join(outdata,"")
310			# cmd_err = string.join(errdata,"")
311	nsmirnov	1.1	cmd_out = cmd_out + cmd_err
312	slacapra	1.11	if printout:
313			common.logger.message(cmd_out)
314			else:
315	slacapra	1.13	common.logger.debug(10,cmd_out)
316	slacapra	1.11	common.logger.write(cmd_out)
317	slacapra	1.29	pass
318	slacapra	1.16	#print "<"+cmd_out+">"
319	nsmirnov	1.1	return cmd_out
320	nsmirnov	1.4
321	slacapra	1.11	####################################
322	gutsche	1.20	def makeCksum(filename) :
323			"""
324			make chksum using filename and content of file
325			"""
326
327	slacapra	1.29	try:
328			import tempfile
329			tmpfile= tempfile.NamedTemporaryFile(mode='w')
330			tmp_filename = tmpfile.name
331			except:
332			## SL in case py2.2 is used, fall back to old solution
333			tmp_filename = tempfile.mktemp()
334			tmpfile= open(tmp_filename,'w')
335			tobedeleted=1
336	gutsche	1.20
337			# add filename as first line
338	slacapra	1.29	tmpfile.write(filename+'\n')
339	gutsche	1.20
340			# fill input file in tmp file
341	slacapra	1.30	infile = open(filename)
342			tmpfile.writelines(infile.readlines())
343	slacapra	1.29	tmpfile.flush()
344	slacapra	1.30	infile.close()
345	gutsche	1.20
346			cmd = 'cksum '+tmp_filename
347			cmd_out = runCommand(cmd)
348
349			cksum = cmd_out.split()[0]
350
351	slacapra	1.29	## SL this is not needed if we use NamedTemporaryFile, which is not available in py2.2
352			if (tobedeleted): os.remove(tmp_filename)
353	gutsche	1.20
354			return cksum
355
356			####################################
357	nsmirnov	1.4	if __name__ == '__main__':
358			print 'sys.argv[1] =',sys.argv[1]
359			list = parseRange2(sys.argv[1])
360			print list
361	slacapra	1.29	cksum = makeCksum("crab_util.py")
362			print cksum
363	nsmirnov	1.4