CRAB/python/crab_util.py

###########################################################################
#
#   C O N V E N I E N C E    F U N C T I O N S
#
###########################################################################

import string, sys, os, time
import ConfigParser, re, popen2, select, fcntl

import common

###########################################################################
def parseOptions(argv):
    """
    Parses command-line options.
    Returns a dictionary with specified options as keys:
    -opt1             --> 'opt1' : None
    -opt2 val         --> 'opt2' : 'val'
    -opt3=val         --> 'opt3' : 'val'
    Usually called as
    options = parseOptions(sys.argv[1:])
    """
    options = {}
    argc = len(argv)
    i = 0
    while ( i < argc ):
        if argv[i][0] != '-':
            i = i + 1
            continue
        eq = string.find(argv[i], '=')
        if eq > 0 :
            opt = argv[i][:eq]
            val = argv[i][eq+1:]
            pass
        else:
            opt = argv[i]
            val = None
            if ( i+1 < argc and argv[i+1][0] != '-' ):
                i = i + 1
                val = argv[i]
                pass
            pass
        options[opt] = val
        i = i + 1
        pass
    return options

###########################################################################
def loadConfig(file):
    """
    returns a dictionary with keys of the form
    <section>.<option> and the corresponding values
    """
    config={}
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        # print 'Section',sec
        for opt in cp.options(sec):
            #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
            config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
    # marco. Try to prevent user from switching off Monalisa reporting
    config['USER.activate_monalisa'] = 1
    # marco
    return config

###########################################################################
def isInt(str):
    """ Is the given string an integer ?"""
    try: int(str)
    except ValueError: return 0
    return 1

###########################################################################
def isBool(str):
    """ Is the given string 0 or 1 ?"""
    if (str in ('0','1')): return 1
    return 0

###########################################################################
def parseRange(range):
    """
    Takes as the input a string with two integers separated by
    the minus sign and returns the tuple with these numbers:
    'n1-n2' -> (n1, n2)
    'n1'    -> (n1, n1)
    """
    start = None
    end   = None
    minus = string.find(range, '-')
    if ( minus < 0 ):
        if isInt(range):
            start = int(range)
            end   = start
            pass
        pass
    else:
        if isInt(range[:minus]) and isInt(range[minus+1:]):
            start = int(range[:minus])
            end   = int(range[minus+1:])
            pass
        pass
    return (start, end)

###########################################################################
def parseRange2(range):
    """
    Takes as the input a string in the form of a comma-separated
    numbers and ranges
    and returns a list with all specified numbers:
    'n1'          -> [n1]
    'n1-n2'       -> [n1, n1+1, ..., n2]
    'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
    """
    list = []
    if not range: return list

    comma = string.find(range, ',')
    if comma == -1: left = range
    else:           left = range[:comma]

    (n1, n2) = parseRange(left)
    while ( n1 <= n2 ):
        try:
            list.append(n1)
            n1 += 1
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    if comma != -1:
        try:
            list.extend(parseRange2(range[comma+1:]))
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    return list

###########################################################################
def crabJobStatusToString(crab_status):
    """
    Convert one-letter crab job status into more readable string.
    """
    if   crab_status == 'C': status = 'Created'
    elif crab_status == 'D': status = 'Done'
    elif crab_status == 'S': status = 'Submitted'
    elif crab_status == 'K': status = 'Killed'
    elif crab_status == 'X': status = 'None'
    elif crab_status == 'Y': status = 'Output retrieved'
    elif crab_status == 'A': status = 'Aborted'
    elif crab_status == 'RC': status = 'ReCreated'
    else: status = '???'
    return status

###########################################################################
def findLastWorkDir(dir_prefix, where = None):

    if not where: where = os.getcwd() + '/'
    # dir_prefix usually has the form 'crab_0_'
    pattern = re.compile(dir_prefix)

    file_list = []
    for fl in os.listdir(where):
        if pattern.match(fl):
            file_list.append(fl)
            pass
        pass

    if len(file_list) == 0: return None

    file_list.sort()

    wdir = where + file_list[len(file_list)-1]
    return wdir

###########################################################################
def importName(module_name, name):
    """
    Import a named object from a Python module,
    i.e., it is an equivalent of 'from module_name import name'.
    """
    module = __import__(module_name, globals(), locals(), [name])
    return vars(module)[name]


###########################################################################
def runBossCommand(cmd, printout=0, timeout=3600):
    """
    Cd to correct directory before running a boss command
    """
    cwd = os.getcwd()
    os.chdir(common.work_space.shareDir())
    out = runCommand(cmd, printout, timeout)
    os.chdir(cwd)
    return out

###########################################################################
def readable(fd):
    return bool(select.select([fd], [], [], 0)) 

###########################################################################
def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

###########################################################################
def runCommand(cmd, printout=0, timeout=-1):
    """
    Run command 'cmd'.
    Returns command stdoutput+stderror string on success,
    or None if an error occurred.
    Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
    """

    if printout:
        common.logger.message(cmd)
    else:
        common.logger.debug(10,cmd)
        common.logger.write(cmd)

    child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild 
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = []
    errdata = []
    outeof = erreof = 0

    if timeout > 0 :
        maxwaittime = time.time() + timeout

    err = -1
    while (timeout == -1 or time.time() < maxwaittime):
        ready = select.select([outfd,errfd],[],[]) # wait for input
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '': outeof = 1
            outdata.append(outchunk)
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '': erreof = 1
            errdata.append(errchunk)
        if outeof and erreof:
            err = child.wait()
            break
        select.select([],[],[],.01) # give a little time for buffers to fill
    if err == -1:
        # kill the pid
        common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
        os.kill (child.pid, 9)
        err = child.wait()

    cmd_out = string.join(outdata,"")
    cmd_err = string.join(errdata,"")

    if err:
        common.logger.message('`'+cmd+'`\n   failed with exit code '
                           +`err`+'='+`(err&0xff)`+'(signal)+'
                           +`(err>>8)`+'(status)')
        common.logger.message(cmd_out)
        common.logger.message(cmd_err)
        return None

#    cmd_out = string.join(outdata,"")
#    cmd_err = string.join(errdata,"")
    cmd_out = cmd_out + cmd_err
    if printout:
        common.logger.message(cmd_out)
    else:
        common.logger.debug(10,cmd_out)
        common.logger.write(cmd_out)
    #print "<"+cmd_out+">"
    return cmd_out

####################################
def makeCksum(filename) :
    """
    make chksum using filename and content of file
    """

    tmp_filename = runCommand('mktemp crab_hash_XXXXXXXXXXXX').strip()
    output = open(tmp_filename,'w')

    # add filename as first line
    output.write(filename+'\n')

    # fill input file in tmp file
    input = open(filename)
    input_line = input.readline()
    while input_line :
        output.write(input_line+'\n')
        input_line = input.readline()
    output.close()


    cmd = 'cksum '+tmp_filename
    cmd_out = runCommand(cmd)

    cksum = cmd_out.split()[0]

    # delete file
    runCommand('rm -f '+tmp_filename)

    return cksum

####################################
if __name__ == '__main__':
    import sys
    print 'sys.argv[1] =',sys.argv[1]
    list = parseRange2(sys.argv[1])
    print list
    
Revision:	1.24
Committed:	Mon Oct 9 23:22:44 2006 UTC (18 years, 6 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_4_0_pre4, CRAB_1_4_0_pre3, CRAB_1_4_0_pre2
Changes since 1.23:	+1 -1 lines
Log Message:	increase timeout for runBossCommand to 1 hour. Due to the group submission, a boss submit can take longer than 10 minutes
#	User	Rev	Content
1	nsmirnov	1.1	###########################################################################
2			#
3			# C O N V E N I E N C E F U N C T I O N S
4			#
5			###########################################################################
6
7	slacapra	1.12	import string, sys, os, time
8	slacapra	1.16	import ConfigParser, re, popen2, select, fcntl
9	nsmirnov	1.1
10	nsmirnov	1.2	import common
11	nsmirnov	1.1
12			###########################################################################
13			def parseOptions(argv):
14			"""
15			Parses command-line options.
16			Returns a dictionary with specified options as keys:
17			-opt1 --> 'opt1' : None
18			-opt2 val --> 'opt2' : 'val'
19			-opt3=val --> 'opt3' : 'val'
20			Usually called as
21			options = parseOptions(sys.argv[1:])
22			"""
23			options = {}
24			argc = len(argv)
25			i = 0
26			while ( i < argc ):
27			if argv[i][0] != '-':
28			i = i + 1
29			continue
30			eq = string.find(argv[i], '=')
31			if eq > 0 :
32			opt = argv[i][:eq]
33			val = argv[i][eq+1:]
34			pass
35			else:
36			opt = argv[i]
37			val = None
38			if ( i+1 < argc and argv[i+1][0] != '-' ):
39			i = i + 1
40			val = argv[i]
41			pass
42			pass
43			options[opt] = val
44			i = i + 1
45			pass
46			return options
47
48			###########################################################################
49			def loadConfig(file):
50			"""
51			returns a dictionary with keys of the form
52			<section>.<option> and the corresponding values
53			"""
54			config={}
55			cp = ConfigParser.ConfigParser()
56			cp.read(file)
57			for sec in cp.sections():
58			# print 'Section',sec
59			for opt in cp.options(sec):
60			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
61			config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
62	corvo	1.19	# marco. Try to prevent user from switching off Monalisa reporting
63			config['USER.activate_monalisa'] = 1
64			# marco
65	nsmirnov	1.1	return config
66
67			###########################################################################
68			def isInt(str):
69			""" Is the given string an integer ?"""
70			try: int(str)
71			except ValueError: return 0
72			return 1
73
74			###########################################################################
75			def isBool(str):
76			""" Is the given string 0 or 1 ?"""
77			if (str in ('0','1')): return 1
78			return 0
79
80			###########################################################################
81	nsmirnov	1.3	def parseRange(range):
82			"""
83			Takes as the input a string with two integers separated by
84			the minus sign and returns the tuple with these numbers:
85			'n1-n2' -> (n1, n2)
86			'n1' -> (n1, n1)
87			"""
88			start = None
89			end = None
90			minus = string.find(range, '-')
91			if ( minus < 0 ):
92			if isInt(range):
93			start = int(range)
94			end = start
95			pass
96			pass
97			else:
98			if isInt(range[:minus]) and isInt(range[minus+1:]):
99			start = int(range[:minus])
100			end = int(range[minus+1:])
101			pass
102			pass
103			return (start, end)
104
105			###########################################################################
106	nsmirnov	1.4	def parseRange2(range):
107			"""
108			Takes as the input a string in the form of a comma-separated
109			numbers and ranges
110			and returns a list with all specified numbers:
111			'n1' -> [n1]
112			'n1-n2' -> [n1, n1+1, ..., n2]
113			'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
114			"""
115			list = []
116			if not range: return list
117
118			comma = string.find(range, ',')
119			if comma == -1: left = range
120			else: left = range[:comma]
121
122			(n1, n2) = parseRange(left)
123			while ( n1 <= n2 ):
124	slacapra	1.11	try:
125			list.append(n1)
126			n1 += 1
127			pass
128			except:
129			msg = 'Syntax error in range <'+range+'>'
130			raise CrabException(msg)
131	nsmirnov	1.4
132			if comma != -1:
133	slacapra	1.11	try:
134			list.extend(parseRange2(range[comma+1:]))
135			pass
136			except:
137			msg = 'Syntax error in range <'+range+'>'
138			raise CrabException(msg)
139	nsmirnov	1.4
140			return list
141
142			###########################################################################
143	nsmirnov	1.3	def crabJobStatusToString(crab_status):
144			"""
145			Convert one-letter crab job status into more readable string.
146			"""
147			if crab_status == 'C': status = 'Created'
148	slacapra	1.6	elif crab_status == 'D': status = 'Done'
149	nsmirnov	1.3	elif crab_status == 'S': status = 'Submitted'
150			elif crab_status == 'K': status = 'Killed'
151			elif crab_status == 'X': status = 'None'
152			elif crab_status == 'Y': status = 'Output retrieved'
153	slacapra	1.5	elif crab_status == 'A': status = 'Aborted'
154	spiga	1.7	elif crab_status == 'RC': status = 'ReCreated'
155	nsmirnov	1.3	else: status = '???'
156			return status
157
158			###########################################################################
159	nsmirnov	1.1	def findLastWorkDir(dir_prefix, where = None):
160
161			if not where: where = os.getcwd() + '/'
162			# dir_prefix usually has the form 'crab_0_'
163			pattern = re.compile(dir_prefix)
164
165			file_list = []
166			for fl in os.listdir(where):
167			if pattern.match(fl):
168			file_list.append(fl)
169			pass
170			pass
171
172			if len(file_list) == 0: return None
173
174			file_list.sort()
175
176			wdir = where + file_list[len(file_list)-1]
177			return wdir
178
179			###########################################################################
180			def importName(module_name, name):
181			"""
182			Import a named object from a Python module,
183			i.e., it is an equivalent of 'from module_name import name'.
184			"""
185			module = __import__(module_name, globals(), locals(), [name])
186			return vars(module)[name]
187
188	spiga	1.9
189	slacapra	1.11	###########################################################################
190	gutsche	1.24	def runBossCommand(cmd, printout=0, timeout=3600):
191	slacapra	1.11	"""
192			Cd to correct directory before running a boss command
193			"""
194			cwd = os.getcwd()
195			os.chdir(common.work_space.shareDir())
196			out = runCommand(cmd, printout, timeout)
197			os.chdir(cwd)
198			return out
199	spiga	1.9
200	nsmirnov	1.1	###########################################################################
201	slacapra	1.16	def readable(fd):
202			return bool(select.select([fd], [], [], 0))
203
204			###########################################################################
205			def makeNonBlocking(fd):
206			fl = fcntl.fcntl(fd, fcntl.F_GETFL)
207			try:
208			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.O_NDELAY)
209			except AttributeError:
210			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.FNDELAY)
211
212			###########################################################################
213	slacapra	1.11	def runCommand(cmd, printout=0, timeout=-1):
214	nsmirnov	1.1	"""
215			Run command 'cmd'.
216			Returns command stdoutput+stderror string on success,
217			or None if an error occurred.
218	slacapra	1.16	Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
219	nsmirnov	1.1	"""
220	fanzago	1.15
221	slacapra	1.11	if printout:
222			common.logger.message(cmd)
223			else:
224	slacapra	1.13	common.logger.debug(10,cmd)
225	slacapra	1.11	common.logger.write(cmd)
226
227	slacapra	1.16	child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
228			child.tochild.close() # don't need to talk to child
229			outfile = child.fromchild
230			outfd = outfile.fileno()
231			errfile = child.childerr
232			errfd = errfile.fileno()
233			makeNonBlocking(outfd) # don't deadlock!
234			makeNonBlocking(errfd)
235			outdata = []
236			errdata = []
237			outeof = erreof = 0
238
239			if timeout > 0 :
240			maxwaittime = time.time() + timeout
241
242			err = -1
243			while (timeout == -1 or time.time() < maxwaittime):
244			ready = select.select([outfd,errfd],[],[]) # wait for input
245			if outfd in ready[0]:
246			outchunk = outfile.read()
247			if outchunk == '': outeof = 1
248			outdata.append(outchunk)
249			if errfd in ready[0]:
250			errchunk = errfile.read()
251			if errchunk == '': erreof = 1
252			errdata.append(errchunk)
253			if outeof and erreof:
254			err = child.wait()
255			break
256	slacapra	1.21	select.select([],[],[],.01) # give a little time for buffers to fill
257	slacapra	1.16	if err == -1:
258			# kill the pid
259	slacapra	1.18	common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
260	slacapra	1.16	os.kill (child.pid, 9)
261	slacapra	1.11	err = child.wait()
262	slacapra	1.16
263	fanzago	1.17	cmd_out = string.join(outdata,"")
264			cmd_err = string.join(errdata,"")
265
266	slacapra	1.11	if err:
267			common.logger.message('`'+cmd+'`\n failed with exit code '
268			+`err`+'='+`(err&0xff)`+'(signal)+'
269			+`(err>>8)`+'(status)')
270			common.logger.message(cmd_out)
271			common.logger.message(cmd_err)
272	nsmirnov	1.1	return None
273
274	fanzago	1.17	# cmd_out = string.join(outdata,"")
275			# cmd_err = string.join(errdata,"")
276	nsmirnov	1.1	cmd_out = cmd_out + cmd_err
277	slacapra	1.11	if printout:
278			common.logger.message(cmd_out)
279			else:
280	slacapra	1.13	common.logger.debug(10,cmd_out)
281	slacapra	1.11	common.logger.write(cmd_out)
282	slacapra	1.16	#print "<"+cmd_out+">"
283	nsmirnov	1.1	return cmd_out
284	nsmirnov	1.4
285	slacapra	1.11	####################################
286	gutsche	1.20	def makeCksum(filename) :
287			"""
288			make chksum using filename and content of file
289			"""
290
291	gutsche	1.23	tmp_filename = runCommand('mktemp crab_hash_XXXXXXXXXXXX').strip()
292	gutsche	1.20	output = open(tmp_filename,'w')
293
294			# add filename as first line
295			output.write(filename+'\n')
296
297			# fill input file in tmp file
298			input = open(filename)
299			input_line = input.readline()
300			while input_line :
301			output.write(input_line+'\n')
302			input_line = input.readline()
303			output.close()
304
305
306			cmd = 'cksum '+tmp_filename
307			cmd_out = runCommand(cmd)
308
309			cksum = cmd_out.split()[0]
310
311			# delete file
312			runCommand('rm -f '+tmp_filename)
313
314			return cksum
315
316			####################################
317	nsmirnov	1.4	if __name__ == '__main__':
318			import sys
319			print 'sys.argv[1] =',sys.argv[1]
320			list = parseRange2(sys.argv[1])
321			print list
322