ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab_util.py
Revision: 1.35
Committed: Wed Oct 10 12:55:58 2007 UTC (17 years, 6 months ago) by mcinquil
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_2_0_2_pre1, CRAB_2_0_1, CRAB_2_0_1_pre1
Changes since 1.34: +3 -1 lines
Log Message:
Allowing submission for tasks with a missing location for a block - this is a first version that block the creation and then the submision of jobs assigned to no location

File Contents

# User Rev Content
1 nsmirnov 1.1 ###########################################################################
2     #
3     # C O N V E N I E N C E F U N C T I O N S
4     #
5     ###########################################################################
6    
7 slacapra 1.12 import string, sys, os, time
8 slacapra 1.16 import ConfigParser, re, popen2, select, fcntl
9 nsmirnov 1.1
10 nsmirnov 1.2 import common
11 slacapra 1.30 from crab_exceptions import CrabException
12 nsmirnov 1.1
13     ###########################################################################
14     def parseOptions(argv):
15     """
16     Parses command-line options.
17     Returns a dictionary with specified options as keys:
18     -opt1 --> 'opt1' : None
19     -opt2 val --> 'opt2' : 'val'
20     -opt3=val --> 'opt3' : 'val'
21     Usually called as
22     options = parseOptions(sys.argv[1:])
23     """
24     options = {}
25     argc = len(argv)
26     i = 0
27     while ( i < argc ):
28     if argv[i][0] != '-':
29     i = i + 1
30     continue
31     eq = string.find(argv[i], '=')
32     if eq > 0 :
33     opt = argv[i][:eq]
34     val = argv[i][eq+1:]
35     pass
36     else:
37     opt = argv[i]
38     val = None
39     if ( i+1 < argc and argv[i+1][0] != '-' ):
40     i = i + 1
41     val = argv[i]
42     pass
43     pass
44     options[opt] = val
45     i = i + 1
46     pass
47     return options
48    
49 spiga 1.33 def definePath(isOriginal):
50     """
51     read and cache path
52     remove cmssw stuff from path
53     return original/modified path
54     """
55     bpath_original = os.getenv("PATH", None)
56     try:
57     cmssw = os.getenv("CMS_PATH", None)
58     paths = bpath_original.split(':')
59     bpath = ''
60     for p in paths:
61 spiga 1.34 if p.find( cmssw ) != -1 and p.find( 'python') != -1 and p.find( 'Crab') == -1:continue
62 spiga 1.33 bpath+= p + ':'
63     except:
64     import traceback
65     print traceback.format_exc()
66     print "No CMSSW"
67     if isOriginal == 'original':
68     return bpath_original
69     else:
70     return bpath[:-1]
71 nsmirnov 1.1 ###########################################################################
72     def loadConfig(file):
73     """
74     returns a dictionary with keys of the form
75     <section>.<option> and the corresponding values
76     """
77     config={}
78     cp = ConfigParser.ConfigParser()
79     cp.read(file)
80     for sec in cp.sections():
81     # print 'Section',sec
82     for opt in cp.options(sec):
83     #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
84     config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
85 corvo 1.19 # marco. Try to prevent user from switching off Monalisa reporting
86     config['USER.activate_monalisa'] = 1
87     # marco
88 nsmirnov 1.1 return config
89    
90     ###########################################################################
91     def isInt(str):
92     """ Is the given string an integer ?"""
93     try: int(str)
94     except ValueError: return 0
95     return 1
96    
97     ###########################################################################
98     def isBool(str):
99     """ Is the given string 0 or 1 ?"""
100     if (str in ('0','1')): return 1
101     return 0
102    
103     ###########################################################################
104 nsmirnov 1.3 def parseRange(range):
105     """
106     Takes as the input a string with two integers separated by
107     the minus sign and returns the tuple with these numbers:
108     'n1-n2' -> (n1, n2)
109     'n1' -> (n1, n1)
110     """
111     start = None
112     end = None
113     minus = string.find(range, '-')
114     if ( minus < 0 ):
115     if isInt(range):
116     start = int(range)
117     end = start
118     pass
119     pass
120     else:
121     if isInt(range[:minus]) and isInt(range[minus+1:]):
122     start = int(range[:minus])
123     end = int(range[minus+1:])
124     pass
125     pass
126     return (start, end)
127    
128     ###########################################################################
129 nsmirnov 1.4 def parseRange2(range):
130     """
131     Takes as the input a string in the form of a comma-separated
132     numbers and ranges
133     and returns a list with all specified numbers:
134     'n1' -> [n1]
135     'n1-n2' -> [n1, n1+1, ..., n2]
136     'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
137     """
138 slacapra 1.30 result = []
139     if not range: return result
140 nsmirnov 1.4
141     comma = string.find(range, ',')
142     if comma == -1: left = range
143     else: left = range[:comma]
144    
145     (n1, n2) = parseRange(left)
146     while ( n1 <= n2 ):
147 slacapra 1.11 try:
148 slacapra 1.30 result.append(n1)
149 slacapra 1.11 n1 += 1
150     pass
151     except:
152     msg = 'Syntax error in range <'+range+'>'
153     raise CrabException(msg)
154 nsmirnov 1.4
155     if comma != -1:
156 slacapra 1.11 try:
157 slacapra 1.30 result.extend(parseRange2(range[comma+1:]))
158 slacapra 1.11 pass
159     except:
160     msg = 'Syntax error in range <'+range+'>'
161     raise CrabException(msg)
162 nsmirnov 1.4
163 slacapra 1.30 return result
164 nsmirnov 1.4
165     ###########################################################################
166 nsmirnov 1.3 def crabJobStatusToString(crab_status):
167     """
168     Convert one-letter crab job status into more readable string.
169     """
170 corvo 1.27 status={
171     'H':'Hold',
172     'U':'Ready',
173     'I':'Scheduled',
174     'X':'Canceled',
175     'W':'Created',
176     'R':'Running',
177     'SC':'Checkpointed',
178     'SS':'Scheduled',
179     'SR':'Ready',
180     'RE':'Ready',
181     'SW':'Waiting',
182     'SU':'Submitted',
183     'S' :'Submitted (Boss)',
184     'UN':'Undefined',
185     'SK':'Cancelled',
186     'SD':'Done (Success)',
187     'SA':'Aborted',
188     'DA':'Done (Aborted)',
189     'SE':'Cleared',
190     'OR':'Done (Success)',
191     'A?':'Aborted',
192     'K':'Killed',
193     'E':'Cleared',
194     'Z':'Cleared (Corrupt)',
195     'NA':'Unknown',
196     'I?':'Idle',
197     'O?':'Done',
198     'R?':'Running'
199     }
200     # if crab_status == 'C': status = 'Created'
201     # elif crab_status == 'D': status = 'Done'
202     # elif crab_status == 'R': status = 'Submitted'#Should be running? ds
203     # elif crab_status == 'S': status = 'Submitted'
204     # elif crab_status == 'K': status = 'Killed'
205     # elif crab_status == 'X': status = 'None'
206     # elif crab_status == 'Y': status = 'Output retrieved'
207     # elif crab_status == 'A': status = 'Aborted'
208     # elif crab_status == 'RC': status = 'ReCreated'
209     # else: status = '???'
210     return status[crab_status]
211 nsmirnov 1.3
212     ###########################################################################
213 nsmirnov 1.1 def findLastWorkDir(dir_prefix, where = None):
214    
215     if not where: where = os.getcwd() + '/'
216     # dir_prefix usually has the form 'crab_0_'
217     pattern = re.compile(dir_prefix)
218    
219     file_list = []
220     for fl in os.listdir(where):
221     if pattern.match(fl):
222     file_list.append(fl)
223     pass
224     pass
225    
226     if len(file_list) == 0: return None
227    
228     file_list.sort()
229    
230     wdir = where + file_list[len(file_list)-1]
231     return wdir
232    
233     ###########################################################################
234     def importName(module_name, name):
235     """
236     Import a named object from a Python module,
237     i.e., it is an equivalent of 'from module_name import name'.
238     """
239     module = __import__(module_name, globals(), locals(), [name])
240     return vars(module)[name]
241    
242 spiga 1.9
243 slacapra 1.11 ###########################################################################
244 spiga 1.28
245 slacapra 1.29 ### WARNING This Function become USELESS after Boss API implementation
246 gutsche 1.24 def runBossCommand(cmd, printout=0, timeout=3600):
247 slacapra 1.11 """
248     Cd to correct directory before running a boss command
249     """
250     cwd = os.getcwd()
251     os.chdir(common.work_space.shareDir())
252     out = runCommand(cmd, printout, timeout)
253     os.chdir(cwd)
254     return out
255 spiga 1.9
256 nsmirnov 1.1 ###########################################################################
257 slacapra 1.16 def readable(fd):
258     return bool(select.select([fd], [], [], 0))
259    
260     ###########################################################################
261     def makeNonBlocking(fd):
262     fl = fcntl.fcntl(fd, fcntl.F_GETFL)
263     try:
264     fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
265     except AttributeError:
266     fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)
267    
268     ###########################################################################
269 slacapra 1.11 def runCommand(cmd, printout=0, timeout=-1):
270 nsmirnov 1.1 """
271     Run command 'cmd'.
272     Returns command stdoutput+stderror string on success,
273     or None if an error occurred.
274 slacapra 1.16 Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
275 nsmirnov 1.1 """
276 fanzago 1.15
277 slacapra 1.11 if printout:
278     common.logger.message(cmd)
279     else:
280 slacapra 1.13 common.logger.debug(10,cmd)
281 slacapra 1.11 common.logger.write(cmd)
282 slacapra 1.29 pass
283 slacapra 1.11
284 slacapra 1.16 child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
285     child.tochild.close() # don't need to talk to child
286     outfile = child.fromchild
287     outfd = outfile.fileno()
288     errfile = child.childerr
289     errfd = errfile.fileno()
290     makeNonBlocking(outfd) # don't deadlock!
291     makeNonBlocking(errfd)
292     outdata = []
293     errdata = []
294     outeof = erreof = 0
295    
296     if timeout > 0 :
297     maxwaittime = time.time() + timeout
298    
299     err = -1
300     while (timeout == -1 or time.time() < maxwaittime):
301     ready = select.select([outfd,errfd],[],[]) # wait for input
302     if outfd in ready[0]:
303     outchunk = outfile.read()
304     if outchunk == '': outeof = 1
305     outdata.append(outchunk)
306     if errfd in ready[0]:
307     errchunk = errfile.read()
308     if errchunk == '': erreof = 1
309     errdata.append(errchunk)
310     if outeof and erreof:
311     err = child.wait()
312     break
313 slacapra 1.21 select.select([],[],[],.01) # give a little time for buffers to fill
314 slacapra 1.16 if err == -1:
315     # kill the pid
316 slacapra 1.18 common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
317 slacapra 1.16 os.kill (child.pid, 9)
318 slacapra 1.11 err = child.wait()
319 slacapra 1.16
320 fanzago 1.17 cmd_out = string.join(outdata,"")
321     cmd_err = string.join(errdata,"")
322    
323 slacapra 1.11 if err:
324     common.logger.message('`'+cmd+'`\n failed with exit code '
325     +`err`+'='+`(err&0xff)`+'(signal)+'
326     +`(err>>8)`+'(status)')
327     common.logger.message(cmd_out)
328     common.logger.message(cmd_err)
329 nsmirnov 1.1 return None
330    
331 fanzago 1.17 # cmd_out = string.join(outdata,"")
332     # cmd_err = string.join(errdata,"")
333 nsmirnov 1.1 cmd_out = cmd_out + cmd_err
334 slacapra 1.11 if printout:
335     common.logger.message(cmd_out)
336     else:
337 slacapra 1.13 common.logger.debug(10,cmd_out)
338 slacapra 1.11 common.logger.write(cmd_out)
339 slacapra 1.29 pass
340 slacapra 1.16 #print "<"+cmd_out+">"
341 nsmirnov 1.1 return cmd_out
342 nsmirnov 1.4
343 slacapra 1.11 ####################################
344 gutsche 1.20 def makeCksum(filename) :
345     """
346     make chksum using filename and content of file
347     """
348    
349 slacapra 1.31 tobedeleted=0
350 slacapra 1.29 try:
351     import tempfile
352     tmpfile= tempfile.NamedTemporaryFile(mode='w')
353     tmp_filename = tmpfile.name
354     except:
355     ## SL in case py2.2 is used, fall back to old solution
356     tmp_filename = tempfile.mktemp()
357     tmpfile= open(tmp_filename,'w')
358     tobedeleted=1
359 gutsche 1.20
360     # add filename as first line
361 slacapra 1.29 tmpfile.write(filename+'\n')
362 gutsche 1.20
363     # fill input file in tmp file
364 slacapra 1.30 infile = open(filename)
365     tmpfile.writelines(infile.readlines())
366 slacapra 1.29 tmpfile.flush()
367 slacapra 1.30 infile.close()
368 gutsche 1.20
369     cmd = 'cksum '+tmp_filename
370     cmd_out = runCommand(cmd)
371    
372     cksum = cmd_out.split()[0]
373    
374 slacapra 1.29 ## SL this is not needed if we use NamedTemporaryFile, which is not available in py2.2
375     if (tobedeleted): os.remove(tmp_filename)
376 gutsche 1.20
377     return cksum
378    
379 gutsche 1.32 def spanRanges(jobArray):
380     """
381     take array of job numbers and concatenate 1,2,3 to 1-3
382     return string
383     """
384    
385     output = ""
386 mcinquil 1.35 jobArray.sort()
387    
388 gutsche 1.32 previous = jobArray[0]-1
389     for job in jobArray:
390     if previous+1 == job:
391     previous = job
392     if len(output) > 0 :
393     if output[-1] != "-":
394     output += "-"
395     else :
396     output += str(previous)
397     else:
398     output += str(previous) + "," + str(job)
399 mcinquil 1.35 #output += "," + str(job)
400 gutsche 1.32 previous = job
401     if len(jobArray) > 1 :
402     output += str(previous)
403    
404     return output
405    
406 gutsche 1.20 ####################################
407 nsmirnov 1.4 if __name__ == '__main__':
408     print 'sys.argv[1] =',sys.argv[1]
409     list = parseRange2(sys.argv[1])
410     print list
411 slacapra 1.29 cksum = makeCksum("crab_util.py")
412     print cksum
413 nsmirnov 1.4