ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.43
Committed: Thu Dec 15 15:28:51 2005 UTC (19 years, 4 months ago) by fanzago
Content type: text/x-python
Branch: MAIN
Changes since 1.42: +2 -1 lines
Log Message: 
added other status

File Contents

# User Rev Content
1 slacapra 1.27 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 corvo 1.42 from ApmonIf import ApmonIf
16 slacapra 1.36 from Cleaner import Cleaner
17 nsmirnov 1.1 import common
18 spiga 1.13 import Statistic
19 nsmirnov 1.1
20     import sys, os, time, string
21    
22     ###########################################################################
23     class Crab:
24 nsmirnov 1.3 def __init__(self, opts):
25 nsmirnov 1.1
26     # The order of main_actions is important !
27 slacapra 1.28 self.main_actions = [ '-create', '-submit' ]
28 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
29 slacapra 1.28 '-resubmit' , '-cancelAndResubmit', '-testJdl', '-postMortem', '-clean']
30 nsmirnov 1.1
31     # Dictionary of actions, e.g. '-create' -> object of class Creator
32     self.actions = {}
33    
34     # Configuration file
35     self.cfg_fname = None
36     # Dictionary with configuration parameters
37     self.cfg_params = {}
38    
39     # Current working directory
40     self.cwd = os.getcwd()+'/'
41     # Current time in format 'yymmdd_hhmmss'
42     self.current_time = time.strftime('%y%m%d_%H%M%S',
43     time.localtime(time.time()))
44    
45 nsmirnov 1.3 # Session name (?) Do we need this ?
46 nsmirnov 1.1 self.name = '0'
47    
48     # Job type
49     self.job_type_name = None
50    
51     # Continuation flag
52     self.flag_continue = 0
53    
54     # quiet mode, i.e. no output on screen
55     self.flag_quiet = 0
56     # produce more output
57     self.debug_level = 0
58    
59 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
60 fanzago 1.14 self.scheduler_name = ''
61 nsmirnov 1.1
62 nsmirnov 1.3 self.initialize_(opts)
63 nsmirnov 1.1
64     return
65    
66 nsmirnov 1.7 def version():
67 nsmirnov 1.1 return common.prog_version_str
68    
69 nsmirnov 1.7 version = staticmethod(version)
70    
71 nsmirnov 1.3 def initialize_(self, opts):
72 nsmirnov 1.1
73     # Process the '-continue' option first because
74     # in the case of continuation the CRAB configuration
75     # parameters are loaded from already existing Working Space.
76 nsmirnov 1.3 self.processContinueOption_(opts)
77 nsmirnov 1.1
78     # Process ini-file first, then command line options
79     # because they override ini-file settings.
80    
81 nsmirnov 1.3 self.processIniFile_(opts)
82 nsmirnov 1.1
83 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
84 nsmirnov 1.1
85 nsmirnov 1.3 self.processOptions_(opts)
86 nsmirnov 1.1
87     if not self.flag_continue:
88 nsmirnov 1.3 self.createWorkingSpace_()
89 slacapra 1.9 optsToBeSaved={}
90     for it in opts.keys():
91     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
92     pass
93     else:
94     optsToBeSaved[it]=opts[it]
95     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
96 nsmirnov 1.1 pass
97    
98     # At this point all configuration options have been read.
99    
100     args = string.join(sys.argv,' ')
101 slacapra 1.11
102 nsmirnov 1.3 self.updateHistory_(args)
103 slacapra 1.11
104 nsmirnov 1.3 self.createLogger_(args)
105 slacapra 1.11
106 nsmirnov 1.1 common.jobDB = JobDB()
107 slacapra 1.11
108 nsmirnov 1.3 if self.flag_continue:
109 slacapra 1.12 try:
110     common.jobDB.load()
111     common.logger.debug(6, str(common.jobDB))
112     except DBException,e:
113     pass
114 nsmirnov 1.3 pass
115 slacapra 1.11
116 nsmirnov 1.3 self.createScheduler_()
117 slacapra 1.11
118 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
119     common.logger.debug(6, 'Used properties:')
120     keys = self.cfg_params.keys()
121     keys.sort()
122     for k in keys:
123     if self.cfg_params[k]:
124 corvo 1.42 common.logger.debug(6, ' '+k+' : '+str(self.cfg_params[k]))
125 nsmirnov 1.3 pass
126     else:
127     common.logger.debug(6, ' '+k+' : ')
128     pass
129     pass
130     common.logger.debug(6, 'End of used properties.\n')
131     pass
132     self.initializeActions_(opts)
133 nsmirnov 1.1 return
134    
135 nsmirnov 1.3 def processContinueOption_(self,opts):
136 nsmirnov 1.1
137     continue_dir = None
138 nsmirnov 1.4
139     # Look for the '-continue' option.
140    
141 nsmirnov 1.1 for opt in opts.keys():
142     if ( opt in ('-continue','-c') ):
143     self.flag_continue = 1
144     val = opts[opt]
145     if val:
146     if val[0] == '/': continue_dir = val # abs path
147     else: continue_dir = self.cwd + val # rel path
148     pass
149 nsmirnov 1.4 break
150     pass
151    
152     # Look for actions which has sense only with '-continue'
153    
154     if not self.flag_continue:
155     for opt in opts.keys():
156 slacapra 1.6 if ( opt in (self.aux_actions) ):
157 nsmirnov 1.4 self.flag_continue = 1
158     break
159 nsmirnov 1.1 pass
160     pass
161 slacapra 1.6 submit_flag=0
162     create_flag=0
163     for opt in opts.keys():
164     if opt == "-submit": submit_flag=1
165     if opt == "-create": create_flag=1
166     pass
167     if (submit_flag and not create_flag):
168 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
169     raise CrabException(msg)
170 slacapra 1.6 pass
171 nsmirnov 1.1
172     if not self.flag_continue: return
173    
174     if not continue_dir:
175     prefix = common.prog_name + '_' + self.name + '_'
176     continue_dir = findLastWorkDir(prefix)
177     pass
178    
179     if not continue_dir:
180     raise CrabException('Cannot find last working directory.')
181    
182     if not os.path.exists(continue_dir):
183     msg = 'Cannot continue because the working directory <'
184     msg += continue_dir
185     msg += '> does not exist.'
186     raise CrabException(msg)
187    
188     # Instantiate WorkSpace
189     common.work_space = WorkSpace(continue_dir)
190    
191     return
192    
193 nsmirnov 1.3 def processIniFile_(self, opts):
194 nsmirnov 1.1 """
195     Processes a configuration INI-file.
196     """
197    
198     # Extract cfg-file name from the cmd-line options.
199    
200     for opt in opts.keys():
201     if ( opt == '-cfg' ):
202     if self.flag_continue:
203     raise CrabException('-continue and -cfg cannot coexist.')
204     if opts[opt] : self.cfg_fname = opts[opt]
205     else : usage()
206     pass
207    
208     elif ( opt == '-name' ):
209     self.name = opts[opt]
210     pass
211    
212     pass
213    
214     # Set default cfg-fname
215    
216     if self.cfg_fname == None:
217     if self.flag_continue:
218     self.cfg_fname = common.work_space.cfgFileName()
219     else:
220     self.cfg_fname = common.prog_name+'.cfg'
221     pass
222     pass
223    
224     # Load cfg-file
225    
226     if string.lower(self.cfg_fname) != 'none':
227     if os.path.exists(self.cfg_fname):
228     self.cfg_params = loadConfig(self.cfg_fname)
229 corvo 1.42 # Better idea on where to put user info for ML?
230     self.cfg_params['user'] = os.getlogin()
231     # Better idea on where to put ML???? Looks crap here, but had no better thought!
232     if int(self.cfg_params['USER.activate_monalisa']): self.cfg_params['apmon'] = ApmonIf()
233 nsmirnov 1.1 pass
234     else:
235     msg = 'cfg-file '+self.cfg_fname+' not found.'
236     raise CrabException(msg)
237     pass
238     pass
239    
240     # process the [CRAB] section
241    
242     lhp = len('CRAB.')
243     for k in self.cfg_params.keys():
244     if len(k) >= lhp and k[:lhp] == 'CRAB.':
245     opt = '-'+k[lhp:]
246     if len(opt) >= 3 and opt[:3] == '-__': continue
247     if opt not in opts.keys():
248     opts[opt] = self.cfg_params[k]
249     pass
250     pass
251     pass
252    
253     return
254    
255 nsmirnov 1.3 def processOptions_(self, opts):
256 nsmirnov 1.1 """
257     Processes the command-line options.
258     """
259    
260     for opt in opts.keys():
261     val = opts[opt]
262    
263 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
264     if opt in self.main_actions:
265     self.cfg_params['CRAB.'+opt[1:]] = val
266     continue
267     if opt in self.aux_actions:
268     self.cfg_params['CRAB.'+opt[1:]] = val
269     continue
270 nsmirnov 1.1
271    
272     elif ( opt == '-cfg' ):
273     pass
274    
275     elif ( opt in ('-continue', '-c') ):
276 nsmirnov 1.4 # Already processed in processContinueOption_()
277 nsmirnov 1.1 pass
278    
279     elif ( opt == '-jobtype' ):
280     if val : self.job_type_name = string.upper(val)
281     else : usage()
282     pass
283    
284     elif ( opt == '-Q' ):
285     self.flag_quiet = 1
286     pass
287    
288     elif ( opt == '-debug' ):
289 slacapra 1.6 if val: self.debug_level = int(val)
290     else: self.debug_level = 1
291 nsmirnov 1.1 pass
292    
293     elif ( opt == '-scheduler' ):
294     if val: self.scheduler_name = val
295     else:
296     print common.prog_name+". No value for '-scheduler'."
297     usage()
298     pass
299     pass
300 slacapra 1.22
301 fanzago 1.18 elif ( opt in ('-use_boss', '-useboss') ):
302     if ( val == '1' ):
303     self.scheduler_name = 'boss'
304     pass
305     elif ( val == '0' ):
306     pass
307     else:
308     print common.prog_name+'. Bad flag for -use_boss option:',\
309     val,'Possible values are 0(=No) or 1(=Yes)'
310     usage()
311     pass
312 fanzago 1.14 pass
313    
314 nsmirnov 1.3 elif string.find(opt,'.') == -1:
315     print common.prog_name+'. Unrecognized option '+opt
316     usage()
317     pass
318 nsmirnov 1.1
319 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
320     if string.find(opt,'.') == -1 :
321     self.cfg_params['CRAB.'+opt[1:]] = val
322 nsmirnov 1.1 pass
323 nsmirnov 1.3 else:
324 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
325     self.cfg_params[opt[1:]] = val
326     pass
327     pass
328     return
329    
330 slacapra 1.8 def parseRange_(self, aRange):
331 nsmirnov 1.4 """
332 slacapra 1.8 Takes as the input a string with a range defined in any of the following
333     way, including combination, and return a tuple with the ints defined
334     according to following table. A consistency check is done.
335     NB: the first job is "1", not "0".
336     'all' -> [1,2,..., NJobs]
337     '' -> [1,2,..., NJobs]
338     'n1' -> [n1]
339     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
340     'n1,n2' -> [n1, n2]
341     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
342     """
343     result = []
344    
345 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
346     if aRange=='all' or aRange==None or aRange=='':
347 slacapra 1.8 result=range(0,common.jobDB.nJobs())
348     return result
349 slacapra 1.9 elif aRange=='0':
350     return result
351 slacapra 1.8
352     subRanges = string.split(aRange, ',')
353     for subRange in subRanges:
354     result = result+self.parseSimpleRange_(subRange)
355    
356     if self.checkUniqueness_(result):
357     return result
358     else:
359 slacapra 1.33 common.logger.message("Error "+result)
360 slacapra 1.8 return []
361    
362     def checkUniqueness_(self, list):
363     """
364 slacapra 1.9 check if a list contains only unique elements
365 slacapra 1.8 """
366    
367     uniqueList = []
368     # use a list comprehension statement (takes a while to understand)
369    
370     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
371    
372     return (len(list)==len(uniqueList))
373    
374     def parseSimpleRange_(self, aRange):
375     """
376     Takes as the input a string with two integers separated by
377     the minus sign and returns the tuple with these numbers:
378     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
379     'n1' -> [n1]
380     """
381     (start, end) = (None, None)
382    
383     result = []
384     minus = string.find(aRange, '-')
385     if ( minus < 0 ):
386     if isInt(aRange) and int(aRange)>0:
387 fanzago 1.37 # FEDE
388     #result.append(int(aRange)-1)
389     ###
390     result.append(aRange)
391 slacapra 1.8 else:
392 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
393 nsmirnov 1.4 pass
394     else:
395 slacapra 1.8 (start, end) = string.split(aRange, '-')
396     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
397 spiga 1.38 #result=range(int(start)-1, int(end))
398     result=range(int(start), int(end)+1) #Daniele
399 slacapra 1.8 else:
400 slacapra 1.33 common.logger.message("parseSimpleRange_ ERROR "+start+end)
401 slacapra 1.8
402     return result
403 nsmirnov 1.4
404 nsmirnov 1.3 def initializeActions_(self, opts):
405 nsmirnov 1.1 """
406     For each user action instantiate a corresponding
407     object and put it in the action dictionary.
408     """
409     for opt in opts.keys():
410 spiga 1.32 self.flag_useboss = 0
411 spiga 1.16 if ( opt == '-use_boss'):
412 slacapra 1.25 val = opts[opt]
413     if ( val == '1' ):
414     self.flag_useboss = 1
415     common.logger.message('Using BOSS')
416     pass
417     else:
418     self.flag_useboss = 0
419     pass
420 spiga 1.15
421     for opt in opts.keys():
422    
423 nsmirnov 1.1 val = opts[opt]
424 spiga 1.15
425    
426     if ( opt == '-create' ):
427 slacapra 1.22 ncjobs = 0
428 nsmirnov 1.1 if val:
429     if ( isInt(val) ):
430     ncjobs = int(val)
431     elif ( val == 'all'):
432     ncjobs = val
433     else:
434 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
435     msg += ' Must be an integer or "all"'
436 slacapra 1.8 msg += ' Generic range is not allowed"'
437 nsmirnov 1.5 raise CrabException(msg)
438 nsmirnov 1.1 pass
439     else: ncjobs = 'all'
440    
441     if ncjobs != 0:
442 nsmirnov 1.3 # Instantiate Creator object
443 nsmirnov 1.1 creator = Creator(self.job_type_name,
444     self.cfg_params,
445     ncjobs)
446     self.actions[opt] = creator
447 nsmirnov 1.3
448     # Initialize the JobDB object if needed
449 nsmirnov 1.1 if not self.flag_continue:
450     common.jobDB.create(creator.nJobs())
451     pass
452 nsmirnov 1.3
453     # Create and initialize JobList
454    
455     common.job_list = JobList(common.jobDB.nJobs(),
456     creator.jobType())
457    
458     common.job_list.setScriptNames(self.job_type_name+'.sh')
459     common.job_list.setJDLNames(self.job_type_name+'.jdl')
460 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
461 slacapra 1.9
462     creator.writeJobsSpecsToDB()
463 nsmirnov 1.1 pass
464 nsmirnov 1.3 pass
465 nsmirnov 1.1
466     elif ( opt == '-submit' ):
467 slacapra 1.23
468     # total jobs
469     # get the first not already submitted
470 slacapra 1.24 common.logger.debug(5,'Total jobs '+str(common.jobDB.nJobs()))
471     lastSubmittedJob=0
472     for nj in range(common.jobDB.nJobs()):
473 fanzago 1.43 #if (common.jobDB.status(nj)=='S') or (common.jobDB.status(nj)=='K') or (common.jobDB.status(nj)=='A'): #Da
474     if (common.jobDB.status(nj) in ['S', 'K', 'RC', 'Y', 'A']):
475 slacapra 1.24 lastSubmittedJob +=1
476 slacapra 1.30 else: break
477 slacapra 1.24 # count job from 1
478 slacapra 1.30 totalJobsSubmittable = common.jobDB.nJobs()-lastSubmittedJob
479 slacapra 1.24 common.logger.debug(5,'lastSubmittedJob '+str(lastSubmittedJob))
480     common.logger.debug(5,'totalJobsSubmittable '+str(totalJobsSubmittable))
481 slacapra 1.23
482 slacapra 1.24 nsjobs = lastSubmittedJob+totalJobsSubmittable
483 slacapra 1.23 # get user request
484 slacapra 1.22 if val:
485     if ( isInt(val) ):
486 slacapra 1.24 tmp = int(val)
487     if (tmp >= totalJobsSubmittable):
488     common.logger.message('asking to submit '+str(tmp)+' jobs, but only '+str(totalJobsSubmittable)+' left: submitting those')
489     pass
490     else:
491     nsjobs=lastSubmittedJob+int(val)
492 slacapra 1.23 elif (val=='all'):
493     pass
494 slacapra 1.22 else:
495     msg = 'Bad submission option <'+str(val)+'>\n'
496     msg += ' Must be an integer or "all"'
497     msg += ' Generic range is not allowed"'
498     raise CrabException(msg)
499     pass
500 slacapra 1.24 common.logger.debug(5,'nsjobs '+str(nsjobs))
501 slacapra 1.23
502     # submit N from last submitted job
503 slacapra 1.24 nj_list = range(lastSubmittedJob, nsjobs)
504     common.logger.debug(5,'nj_list '+str(nj_list))
505 nsmirnov 1.5
506     if len(nj_list) != 0:
507 nsmirnov 1.3 # Instantiate Submitter object
508 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
509 nsmirnov 1.3
510     # Create and initialize JobList
511     if len(common.job_list) == 0 :
512     common.job_list = JobList(common.jobDB.nJobs(),
513     None)
514     common.job_list.setJDLNames(self.job_type_name+'.jdl')
515     pass
516 nsmirnov 1.1 pass
517 slacapra 1.30 pass
518 nsmirnov 1.1
519 nsmirnov 1.4 elif ( opt == '-list' ):
520 slacapra 1.8 jobs = self.parseRange_(val)
521    
522     common.jobDB.dump(jobs)
523 nsmirnov 1.4 pass
524    
525     elif ( opt == '-status' ):
526 slacapra 1.8 jobs = self.parseRange_(val)
527    
528 slacapra 1.9 if len(jobs) != 0:
529 spiga 1.16 if ( self.flag_useboss == 1 ):
530 corvo 1.42 self.actions[opt] = StatusBoss(self.cfg_params)
531 spiga 1.15 else:
532 slacapra 1.36 self.actions[opt] = Status(jobs)
533 spiga 1.15 pass
534 nsmirnov 1.1 pass
535     pass
536 slacapra 1.22
537 nsmirnov 1.4 elif ( opt == '-kill' ):
538 slacapra 1.8
539 spiga 1.31 if ( self.flag_useboss == 1 ):
540     if val:
541     if val =='all':
542 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
543     jobs = allBoss_id.keys()
544 spiga 1.31 else:
545     jobs = self.parseRange_(val)
546     common.scheduler.cancel(jobs)
547     else:
548     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
549     else:
550     if val:
551     jobs = self.parseRange_(val)
552    
553     for nj in jobs:
554     st = common.jobDB.status(nj)
555     if st == 'S':
556     jid = common.jobDB.jobId(nj)
557     common.logger.message("Killing job # "+`(nj+1)`)
558     common.scheduler.cancel(jid)
559     common.jobDB.setStatus(nj, 'K')
560     pass
561 slacapra 1.9 pass
562 spiga 1.31
563     common.jobDB.save()
564 nsmirnov 1.4 pass
565 spiga 1.31 else:
566     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
567 nsmirnov 1.1
568 slacapra 1.8 elif ( opt == '-getoutput' ):
569    
570 spiga 1.20 if ( self.flag_useboss == 1 ):
571     if val=='all' or val==None or val=='':
572 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
573     jobs = allBoss_id.keys()
574 spiga 1.20 else:
575     jobs = self.parseRange_(val)
576     common.scheduler.getOutput(jobs)
577     else:
578     jobs = self.parseRange_(val)
579 spiga 1.13
580 slacapra 1.22 ## also this: create a ActorClass (GetOutput)
581 spiga 1.20 jobs_done = []
582     for nj in jobs:
583     st = common.jobDB.status(nj)
584     if st == 'D':
585 slacapra 1.12 jobs_done.append(nj)
586 spiga 1.20 pass
587     elif st == 'S':
588     jid = common.jobDB.jobId(nj)
589     currStatus = common.scheduler.queryStatus(jid)
590     if currStatus=="Done":
591     jobs_done.append(nj)
592     else:
593     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
594     common.logger.message(msg)
595     pass
596 slacapra 1.9 else:
597 spiga 1.20 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
598     pass
599 slacapra 1.12 pass
600    
601 spiga 1.20 for nj in jobs_done:
602 spiga 1.15 jid = common.jobDB.jobId(nj)
603     dir = common.scheduler.getOutput(jid)
604     common.jobDB.setStatus(nj, 'Y')
605 slacapra 1.12
606     # Rename the directory with results to smth readable
607     new_dir = common.work_space.resDir()
608 spiga 1.20 try:
609     files = os.listdir(dir)
610     for file in files:
611     os.rename(dir+'/'+file, new_dir+'/'+file)
612     os.rmdir(dir)
613     except OSError, e:
614     msg = 'rename files from '+dir+' to '+new_dir+' error: '
615     msg += str(e)
616     common.logger.message(msg)
617     # ignore error
618 slacapra 1.12 pass
619 spiga 1.20 pass
620 slacapra 1.22 ###
621    
622 spiga 1.13 resFlag = 0
623     exCode = common.scheduler.getExitStatus(jid)
624 spiga 1.34 Statistic.Monitor('retrieved',resFlag,jid,exCode)
625 slacapra 1.12
626     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
627     common.logger.message(msg)
628 nsmirnov 1.4 pass
629    
630     common.jobDB.save()
631     pass
632    
633     elif ( opt == '-resubmit' ):
634 fanzago 1.37 if ( self.flag_useboss == 1 ):
635     if val=='all' or val==None or val=='':
636     allBoss_id = common.scheduler.listBoss()
637     # sono gli internal_id di BOSS
638     jobs = allBoss_id.keys()
639     else:
640     jobs = self.parseRange_(val)
641     else:
642     if val:
643     jobs = self.parseRange_(val)
644    
645 slacapra 1.11 if val:
646     # create a list of jobs to be resubmitted.
647 slacapra 1.8
648 slacapra 1.22 ### as before, create a Resubmittter Class
649 slacapra 1.11 nj_list = []
650     for nj in jobs:
651 fanzago 1.37 st = common.jobDB.status(int(nj)-1)
652     #print "stato del job = ", st
653 slacapra 1.11 if st in ['K','A']:
654 fanzago 1.37 nj_list.append(int(nj)-1)
655     common.jobDB.setStatus(int(nj)-1,'C')
656 slacapra 1.11 elif st == 'Y':
657 fanzago 1.39 common.scheduler.moveOutput(nj)
658 fanzago 1.37 nj_list.append(int(nj)-1)
659     st = common.jobDB.setStatus(int(nj)-1,'RC')
660     elif st in ['D','C','X']:
661     # Nel caso DONE bisogna prima ritirare l'output
662     # Nel caso C va prima sottomesso
663     # Nel caso X va prima creato
664 slacapra 1.11 pass
665     else:
666 fanzago 1.37 #common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
667     common.logger.message('Job #'+nj+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
668 slacapra 1.9 pass
669 slacapra 1.8
670 fanzago 1.37 if len(common.job_list) == 0 :
671     #print "common.job_list = ", common.job_list
672     common.job_list = JobList(common.jobDB.nJobs(),None)
673     common.job_list.setJDLNames(self.job_type_name+'.jdl')
674     pass
675    
676 slacapra 1.11 if len(nj_list) != 0:
677 fanzago 1.37 #print "nj_list = ", nj_list
678     common.scheduler.resubmit(nj_list)
679 slacapra 1.11 # Instantiate Submitter object
680     self.actions[opt] = Submitter(self.cfg_params, nj_list)
681    
682 slacapra 1.8 pass
683     pass
684 slacapra 1.11 else:
685     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
686     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
687     pass
688 fanzago 1.37 common.jobDB.save()
689 slacapra 1.8 pass
690    
691     elif ( opt == '-cancelAndResubmit' ):
692     jobs = self.parseRange_(val)
693 nsmirnov 1.5
694 nsmirnov 1.7 # Cancel submitted jobs.
695 nsmirnov 1.5
696 slacapra 1.8 nj_list = []
697     for nj in jobs:
698 nsmirnov 1.5 st = common.jobDB.status(nj)
699     if st == 'S':
700     jid = common.jobDB.jobId(nj)
701     common.scheduler.cancel(jid)
702     st = 'K'
703     common.jobDB.setStatus(nj, st)
704     pass
705 nsmirnov 1.7 pass
706 nsmirnov 1.5
707 slacapra 1.8 if st != 'X': nj_list.append(nj)
708     pass
709 nsmirnov 1.5
710     if len(nj_list) != 0:
711     # Instantiate Submitter object
712     self.actions[opt] = Submitter(self.cfg_params, nj_list)
713    
714     # Create and initialize JobList
715    
716     if len(common.job_list) == 0 :
717     common.job_list = JobList(common.jobDB.nJobs(),
718     None)
719     common.job_list.setJDLNames(self.job_type_name+'.jdl')
720     pass
721     pass
722 nsmirnov 1.4 pass
723    
724 slacapra 1.28 elif ( opt == '-testJdl' ):
725 slacapra 1.8 jobs = self.parseRange_(val)
726     nj_list = []
727     for nj in jobs:
728     st = common.jobDB.status(nj)
729 slacapra 1.12 if st == 'C': nj_list.append(nj)
730 slacapra 1.8 pass
731    
732     if len(nj_list) != 0:
733     # Instantiate Submitter object
734     self.actions[opt] = Checker(self.cfg_params, nj_list)
735    
736     # Create and initialize JobList
737    
738     if len(common.job_list) == 0 :
739     common.job_list = JobList(common.jobDB.nJobs(), None)
740     common.job_list.setJDLNames(self.job_type_name+'.jdl')
741     pass
742     pass
743    
744 slacapra 1.9 elif ( opt == '-postMortem' ):
745     jobs = self.parseRange_(val)
746     nj_list = []
747     for nj in jobs:
748     st = common.jobDB.status(nj)
749 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
750 slacapra 1.9 pass
751    
752     if len(nj_list) != 0:
753     # Instantiate Submitter object
754 spiga 1.35 self.actions[opt] = PostMortem(self.cfg_params, nj_list,self.flag_useboss)
755 slacapra 1.9
756     # Create and initialize JobList
757    
758     if len(common.job_list) == 0 :
759     common.job_list = JobList(common.jobDB.nJobs(), None)
760     common.job_list.setJDLNames(self.job_type_name+'.jdl')
761     pass
762     pass
763 slacapra 1.33 else:
764     common.logger.message("No jobs to analyze")
765 slacapra 1.9
766     elif ( opt == '-clean' ):
767     if val != None:
768     raise CrabException("No range allowed for '-clean'")
769    
770 slacapra 1.36 theCleaner = Cleaner(self.scheduler_name == 'boss')
771     theCleaner.clean()
772 slacapra 1.9
773 nsmirnov 1.1 pass
774     return
775    
776 nsmirnov 1.3 def createWorkingSpace_(self):
777 slacapra 1.9 new_dir = ''
778    
779     try:
780     new_dir = self.cfg_params['USER.ui_working_dir']
781     except KeyError:
782     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
783     new_dir = self.cwd + new_dir
784     pass
785    
786 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
787     common.work_space.create()
788     return
789    
790 nsmirnov 1.3 def loadConfiguration_(self, opts):
791 nsmirnov 1.1
792     save_opts = common.work_space.loadSavedOptions()
793    
794     # Override saved options with new command-line options
795    
796     for k in opts.keys():
797     save_opts[k] = opts[k]
798     pass
799    
800     # Return updated options
801     return save_opts
802    
803 nsmirnov 1.3 def createLogger_(self, args):
804 nsmirnov 1.1
805     log = Logger()
806     log.quiet(self.flag_quiet)
807     log.setDebugLevel(self.debug_level)
808     log.write(args+'\n')
809 nsmirnov 1.3 log.message(self.headerString_())
810 nsmirnov 1.1 log.flush()
811     common.logger = log
812     return
813    
814 nsmirnov 1.3 def updateHistory_(self, args):
815 nsmirnov 1.1 history_fname = common.prog_name+'.history'
816     history_file = open(history_fname, 'a')
817     history_file.write(self.current_time+': '+args+'\n')
818     history_file.close()
819     return
820    
821 nsmirnov 1.3 def headerString_(self):
822 nsmirnov 1.1 """
823     Creates a string describing program options either given in
824     the command line or their default values.
825     """
826     header = common.prog_name + ' (version ' + common.prog_version_str + \
827     ') running on ' + \
828     time.ctime(time.time())+'\n\n' + \
829     common.prog_name+'. Working options:\n'
830     header = header +\
831     ' scheduler ' + self.scheduler_name + '\n'+\
832     ' job type ' + self.job_type_name + '\n'+\
833     ' working directory ' + common.work_space.topDir()\
834     + '\n'
835     return header
836    
837 nsmirnov 1.3 def createScheduler_(self):
838 nsmirnov 1.1 """
839     Creates a scheduler object instantiated by its name.
840     """
841     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
842     file_name = klass_name
843     try:
844     klass = importName(file_name, klass_name)
845     except KeyError:
846     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
847     raise CrabException(msg)
848     except ImportError, e:
849     msg = 'Cannot create scheduler '+self.scheduler_name
850     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
851     msg += str(e)
852     raise CrabException(msg)
853    
854     common.scheduler = klass()
855     common.scheduler.configure(self.cfg_params)
856     return
857    
858     def run(self):
859     """
860     For each
861     """
862    
863     for act in self.main_actions:
864     if act in self.actions.keys(): self.actions[act].run()
865     pass
866    
867     for act in self.aux_actions:
868     if act in self.actions.keys(): self.actions[act].run()
869     pass
870     return
871    
872     ###########################################################################
873     def processHelpOptions(opts):
874    
875 slacapra 1.11 if len(opts):
876     for opt in opts.keys():
877     if opt in ('-v', '-version', '--version') :
878     print Crab.version()
879     return 1
880     if opt in ('-h','-help','--help') :
881     if opts[opt] : help(opts[opt])
882     else: help()
883     return 1
884     else:
885     usage()
886 nsmirnov 1.1
887     return 0
888    
889     ###########################################################################
890     if __name__ == '__main__':
891    
892 fanzago 1.18
893 fanzago 1.19 # # Initial settings for Python modules. Avoid appending manually lib paths.
894     # try:
895     # path=os.environ['EDG_WL_LOCATION']
896     # except:
897     # print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
898     # sys.exit(1)
899     #
900     # libPath=os.path.join(path, "lib")
901     # sys.path.append(libPath)
902     # libPath=os.path.join(path, "lib", "python")
903     # sys.path.append(libPath)
904 fanzago 1.18
905    
906 nsmirnov 1.1 # Parse command-line options and create a dictionary with
907     # key-value pairs.
908    
909     options = parseOptions(sys.argv[1:])
910    
911     # Process "help" options, such as '-help', '-version'
912    
913 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
914 nsmirnov 1.1
915     # Create, initialize, and run a Crab object
916    
917     try:
918 nsmirnov 1.3 crab = Crab(options)
919 nsmirnov 1.1 crab.run()
920     except CrabException, e:
921     print '\n' + common.prog_name + ': ' + str(e) + '\n'
922     if common.logger:
923     common.logger.write('ERROR: '+str(e)+'\n')
924     pass
925     pass
926    
927     pass