ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.16
Committed: Sat Sep 24 14:29:44 2005 UTC (19 years, 7 months ago) by spiga
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_1_0_0_pre1_boss
Changes since 1.15: +8 -8 lines
Log Message: 
boss integration

File Contents

# User Rev Content
1 slacapra 1.9 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 nsmirnov 1.1 import common
16 spiga 1.13 import Statistic
17 nsmirnov 1.1
18     import sys, os, time, string
19    
20     ###########################################################################
21     class Crab:
22 nsmirnov 1.3 def __init__(self, opts):
23 nsmirnov 1.1
24     # The order of main_actions is important !
25 slacapra 1.9 self.main_actions = [ '-create', '-submit', '-monitor' ]
26 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
27 slacapra 1.9 '-resubmit' , '-cancelAndResubmit', '-check', '-postMortem', '-clean']
28 nsmirnov 1.1
29     # Dictionary of actions, e.g. '-create' -> object of class Creator
30     self.actions = {}
31    
32     # Configuration file
33     self.cfg_fname = None
34     # Dictionary with configuration parameters
35     self.cfg_params = {}
36    
37     # Current working directory
38     self.cwd = os.getcwd()+'/'
39     # Current time in format 'yymmdd_hhmmss'
40     self.current_time = time.strftime('%y%m%d_%H%M%S',
41     time.localtime(time.time()))
42    
43 nsmirnov 1.3 # Session name (?) Do we need this ?
44 nsmirnov 1.1 self.name = '0'
45    
46     # Job type
47     self.job_type_name = None
48    
49     # Continuation flag
50     self.flag_continue = 0
51    
52     # quiet mode, i.e. no output on screen
53     self.flag_quiet = 0
54     # produce more output
55     self.debug_level = 0
56    
57 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
58 fanzago 1.14 self.scheduler_name = ''
59 nsmirnov 1.1
60 nsmirnov 1.3 self.initialize_(opts)
61 nsmirnov 1.1
62     return
63    
64 nsmirnov 1.7 def version():
65 nsmirnov 1.1 return common.prog_version_str
66    
67 nsmirnov 1.7 version = staticmethod(version)
68    
69 nsmirnov 1.3 def initialize_(self, opts):
70 nsmirnov 1.1
71     # Process the '-continue' option first because
72     # in the case of continuation the CRAB configuration
73     # parameters are loaded from already existing Working Space.
74 nsmirnov 1.3 self.processContinueOption_(opts)
75 nsmirnov 1.1
76     # Process ini-file first, then command line options
77     # because they override ini-file settings.
78    
79 nsmirnov 1.3 self.processIniFile_(opts)
80 nsmirnov 1.1
81 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
82 nsmirnov 1.1
83 nsmirnov 1.3 self.processOptions_(opts)
84 nsmirnov 1.1
85     if not self.flag_continue:
86 nsmirnov 1.3 self.createWorkingSpace_()
87 slacapra 1.9 optsToBeSaved={}
88     for it in opts.keys():
89     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
90     pass
91     else:
92     optsToBeSaved[it]=opts[it]
93     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
94 nsmirnov 1.1 pass
95    
96     # At this point all configuration options have been read.
97    
98     args = string.join(sys.argv,' ')
99 slacapra 1.11
100 nsmirnov 1.3 self.updateHistory_(args)
101 slacapra 1.11
102 nsmirnov 1.3 self.createLogger_(args)
103 slacapra 1.11
104 nsmirnov 1.1 common.jobDB = JobDB()
105 slacapra 1.11
106 nsmirnov 1.3 if self.flag_continue:
107 slacapra 1.12 try:
108     common.jobDB.load()
109     common.logger.debug(6, str(common.jobDB))
110     except DBException,e:
111     pass
112 nsmirnov 1.3 pass
113 slacapra 1.11
114 nsmirnov 1.3 self.createScheduler_()
115 slacapra 1.11
116 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
117     common.logger.debug(6, 'Used properties:')
118     keys = self.cfg_params.keys()
119     keys.sort()
120     for k in keys:
121     if self.cfg_params[k]:
122     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
123     pass
124     else:
125     common.logger.debug(6, ' '+k+' : ')
126     pass
127     pass
128     common.logger.debug(6, 'End of used properties.\n')
129     pass
130     self.initializeActions_(opts)
131 nsmirnov 1.1 return
132    
133 nsmirnov 1.3 def processContinueOption_(self,opts):
134 nsmirnov 1.1
135     continue_dir = None
136 nsmirnov 1.4
137     # Look for the '-continue' option.
138    
139 nsmirnov 1.1 for opt in opts.keys():
140     if ( opt in ('-continue','-c') ):
141     self.flag_continue = 1
142     val = opts[opt]
143     if val:
144     if val[0] == '/': continue_dir = val # abs path
145     else: continue_dir = self.cwd + val # rel path
146     pass
147 nsmirnov 1.4 break
148     pass
149    
150     # Look for actions which has sense only with '-continue'
151    
152     if not self.flag_continue:
153     for opt in opts.keys():
154 slacapra 1.6 if ( opt in (self.aux_actions) ):
155 nsmirnov 1.4 self.flag_continue = 1
156     break
157 nsmirnov 1.1 pass
158     pass
159 slacapra 1.6 submit_flag=0
160     create_flag=0
161     for opt in opts.keys():
162     if opt == "-submit": submit_flag=1
163     if opt == "-create": create_flag=1
164     pass
165     if (submit_flag and not create_flag):
166 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
167     raise CrabException(msg)
168 slacapra 1.6 pass
169 nsmirnov 1.1
170     if not self.flag_continue: return
171    
172     if not continue_dir:
173     prefix = common.prog_name + '_' + self.name + '_'
174     continue_dir = findLastWorkDir(prefix)
175     pass
176    
177     if not continue_dir:
178     raise CrabException('Cannot find last working directory.')
179    
180     if not os.path.exists(continue_dir):
181     msg = 'Cannot continue because the working directory <'
182     msg += continue_dir
183     msg += '> does not exist.'
184     raise CrabException(msg)
185    
186     # Instantiate WorkSpace
187     common.work_space = WorkSpace(continue_dir)
188    
189     return
190    
191 nsmirnov 1.3 def processIniFile_(self, opts):
192 nsmirnov 1.1 """
193     Processes a configuration INI-file.
194     """
195    
196     # Extract cfg-file name from the cmd-line options.
197    
198     for opt in opts.keys():
199     if ( opt == '-cfg' ):
200     if self.flag_continue:
201     raise CrabException('-continue and -cfg cannot coexist.')
202     if opts[opt] : self.cfg_fname = opts[opt]
203     else : usage()
204     pass
205    
206     elif ( opt == '-name' ):
207     self.name = opts[opt]
208     pass
209    
210     pass
211    
212     # Set default cfg-fname
213    
214     if self.cfg_fname == None:
215     if self.flag_continue:
216     self.cfg_fname = common.work_space.cfgFileName()
217     else:
218     self.cfg_fname = common.prog_name+'.cfg'
219     pass
220     pass
221    
222     # Load cfg-file
223    
224     if string.lower(self.cfg_fname) != 'none':
225     if os.path.exists(self.cfg_fname):
226     self.cfg_params = loadConfig(self.cfg_fname)
227     pass
228     else:
229     msg = 'cfg-file '+self.cfg_fname+' not found.'
230     raise CrabException(msg)
231     pass
232     pass
233    
234     # process the [CRAB] section
235    
236     lhp = len('CRAB.')
237     for k in self.cfg_params.keys():
238     if len(k) >= lhp and k[:lhp] == 'CRAB.':
239     opt = '-'+k[lhp:]
240     if len(opt) >= 3 and opt[:3] == '-__': continue
241     if opt not in opts.keys():
242     opts[opt] = self.cfg_params[k]
243     pass
244     pass
245     pass
246    
247     return
248    
249 nsmirnov 1.3 def processOptions_(self, opts):
250 nsmirnov 1.1 """
251     Processes the command-line options.
252     """
253    
254     for opt in opts.keys():
255     val = opts[opt]
256    
257 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
258     if opt in self.main_actions:
259     self.cfg_params['CRAB.'+opt[1:]] = val
260     continue
261     if opt in self.aux_actions:
262     self.cfg_params['CRAB.'+opt[1:]] = val
263     continue
264 nsmirnov 1.1
265    
266     elif ( opt == '-cfg' ):
267     pass
268    
269     elif ( opt in ('-continue', '-c') ):
270 nsmirnov 1.4 # Already processed in processContinueOption_()
271 nsmirnov 1.1 pass
272    
273     elif ( opt == '-jobtype' ):
274     if val : self.job_type_name = string.upper(val)
275     else : usage()
276     pass
277    
278     elif ( opt == '-Q' ):
279     self.flag_quiet = 1
280     pass
281    
282     elif ( opt == '-debug' ):
283 slacapra 1.6 if val: self.debug_level = int(val)
284     else: self.debug_level = 1
285 nsmirnov 1.1 pass
286    
287     elif ( opt == '-scheduler' ):
288     if val: self.scheduler_name = val
289     else:
290     print common.prog_name+". No value for '-scheduler'."
291     usage()
292     pass
293     pass
294    
295 fanzago 1.14 elif ( opt == '-use_boss' ):
296     self.scheduler_name = 'boss'
297     pass
298    
299 nsmirnov 1.3 elif string.find(opt,'.') == -1:
300     print common.prog_name+'. Unrecognized option '+opt
301     usage()
302     pass
303 nsmirnov 1.1
304 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
305     if string.find(opt,'.') == -1 :
306     self.cfg_params['CRAB.'+opt[1:]] = val
307 nsmirnov 1.1 pass
308 nsmirnov 1.3 else:
309 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
310     self.cfg_params[opt[1:]] = val
311     pass
312     pass
313     return
314    
315 slacapra 1.8 def parseRange_(self, aRange):
316 nsmirnov 1.4 """
317 slacapra 1.8 Takes as the input a string with a range defined in any of the following
318     way, including combination, and return a tuple with the ints defined
319     according to following table. A consistency check is done.
320     NB: the first job is "1", not "0".
321     'all' -> [1,2,..., NJobs]
322     '' -> [1,2,..., NJobs]
323     'n1' -> [n1]
324     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
325     'n1,n2' -> [n1, n2]
326     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
327     """
328     result = []
329    
330 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
331     if aRange=='all' or aRange==None or aRange=='':
332 slacapra 1.8 result=range(0,common.jobDB.nJobs())
333     return result
334 slacapra 1.9 elif aRange=='0':
335     return result
336 slacapra 1.8
337     subRanges = string.split(aRange, ',')
338     for subRange in subRanges:
339     result = result+self.parseSimpleRange_(subRange)
340    
341     if self.checkUniqueness_(result):
342     return result
343     else:
344     print "Error ", result
345     return []
346    
347     def checkUniqueness_(self, list):
348     """
349 slacapra 1.9 check if a list contains only unique elements
350 slacapra 1.8 """
351    
352     uniqueList = []
353     # use a list comprehension statement (takes a while to understand)
354    
355     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
356    
357     return (len(list)==len(uniqueList))
358    
359     def parseSimpleRange_(self, aRange):
360     """
361     Takes as the input a string with two integers separated by
362     the minus sign and returns the tuple with these numbers:
363     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
364     'n1' -> [n1]
365     """
366     (start, end) = (None, None)
367    
368     result = []
369     minus = string.find(aRange, '-')
370     if ( minus < 0 ):
371     if isInt(aRange) and int(aRange)>0:
372     result.append(int(aRange)-1)
373     else:
374 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
375 nsmirnov 1.4 pass
376     else:
377 slacapra 1.8 (start, end) = string.split(aRange, '-')
378     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
379     result=range(int(start)-1, int(end))
380     else:
381     print "ERROR ", start, end
382    
383     return result
384 nsmirnov 1.4
385 nsmirnov 1.3 def initializeActions_(self, opts):
386 nsmirnov 1.1 """
387     For each user action instantiate a corresponding
388     object and put it in the action dictionary.
389     """
390     for opt in opts.keys():
391 spiga 1.16 if ( opt == '-use_boss'):
392 spiga 1.15 self.flag_useboss = 1
393     else:
394     self.flag_useboss = 0
395     pass
396    
397     for opt in opts.keys():
398    
399 nsmirnov 1.1 val = opts[opt]
400 spiga 1.15
401    
402     if ( opt == '-create' ):
403 nsmirnov 1.1 if val:
404     if ( isInt(val) ):
405     ncjobs = int(val)
406     elif ( val == 'all'):
407     ncjobs = val
408     else:
409 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
410     msg += ' Must be an integer or "all"'
411 slacapra 1.8 msg += ' Generic range is not allowed"'
412 nsmirnov 1.5 raise CrabException(msg)
413 nsmirnov 1.1 pass
414     else: ncjobs = 'all'
415    
416     if ncjobs != 0:
417 nsmirnov 1.3 # Instantiate Creator object
418 nsmirnov 1.1 creator = Creator(self.job_type_name,
419     self.cfg_params,
420     ncjobs)
421     self.actions[opt] = creator
422 nsmirnov 1.3
423     # Initialize the JobDB object if needed
424 nsmirnov 1.1 if not self.flag_continue:
425     common.jobDB.create(creator.nJobs())
426     pass
427 nsmirnov 1.3
428     # Create and initialize JobList
429    
430     common.job_list = JobList(common.jobDB.nJobs(),
431     creator.jobType())
432    
433     common.job_list.setScriptNames(self.job_type_name+'.sh')
434     common.job_list.setJDLNames(self.job_type_name+'.jdl')
435 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
436 slacapra 1.9
437     creator.writeJobsSpecsToDB()
438 nsmirnov 1.1 pass
439 nsmirnov 1.3 pass
440 nsmirnov 1.1
441     elif ( opt == '-submit' ):
442 nsmirnov 1.7 nj_list = self.parseRange_(val)
443 nsmirnov 1.5
444     if len(nj_list) != 0:
445 nsmirnov 1.3 # Instantiate Submitter object
446 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
447 nsmirnov 1.3
448     # Create and initialize JobList
449    
450     if len(common.job_list) == 0 :
451     common.job_list = JobList(common.jobDB.nJobs(),
452     None)
453     common.job_list.setJDLNames(self.job_type_name+'.jdl')
454     pass
455 nsmirnov 1.1 pass
456 nsmirnov 1.3 pass
457 nsmirnov 1.1
458 nsmirnov 1.4 elif ( opt == '-list' ):
459 slacapra 1.8 jobs = self.parseRange_(val)
460    
461     common.jobDB.dump(jobs)
462 nsmirnov 1.4 pass
463    
464     elif ( opt == '-status' ):
465 slacapra 1.8 jobs = self.parseRange_(val)
466    
467 slacapra 1.9 if len(jobs) != 0:
468 spiga 1.16 if ( self.flag_useboss == 1 ):
469     self.actions[opt] = StatusBoss(self.cfg_params, jobs)
470 spiga 1.15 else:
471     # Instantiate Submitter object
472     self.actions[opt] = Status(self.cfg_params, jobs)
473     pass
474 nsmirnov 1.1 pass
475     pass
476 nsmirnov 1.4 elif ( opt == '-kill' ):
477 slacapra 1.9 if val:
478     jobs = self.parseRange_(val)
479 slacapra 1.8
480 slacapra 1.9 for nj in jobs:
481     st = common.jobDB.status(nj)
482     if st == 'S':
483     jid = common.jobDB.jobId(nj)
484     common.logger.message("Killing job # "+`(nj+1)`)
485     common.scheduler.cancel(jid)
486     common.jobDB.setStatus(nj, 'K')
487     pass
488 nsmirnov 1.4 pass
489 slacapra 1.9
490     common.jobDB.save()
491 nsmirnov 1.1 pass
492 slacapra 1.9 else:
493     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
494 nsmirnov 1.1
495 slacapra 1.8 elif ( opt == '-getoutput' ):
496     jobs = self.parseRange_(val)
497    
498 spiga 1.13 fileCODE1 = open(common.work_space.logDir()+"/.code","r")
499     array = fileCODE1.read().split('::')
500     self.ID1 = array[0]
501     self.NJC = array[1]
502     self.dataset = array[2]
503     self.owner = array[3]
504     fileCODE1.close()
505    
506    
507 slacapra 1.12 jobs_done = []
508 slacapra 1.8 for nj in jobs:
509 nsmirnov 1.4 st = common.jobDB.status(nj)
510 spiga 1.16 if st == 'D':
511 slacapra 1.12 jobs_done.append(nj)
512     pass
513     elif st == 'S':
514 nsmirnov 1.4 jid = common.jobDB.jobId(nj)
515 fanzago 1.14 print "jid", jid
516 slacapra 1.9 currStatus = common.scheduler.queryStatus(jid)
517     if currStatus=="Done":
518 slacapra 1.12 jobs_done.append(nj)
519 slacapra 1.9 else:
520     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
521 nsmirnov 1.4 common.logger.message(msg)
522     pass
523 slacapra 1.8 else:
524 spiga 1.16 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
525 slacapra 1.12 pass
526     pass
527    
528     for nj in jobs_done:
529 spiga 1.16 if ( self.flag_useboss == 1 ):
530 spiga 1.15 id = common.jobDB.bossId(nj)
531     dir = common.scheduler.getOutput(id)
532     common.jobDB.setStatus(nj, 'Y')
533     jid = common.jobDB.jobId(nj)
534     else:
535     jid = common.jobDB.jobId(nj)
536     dir = common.scheduler.getOutput(jid)
537     common.jobDB.setStatus(nj, 'Y')
538 slacapra 1.12
539     # Rename the directory with results to smth readable
540     new_dir = common.work_space.resDir()
541 spiga 1.16 if ( self.flag_useboss == 0 ):
542 spiga 1.15 try:
543     files = os.listdir(dir)
544     for file in files:
545     os.rename(dir+'/'+file, new_dir+'/'+file)
546     os.rmdir(dir)
547     except OSError, e:
548     msg = 'rename files from '+dir+' to '+new_dir+' error: '
549     msg += str(e)
550     common.logger.message(msg)
551     # ignore error
552     pass
553 slacapra 1.12 pass
554 spiga 1.13 destination = common.scheduler.queryDest(jid).split(":")[0]
555     ID3 = jid.split("/")[3]
556     broker = jid.split("/")[2].split(":")[0]
557     resFlag = 0
558     exCode = common.scheduler.getExitStatus(jid)
559     Statistic.notify('retrieved',resFlag,exCode,self.dataset,self.owner,destination,broker,ID3,self.ID1,self.NJC)
560 slacapra 1.12
561     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
562     common.logger.message(msg)
563 nsmirnov 1.4 pass
564    
565     common.jobDB.save()
566     pass
567    
568     elif ( opt == '-resubmit' ):
569 slacapra 1.11 if val:
570     jobs = self.parseRange_(val)
571 slacapra 1.8
572 slacapra 1.11 # create a list of jobs to be resubmitted.
573 slacapra 1.8
574 slacapra 1.11 nj_list = []
575     for nj in jobs:
576     st = common.jobDB.status(nj)
577 slacapra 1.8
578 slacapra 1.11 if st in ['K','A']:
579     nj_list.append(nj)
580     elif st == 'Y':
581     # here I would like to move the old output to a new place
582     # I would need the name of the output files.
583     # these infos are in the jobType class, which is not accessible from here!
584     outSandbox = common.jobDB.outputSandbox(nj)
585    
586     resDir = common.work_space.resDir()
587     resDirSave = resDir + self.current_time
588     os.mkdir(resDirSave)
589    
590     for file in outSandbox:
591     if os.path.exists(resDir+'/'+file):
592     os.rename(resDir+'/'+file, resDirSave+'/'+file)
593     common.logger.message('Output file '+file+' moved to '+resDirSave)
594     pass
595     nj_list.append(nj)
596 spiga 1.13 st = common.jobDB.setStatus(nj,'RC')
597 slacapra 1.11 elif st == 'D':
598     ## Done but not yet retrieved
599     ## retrieve job, then go to previous ('Y') case
600     ## TODO
601     pass
602     else:
603     common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
604 slacapra 1.9 pass
605 slacapra 1.8
606 slacapra 1.11 if len(nj_list) != 0:
607     # Instantiate Submitter object
608     self.actions[opt] = Submitter(self.cfg_params, nj_list)
609    
610     # Create and initialize JobList
611    
612     if len(common.job_list) == 0 :
613     common.job_list = JobList(common.jobDB.nJobs(),
614     None)
615     common.job_list.setJDLNames(self.job_type_name+'.jdl')
616     pass
617 slacapra 1.8 pass
618     pass
619 slacapra 1.11 else:
620     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
621     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
622     pass
623 slacapra 1.8 pass
624    
625     elif ( opt == '-cancelAndResubmit' ):
626     jobs = self.parseRange_(val)
627 nsmirnov 1.5
628 nsmirnov 1.7 # Cancel submitted jobs.
629 nsmirnov 1.5
630 slacapra 1.8 nj_list = []
631     for nj in jobs:
632 nsmirnov 1.5 st = common.jobDB.status(nj)
633     if st == 'S':
634     jid = common.jobDB.jobId(nj)
635     common.scheduler.cancel(jid)
636     st = 'K'
637     common.jobDB.setStatus(nj, st)
638     pass
639 nsmirnov 1.7 pass
640 nsmirnov 1.5
641 slacapra 1.8 if st != 'X': nj_list.append(nj)
642     pass
643 nsmirnov 1.5
644     if len(nj_list) != 0:
645     # Instantiate Submitter object
646     self.actions[opt] = Submitter(self.cfg_params, nj_list)
647    
648     # Create and initialize JobList
649    
650     if len(common.job_list) == 0 :
651     common.job_list = JobList(common.jobDB.nJobs(),
652     None)
653     common.job_list.setJDLNames(self.job_type_name+'.jdl')
654     pass
655     pass
656 nsmirnov 1.4 pass
657    
658 slacapra 1.8 elif ( opt == '-check' ):
659     jobs = self.parseRange_(val)
660     nj_list = []
661     for nj in jobs:
662     st = common.jobDB.status(nj)
663 slacapra 1.12 if st == 'C': nj_list.append(nj)
664 slacapra 1.8 pass
665    
666     if len(nj_list) != 0:
667     # Instantiate Submitter object
668     self.actions[opt] = Checker(self.cfg_params, nj_list)
669    
670     # Create and initialize JobList
671    
672     if len(common.job_list) == 0 :
673     common.job_list = JobList(common.jobDB.nJobs(), None)
674     common.job_list.setJDLNames(self.job_type_name+'.jdl')
675     pass
676     pass
677    
678 slacapra 1.9 elif ( opt == '-postMortem' ):
679     jobs = self.parseRange_(val)
680     nj_list = []
681     for nj in jobs:
682     st = common.jobDB.status(nj)
683 slacapra 1.11 if st in ['Y','A','D']: nj_list.append(nj)
684 slacapra 1.9 pass
685    
686     if len(nj_list) != 0:
687     # Instantiate Submitter object
688     self.actions[opt] = PostMortem(self.cfg_params, nj_list)
689    
690     # Create and initialize JobList
691    
692     if len(common.job_list) == 0 :
693     common.job_list = JobList(common.jobDB.nJobs(), None)
694     common.job_list.setJDLNames(self.job_type_name+'.jdl')
695     pass
696     pass
697    
698     elif ( opt == '-clean' ):
699     if val != None:
700     raise CrabException("No range allowed for '-clean'")
701    
702     submittedJobs=0
703 slacapra 1.11 doneJobs=0
704 slacapra 1.12 try:
705     for nj in range(0,common.jobDB.nJobs()):
706     st = common.jobDB.status(nj)
707     if st == 'S':
708     submittedJobs = submittedJobs+1
709     if st == 'D':
710     doneJobs = doneJobs+1
711     pass
712     pass
713     except DBException:
714     common.logger.debug(5,'DB not found, so delete all')
715 nsmirnov 1.1 pass
716 slacapra 1.9
717 slacapra 1.11 if submittedJobs or doneJobs:
718     msg = "There are still "
719     if submittedJobs:
720     msg= msg+str(submittedJobs)+" jobs submitted. Kill them '-kill' before '-clean'"
721     if (submittedJobs and doneJobs):
722     msg = msg + "and \nalso"
723     if doneJobs:
724     msg= msg+str(doneJobs)+" jobs Done. Get their outputs '-getoutput' before '-clean'"
725     raise CrabException(msg)
726 slacapra 1.9
727     msg = 'directory '+common.work_space.topDir()+' removed'
728     common.work_space.delete()
729     common.logger.message(msg)
730    
731 nsmirnov 1.1 pass
732 slacapra 1.9
733 nsmirnov 1.1 pass
734     return
735    
736 nsmirnov 1.3 def createWorkingSpace_(self):
737 slacapra 1.9 new_dir = ''
738    
739     try:
740     new_dir = self.cfg_params['USER.ui_working_dir']
741     except KeyError:
742     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
743     new_dir = self.cwd + new_dir
744     pass
745    
746 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
747     common.work_space.create()
748     return
749    
750 nsmirnov 1.3 def loadConfiguration_(self, opts):
751 nsmirnov 1.1
752     save_opts = common.work_space.loadSavedOptions()
753    
754     # Override saved options with new command-line options
755    
756     for k in opts.keys():
757     save_opts[k] = opts[k]
758     pass
759    
760     # Return updated options
761     return save_opts
762    
763 nsmirnov 1.3 def createLogger_(self, args):
764 nsmirnov 1.1
765     log = Logger()
766     log.quiet(self.flag_quiet)
767     log.setDebugLevel(self.debug_level)
768     log.write(args+'\n')
769 nsmirnov 1.3 log.message(self.headerString_())
770 nsmirnov 1.1 log.flush()
771     common.logger = log
772     return
773    
774 nsmirnov 1.3 def updateHistory_(self, args):
775 nsmirnov 1.1 history_fname = common.prog_name+'.history'
776     history_file = open(history_fname, 'a')
777     history_file.write(self.current_time+': '+args+'\n')
778     history_file.close()
779     return
780    
781 nsmirnov 1.3 def headerString_(self):
782 nsmirnov 1.1 """
783     Creates a string describing program options either given in
784     the command line or their default values.
785     """
786     header = common.prog_name + ' (version ' + common.prog_version_str + \
787     ') running on ' + \
788     time.ctime(time.time())+'\n\n' + \
789     common.prog_name+'. Working options:\n'
790     header = header +\
791     ' scheduler ' + self.scheduler_name + '\n'+\
792     ' job type ' + self.job_type_name + '\n'+\
793     ' working directory ' + common.work_space.topDir()\
794     + '\n'
795     return header
796    
797 nsmirnov 1.3 def createScheduler_(self):
798 nsmirnov 1.1 """
799     Creates a scheduler object instantiated by its name.
800     """
801     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
802     file_name = klass_name
803     try:
804     klass = importName(file_name, klass_name)
805     except KeyError:
806     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
807     raise CrabException(msg)
808     except ImportError, e:
809     msg = 'Cannot create scheduler '+self.scheduler_name
810     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
811     msg += str(e)
812     raise CrabException(msg)
813    
814     common.scheduler = klass()
815     common.scheduler.configure(self.cfg_params)
816     return
817    
818     def run(self):
819     """
820     For each
821     """
822    
823     for act in self.main_actions:
824     if act in self.actions.keys(): self.actions[act].run()
825     pass
826    
827     for act in self.aux_actions:
828     if act in self.actions.keys(): self.actions[act].run()
829     pass
830     return
831    
832     ###########################################################################
833     def processHelpOptions(opts):
834    
835 slacapra 1.11 if len(opts):
836     for opt in opts.keys():
837     if opt in ('-v', '-version', '--version') :
838     print Crab.version()
839     return 1
840     if opt in ('-h','-help','--help') :
841     if opts[opt] : help(opts[opt])
842     else: help()
843     return 1
844     else:
845     usage()
846 nsmirnov 1.1
847     return 0
848    
849     ###########################################################################
850     if __name__ == '__main__':
851    
852     # Parse command-line options and create a dictionary with
853     # key-value pairs.
854    
855     options = parseOptions(sys.argv[1:])
856    
857     # Process "help" options, such as '-help', '-version'
858    
859 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
860 nsmirnov 1.1
861     # Create, initialize, and run a Crab object
862    
863     try:
864 nsmirnov 1.3 crab = Crab(options)
865 nsmirnov 1.1 crab.run()
866     except CrabException, e:
867     print '\n' + common.prog_name + ': ' + str(e) + '\n'
868     if common.logger:
869     common.logger.write('ERROR: '+str(e)+'\n')
870     pass
871     pass
872    
873     pass