ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.12
Committed: Tue Aug 23 10:38:09 2005 UTC (19 years, 8 months ago) by slacapra
Content type: text/x-python
Branch: MAIN
CVS Tags: CRAB_1_0_0_pre3, CRAB_1_0_0_pre2
Changes since 1.11: +50 -34 lines
Log Message: 
too many changes to be listed here...

File Contents

# User Rev Content
1 slacapra 1.9 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 nsmirnov 1.1 import common
15    
16     import sys, os, time, string
17    
18     ###########################################################################
19     class Crab:
20 nsmirnov 1.3 def __init__(self, opts):
21 nsmirnov 1.1
22     # The order of main_actions is important !
23 slacapra 1.9 self.main_actions = [ '-create', '-submit', '-monitor' ]
24 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
25 slacapra 1.9 '-resubmit' , '-cancelAndResubmit', '-check', '-postMortem', '-clean']
26 nsmirnov 1.1
27     # Dictionary of actions, e.g. '-create' -> object of class Creator
28     self.actions = {}
29    
30     # Configuration file
31     self.cfg_fname = None
32     # Dictionary with configuration parameters
33     self.cfg_params = {}
34    
35     # Current working directory
36     self.cwd = os.getcwd()+'/'
37     # Current time in format 'yymmdd_hhmmss'
38     self.current_time = time.strftime('%y%m%d_%H%M%S',
39     time.localtime(time.time()))
40    
41 nsmirnov 1.3 # Session name (?) Do we need this ?
42 nsmirnov 1.1 self.name = '0'
43    
44     # Job type
45     self.job_type_name = None
46    
47     # Continuation flag
48     self.flag_continue = 0
49    
50     # quiet mode, i.e. no output on screen
51     self.flag_quiet = 0
52     # produce more output
53     self.debug_level = 0
54    
55 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
56 nsmirnov 1.1 self.scheduler_name = 'edg'
57    
58 nsmirnov 1.3 self.initialize_(opts)
59 nsmirnov 1.1
60     return
61    
62 nsmirnov 1.7 def version():
63 nsmirnov 1.1 return common.prog_version_str
64    
65 nsmirnov 1.7 version = staticmethod(version)
66    
67 nsmirnov 1.3 def initialize_(self, opts):
68 nsmirnov 1.1
69     # Process the '-continue' option first because
70     # in the case of continuation the CRAB configuration
71     # parameters are loaded from already existing Working Space.
72 nsmirnov 1.3 self.processContinueOption_(opts)
73 nsmirnov 1.1
74     # Process ini-file first, then command line options
75     # because they override ini-file settings.
76    
77 nsmirnov 1.3 self.processIniFile_(opts)
78 nsmirnov 1.1
79 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
80 nsmirnov 1.1
81 nsmirnov 1.3 self.processOptions_(opts)
82 nsmirnov 1.1
83     if not self.flag_continue:
84 nsmirnov 1.3 self.createWorkingSpace_()
85 slacapra 1.9 optsToBeSaved={}
86     for it in opts.keys():
87     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
88     pass
89     else:
90     optsToBeSaved[it]=opts[it]
91     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
92 nsmirnov 1.1 pass
93    
94     # At this point all configuration options have been read.
95    
96     args = string.join(sys.argv,' ')
97 slacapra 1.11
98 nsmirnov 1.3 self.updateHistory_(args)
99 slacapra 1.11
100 nsmirnov 1.3 self.createLogger_(args)
101 slacapra 1.11
102 nsmirnov 1.1 common.jobDB = JobDB()
103 slacapra 1.11
104 nsmirnov 1.3 if self.flag_continue:
105 slacapra 1.12 try:
106     common.jobDB.load()
107     common.logger.debug(6, str(common.jobDB))
108     except DBException,e:
109     pass
110 nsmirnov 1.3 pass
111 slacapra 1.11
112 nsmirnov 1.3 self.createScheduler_()
113 slacapra 1.11
114 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
115     common.logger.debug(6, 'Used properties:')
116     keys = self.cfg_params.keys()
117     keys.sort()
118     for k in keys:
119     if self.cfg_params[k]:
120     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
121     pass
122     else:
123     common.logger.debug(6, ' '+k+' : ')
124     pass
125     pass
126     common.logger.debug(6, 'End of used properties.\n')
127     pass
128     self.initializeActions_(opts)
129 nsmirnov 1.1 return
130    
131 nsmirnov 1.3 def processContinueOption_(self,opts):
132 nsmirnov 1.1
133     continue_dir = None
134 nsmirnov 1.4
135     # Look for the '-continue' option.
136    
137 nsmirnov 1.1 for opt in opts.keys():
138     if ( opt in ('-continue','-c') ):
139     self.flag_continue = 1
140     val = opts[opt]
141     if val:
142     if val[0] == '/': continue_dir = val # abs path
143     else: continue_dir = self.cwd + val # rel path
144     pass
145 nsmirnov 1.4 break
146     pass
147    
148     # Look for actions which has sense only with '-continue'
149    
150     if not self.flag_continue:
151     for opt in opts.keys():
152 slacapra 1.6 if ( opt in (self.aux_actions) ):
153 nsmirnov 1.4 self.flag_continue = 1
154     break
155 nsmirnov 1.1 pass
156     pass
157 slacapra 1.6 submit_flag=0
158     create_flag=0
159     for opt in opts.keys():
160     if opt == "-submit": submit_flag=1
161     if opt == "-create": create_flag=1
162     pass
163     if (submit_flag and not create_flag):
164 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
165     raise CrabException(msg)
166 slacapra 1.6 pass
167 nsmirnov 1.1
168     if not self.flag_continue: return
169    
170     if not continue_dir:
171     prefix = common.prog_name + '_' + self.name + '_'
172     continue_dir = findLastWorkDir(prefix)
173     pass
174    
175     if not continue_dir:
176     raise CrabException('Cannot find last working directory.')
177    
178     if not os.path.exists(continue_dir):
179     msg = 'Cannot continue because the working directory <'
180     msg += continue_dir
181     msg += '> does not exist.'
182     raise CrabException(msg)
183    
184     # Instantiate WorkSpace
185     common.work_space = WorkSpace(continue_dir)
186    
187     return
188    
189 nsmirnov 1.3 def processIniFile_(self, opts):
190 nsmirnov 1.1 """
191     Processes a configuration INI-file.
192     """
193    
194     # Extract cfg-file name from the cmd-line options.
195    
196     for opt in opts.keys():
197     if ( opt == '-cfg' ):
198     if self.flag_continue:
199     raise CrabException('-continue and -cfg cannot coexist.')
200     if opts[opt] : self.cfg_fname = opts[opt]
201     else : usage()
202     pass
203    
204     elif ( opt == '-name' ):
205     self.name = opts[opt]
206     pass
207    
208     pass
209    
210     # Set default cfg-fname
211    
212     if self.cfg_fname == None:
213     if self.flag_continue:
214     self.cfg_fname = common.work_space.cfgFileName()
215     else:
216     self.cfg_fname = common.prog_name+'.cfg'
217     pass
218     pass
219    
220     # Load cfg-file
221    
222     if string.lower(self.cfg_fname) != 'none':
223     if os.path.exists(self.cfg_fname):
224     self.cfg_params = loadConfig(self.cfg_fname)
225     pass
226     else:
227     msg = 'cfg-file '+self.cfg_fname+' not found.'
228     raise CrabException(msg)
229     pass
230     pass
231    
232     # process the [CRAB] section
233    
234     lhp = len('CRAB.')
235     for k in self.cfg_params.keys():
236     if len(k) >= lhp and k[:lhp] == 'CRAB.':
237     opt = '-'+k[lhp:]
238     if len(opt) >= 3 and opt[:3] == '-__': continue
239     if opt not in opts.keys():
240     opts[opt] = self.cfg_params[k]
241     pass
242     pass
243     pass
244    
245     return
246    
247 nsmirnov 1.3 def processOptions_(self, opts):
248 nsmirnov 1.1 """
249     Processes the command-line options.
250     """
251    
252     for opt in opts.keys():
253     val = opts[opt]
254    
255 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
256     if opt in self.main_actions:
257     self.cfg_params['CRAB.'+opt[1:]] = val
258     continue
259     if opt in self.aux_actions:
260     self.cfg_params['CRAB.'+opt[1:]] = val
261     continue
262 nsmirnov 1.1
263    
264     elif ( opt == '-cfg' ):
265     pass
266    
267     elif ( opt in ('-continue', '-c') ):
268 nsmirnov 1.4 # Already processed in processContinueOption_()
269 nsmirnov 1.1 pass
270    
271     elif ( opt == '-jobtype' ):
272     if val : self.job_type_name = string.upper(val)
273     else : usage()
274     pass
275    
276     elif ( opt == '-Q' ):
277     self.flag_quiet = 1
278     pass
279    
280     elif ( opt == '-debug' ):
281 slacapra 1.6 if val: self.debug_level = int(val)
282     else: self.debug_level = 1
283 nsmirnov 1.1 pass
284    
285     elif ( opt == '-scheduler' ):
286     if val: self.scheduler_name = val
287     else:
288     print common.prog_name+". No value for '-scheduler'."
289     usage()
290     pass
291     pass
292    
293 nsmirnov 1.3 elif string.find(opt,'.') == -1:
294     print common.prog_name+'. Unrecognized option '+opt
295     usage()
296     pass
297 nsmirnov 1.1
298 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
299     if string.find(opt,'.') == -1 :
300     self.cfg_params['CRAB.'+opt[1:]] = val
301 nsmirnov 1.1 pass
302 nsmirnov 1.3 else:
303 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
304     self.cfg_params[opt[1:]] = val
305     pass
306     pass
307     return
308    
309 slacapra 1.8 def parseRange_(self, aRange):
310 nsmirnov 1.4 """
311 slacapra 1.8 Takes as the input a string with a range defined in any of the following
312     way, including combination, and return a tuple with the ints defined
313     according to following table. A consistency check is done.
314     NB: the first job is "1", not "0".
315     'all' -> [1,2,..., NJobs]
316     '' -> [1,2,..., NJobs]
317     'n1' -> [n1]
318     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
319     'n1,n2' -> [n1, n2]
320     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
321     """
322     result = []
323    
324 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
325     if aRange=='all' or aRange==None or aRange=='':
326 slacapra 1.8 result=range(0,common.jobDB.nJobs())
327     return result
328 slacapra 1.9 elif aRange=='0':
329     return result
330 slacapra 1.8
331     subRanges = string.split(aRange, ',')
332     for subRange in subRanges:
333     result = result+self.parseSimpleRange_(subRange)
334    
335     if self.checkUniqueness_(result):
336     return result
337     else:
338     print "Error ", result
339     return []
340    
341     def checkUniqueness_(self, list):
342     """
343 slacapra 1.9 check if a list contains only unique elements
344 slacapra 1.8 """
345    
346     uniqueList = []
347     # use a list comprehension statement (takes a while to understand)
348    
349     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
350    
351     return (len(list)==len(uniqueList))
352    
353     def parseSimpleRange_(self, aRange):
354     """
355     Takes as the input a string with two integers separated by
356     the minus sign and returns the tuple with these numbers:
357     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
358     'n1' -> [n1]
359     """
360     (start, end) = (None, None)
361    
362     result = []
363     minus = string.find(aRange, '-')
364     if ( minus < 0 ):
365     if isInt(aRange) and int(aRange)>0:
366     result.append(int(aRange)-1)
367     else:
368 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
369 nsmirnov 1.4 pass
370     else:
371 slacapra 1.8 (start, end) = string.split(aRange, '-')
372     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
373     result=range(int(start)-1, int(end))
374     else:
375     print "ERROR ", start, end
376    
377     return result
378 nsmirnov 1.4
379 nsmirnov 1.3 def initializeActions_(self, opts):
380 nsmirnov 1.1 """
381     For each user action instantiate a corresponding
382     object and put it in the action dictionary.
383     """
384    
385     for opt in opts.keys():
386     val = opts[opt]
387    
388     if ( opt == '-create' ):
389     if val:
390     if ( isInt(val) ):
391     ncjobs = int(val)
392     elif ( val == 'all'):
393     ncjobs = val
394     else:
395 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
396     msg += ' Must be an integer or "all"'
397 slacapra 1.8 msg += ' Generic range is not allowed"'
398 nsmirnov 1.5 raise CrabException(msg)
399 nsmirnov 1.1 pass
400     else: ncjobs = 'all'
401    
402     if ncjobs != 0:
403 nsmirnov 1.3 # Instantiate Creator object
404 nsmirnov 1.1 creator = Creator(self.job_type_name,
405     self.cfg_params,
406     ncjobs)
407     self.actions[opt] = creator
408 nsmirnov 1.3
409     # Initialize the JobDB object if needed
410 nsmirnov 1.1 if not self.flag_continue:
411     common.jobDB.create(creator.nJobs())
412     pass
413 nsmirnov 1.3
414     # Create and initialize JobList
415    
416     common.job_list = JobList(common.jobDB.nJobs(),
417     creator.jobType())
418    
419     common.job_list.setScriptNames(self.job_type_name+'.sh')
420     common.job_list.setJDLNames(self.job_type_name+'.jdl')
421 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
422 slacapra 1.9
423     creator.writeJobsSpecsToDB()
424 nsmirnov 1.1 pass
425 nsmirnov 1.3 pass
426 nsmirnov 1.1
427     elif ( opt == '-submit' ):
428 nsmirnov 1.7 nj_list = self.parseRange_(val)
429 nsmirnov 1.5
430     if len(nj_list) != 0:
431 nsmirnov 1.3 # Instantiate Submitter object
432 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
433 nsmirnov 1.3
434     # Create and initialize JobList
435    
436     if len(common.job_list) == 0 :
437     common.job_list = JobList(common.jobDB.nJobs(),
438     None)
439     common.job_list.setJDLNames(self.job_type_name+'.jdl')
440     pass
441 nsmirnov 1.1 pass
442 nsmirnov 1.3 pass
443 nsmirnov 1.1
444 nsmirnov 1.4 elif ( opt == '-list' ):
445 slacapra 1.8 jobs = self.parseRange_(val)
446    
447     common.jobDB.dump(jobs)
448 nsmirnov 1.4 pass
449    
450     elif ( opt == '-status' ):
451 slacapra 1.8 jobs = self.parseRange_(val)
452    
453 slacapra 1.9 if len(jobs) != 0:
454     # Instantiate Submitter object
455     self.actions[opt] = Status(self.cfg_params, jobs)
456 nsmirnov 1.1 pass
457     pass
458    
459 nsmirnov 1.4 elif ( opt == '-kill' ):
460 slacapra 1.9 if val:
461     jobs = self.parseRange_(val)
462 slacapra 1.8
463 slacapra 1.9 for nj in jobs:
464     st = common.jobDB.status(nj)
465     if st == 'S':
466     jid = common.jobDB.jobId(nj)
467     common.logger.message("Killing job # "+`(nj+1)`)
468     common.scheduler.cancel(jid)
469     common.jobDB.setStatus(nj, 'K')
470     pass
471 nsmirnov 1.4 pass
472 slacapra 1.9
473     common.jobDB.save()
474 nsmirnov 1.1 pass
475 slacapra 1.9 else:
476     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
477 nsmirnov 1.1
478 slacapra 1.8 elif ( opt == '-getoutput' ):
479     jobs = self.parseRange_(val)
480    
481 slacapra 1.12 jobs_done = []
482 slacapra 1.8 for nj in jobs:
483 nsmirnov 1.4 st = common.jobDB.status(nj)
484 slacapra 1.12 if st == 'D':
485     jobs_done.append(nj)
486     pass
487     elif st == 'S':
488 nsmirnov 1.4 jid = common.jobDB.jobId(nj)
489 slacapra 1.9 currStatus = common.scheduler.queryStatus(jid)
490     if currStatus=="Done":
491 slacapra 1.12 jobs_done.append(nj)
492 slacapra 1.9 else:
493     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
494 nsmirnov 1.4 common.logger.message(msg)
495     pass
496 slacapra 1.8 else:
497 slacapra 1.9 common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
498 slacapra 1.12 pass
499     pass
500    
501     for nj in jobs_done:
502     jid = common.jobDB.jobId(nj)
503     dir = common.scheduler.getOutput(jid)
504     common.jobDB.setStatus(nj, 'Y')
505    
506     # Rename the directory with results to smth readable
507     new_dir = common.work_space.resDir()
508     try:
509     files = os.listdir(dir)
510     for file in files:
511     os.rename(dir+'/'+file, new_dir+'/'+file)
512     os.rmdir(dir)
513     except OSError, e:
514     msg = 'rename files from '+dir+' to '+new_dir+' error: '
515     msg += str(e)
516     common.logger.message(msg)
517     # ignore error
518     pass
519    
520    
521     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
522     common.logger.message(msg)
523 nsmirnov 1.4 pass
524    
525     common.jobDB.save()
526     pass
527    
528     elif ( opt == '-resubmit' ):
529 slacapra 1.11 if val:
530     jobs = self.parseRange_(val)
531 slacapra 1.8
532 slacapra 1.11 # create a list of jobs to be resubmitted.
533 slacapra 1.8
534 slacapra 1.11 nj_list = []
535     for nj in jobs:
536     st = common.jobDB.status(nj)
537 slacapra 1.8
538 slacapra 1.11 if st in ['K','A']:
539     nj_list.append(nj)
540     elif st == 'Y':
541     # here I would like to move the old output to a new place
542     # I would need the name of the output files.
543     # these infos are in the jobType class, which is not accessible from here!
544     outSandbox = common.jobDB.outputSandbox(nj)
545    
546     resDir = common.work_space.resDir()
547     resDirSave = resDir + self.current_time
548     os.mkdir(resDirSave)
549    
550     for file in outSandbox:
551     if os.path.exists(resDir+'/'+file):
552     os.rename(resDir+'/'+file, resDirSave+'/'+file)
553     common.logger.message('Output file '+file+' moved to '+resDirSave)
554     pass
555     nj_list.append(nj)
556 slacapra 1.12 st = common.jobDB.setStatus(nj,'C')
557 slacapra 1.11 elif st == 'D':
558     ## Done but not yet retrieved
559     ## retrieve job, then go to previous ('Y') case
560     ## TODO
561     pass
562     else:
563     common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
564 slacapra 1.9 pass
565 slacapra 1.8
566 slacapra 1.11 if len(nj_list) != 0:
567     # Instantiate Submitter object
568     self.actions[opt] = Submitter(self.cfg_params, nj_list)
569    
570     # Create and initialize JobList
571    
572     if len(common.job_list) == 0 :
573     common.job_list = JobList(common.jobDB.nJobs(),
574     None)
575     common.job_list.setJDLNames(self.job_type_name+'.jdl')
576     pass
577 slacapra 1.8 pass
578     pass
579 slacapra 1.11 else:
580     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
581     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
582     pass
583 slacapra 1.8 pass
584    
585     elif ( opt == '-cancelAndResubmit' ):
586     jobs = self.parseRange_(val)
587 nsmirnov 1.5
588 nsmirnov 1.7 # Cancel submitted jobs.
589 nsmirnov 1.5
590 slacapra 1.8 nj_list = []
591     for nj in jobs:
592 nsmirnov 1.5 st = common.jobDB.status(nj)
593     if st == 'S':
594     jid = common.jobDB.jobId(nj)
595     common.scheduler.cancel(jid)
596     st = 'K'
597     common.jobDB.setStatus(nj, st)
598     pass
599 nsmirnov 1.7 pass
600 nsmirnov 1.5
601 slacapra 1.8 if st != 'X': nj_list.append(nj)
602     pass
603 nsmirnov 1.5
604     if len(nj_list) != 0:
605     # Instantiate Submitter object
606     self.actions[opt] = Submitter(self.cfg_params, nj_list)
607    
608     # Create and initialize JobList
609    
610     if len(common.job_list) == 0 :
611     common.job_list = JobList(common.jobDB.nJobs(),
612     None)
613     common.job_list.setJDLNames(self.job_type_name+'.jdl')
614     pass
615     pass
616 nsmirnov 1.4 pass
617    
618 slacapra 1.8 elif ( opt == '-check' ):
619     jobs = self.parseRange_(val)
620     nj_list = []
621     for nj in jobs:
622     st = common.jobDB.status(nj)
623 slacapra 1.12 if st == 'C': nj_list.append(nj)
624 slacapra 1.8 pass
625    
626     if len(nj_list) != 0:
627     # Instantiate Submitter object
628     self.actions[opt] = Checker(self.cfg_params, nj_list)
629    
630     # Create and initialize JobList
631    
632     if len(common.job_list) == 0 :
633     common.job_list = JobList(common.jobDB.nJobs(), None)
634     common.job_list.setJDLNames(self.job_type_name+'.jdl')
635     pass
636     pass
637    
638 slacapra 1.9 elif ( opt == '-postMortem' ):
639     jobs = self.parseRange_(val)
640     nj_list = []
641     for nj in jobs:
642     st = common.jobDB.status(nj)
643 slacapra 1.11 if st in ['Y','A','D']: nj_list.append(nj)
644 slacapra 1.9 pass
645    
646     if len(nj_list) != 0:
647     # Instantiate Submitter object
648     self.actions[opt] = PostMortem(self.cfg_params, nj_list)
649    
650     # Create and initialize JobList
651    
652     if len(common.job_list) == 0 :
653     common.job_list = JobList(common.jobDB.nJobs(), None)
654     common.job_list.setJDLNames(self.job_type_name+'.jdl')
655     pass
656     pass
657    
658     elif ( opt == '-clean' ):
659     if val != None:
660     raise CrabException("No range allowed for '-clean'")
661    
662     submittedJobs=0
663 slacapra 1.11 doneJobs=0
664 slacapra 1.12 try:
665     for nj in range(0,common.jobDB.nJobs()):
666     st = common.jobDB.status(nj)
667     if st == 'S':
668     submittedJobs = submittedJobs+1
669     if st == 'D':
670     doneJobs = doneJobs+1
671     pass
672     pass
673     except DBException:
674     common.logger.debug(5,'DB not found, so delete all')
675 nsmirnov 1.1 pass
676 slacapra 1.9
677 slacapra 1.11 if submittedJobs or doneJobs:
678     msg = "There are still "
679     if submittedJobs:
680     msg= msg+str(submittedJobs)+" jobs submitted. Kill them '-kill' before '-clean'"
681     if (submittedJobs and doneJobs):
682     msg = msg + "and \nalso"
683     if doneJobs:
684     msg= msg+str(doneJobs)+" jobs Done. Get their outputs '-getoutput' before '-clean'"
685     raise CrabException(msg)
686 slacapra 1.9
687     msg = 'directory '+common.work_space.topDir()+' removed'
688     common.work_space.delete()
689     common.logger.message(msg)
690    
691 nsmirnov 1.1 pass
692 slacapra 1.9
693 nsmirnov 1.1 pass
694     return
695    
696 nsmirnov 1.3 def createWorkingSpace_(self):
697 slacapra 1.9 new_dir = ''
698    
699     try:
700     new_dir = self.cfg_params['USER.ui_working_dir']
701     except KeyError:
702     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
703     new_dir = self.cwd + new_dir
704     pass
705    
706 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
707     common.work_space.create()
708     return
709    
710 nsmirnov 1.3 def loadConfiguration_(self, opts):
711 nsmirnov 1.1
712     save_opts = common.work_space.loadSavedOptions()
713    
714     # Override saved options with new command-line options
715    
716     for k in opts.keys():
717     save_opts[k] = opts[k]
718     pass
719    
720     # Return updated options
721     return save_opts
722    
723 nsmirnov 1.3 def createLogger_(self, args):
724 nsmirnov 1.1
725     log = Logger()
726     log.quiet(self.flag_quiet)
727     log.setDebugLevel(self.debug_level)
728     log.write(args+'\n')
729 nsmirnov 1.3 log.message(self.headerString_())
730 nsmirnov 1.1 log.flush()
731     common.logger = log
732     return
733    
734 nsmirnov 1.3 def updateHistory_(self, args):
735 nsmirnov 1.1 history_fname = common.prog_name+'.history'
736     history_file = open(history_fname, 'a')
737     history_file.write(self.current_time+': '+args+'\n')
738     history_file.close()
739     return
740    
741 nsmirnov 1.3 def headerString_(self):
742 nsmirnov 1.1 """
743     Creates a string describing program options either given in
744     the command line or their default values.
745     """
746     header = common.prog_name + ' (version ' + common.prog_version_str + \
747     ') running on ' + \
748     time.ctime(time.time())+'\n\n' + \
749     common.prog_name+'. Working options:\n'
750     header = header +\
751     ' scheduler ' + self.scheduler_name + '\n'+\
752     ' job type ' + self.job_type_name + '\n'+\
753     ' working directory ' + common.work_space.topDir()\
754     + '\n'
755     return header
756    
757 nsmirnov 1.3 def createScheduler_(self):
758 nsmirnov 1.1 """
759     Creates a scheduler object instantiated by its name.
760     """
761     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
762     file_name = klass_name
763     try:
764     klass = importName(file_name, klass_name)
765     except KeyError:
766     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
767     raise CrabException(msg)
768     except ImportError, e:
769     msg = 'Cannot create scheduler '+self.scheduler_name
770     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
771     msg += str(e)
772     raise CrabException(msg)
773    
774     common.scheduler = klass()
775     common.scheduler.configure(self.cfg_params)
776     return
777    
778     def run(self):
779     """
780     For each
781     """
782    
783     for act in self.main_actions:
784     if act in self.actions.keys(): self.actions[act].run()
785     pass
786    
787     for act in self.aux_actions:
788     if act in self.actions.keys(): self.actions[act].run()
789     pass
790     return
791    
792     ###########################################################################
793     def processHelpOptions(opts):
794    
795 slacapra 1.11 if len(opts):
796     for opt in opts.keys():
797     if opt in ('-v', '-version', '--version') :
798     print Crab.version()
799     return 1
800     if opt in ('-h','-help','--help') :
801     if opts[opt] : help(opts[opt])
802     else: help()
803     return 1
804     else:
805     usage()
806 nsmirnov 1.1
807     return 0
808    
809     ###########################################################################
810     if __name__ == '__main__':
811    
812     # Parse command-line options and create a dictionary with
813     # key-value pairs.
814    
815     options = parseOptions(sys.argv[1:])
816    
817     # Process "help" options, such as '-help', '-version'
818    
819 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
820 nsmirnov 1.1
821     # Create, initialize, and run a Crab object
822    
823     try:
824 nsmirnov 1.3 crab = Crab(options)
825 nsmirnov 1.1 crab.run()
826     except CrabException, e:
827     print '\n' + common.prog_name + ': ' + str(e) + '\n'
828     if common.logger:
829     common.logger.write('ERROR: '+str(e)+'\n')
830     pass
831     pass
832    
833     pass