ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.13
Committed: Tue Aug 30 14:24:42 2005 UTC (19 years, 8 months ago) by spiga
Content type: text/x-python
Branch: MAIN
Changes since 1.12: +17 -1 lines
Log Message: 
crab-mon

File Contents

# User Rev Content
1 slacapra 1.9 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 nsmirnov 1.1 import common
15 spiga 1.13 import Statistic
16 nsmirnov 1.1
17     import sys, os, time, string
18    
19     ###########################################################################
20     class Crab:
21 nsmirnov 1.3 def __init__(self, opts):
22 nsmirnov 1.1
23     # The order of main_actions is important !
24 slacapra 1.9 self.main_actions = [ '-create', '-submit', '-monitor' ]
25 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
26 slacapra 1.9 '-resubmit' , '-cancelAndResubmit', '-check', '-postMortem', '-clean']
27 nsmirnov 1.1
28     # Dictionary of actions, e.g. '-create' -> object of class Creator
29     self.actions = {}
30    
31     # Configuration file
32     self.cfg_fname = None
33     # Dictionary with configuration parameters
34     self.cfg_params = {}
35    
36     # Current working directory
37     self.cwd = os.getcwd()+'/'
38     # Current time in format 'yymmdd_hhmmss'
39     self.current_time = time.strftime('%y%m%d_%H%M%S',
40     time.localtime(time.time()))
41    
42 nsmirnov 1.3 # Session name (?) Do we need this ?
43 nsmirnov 1.1 self.name = '0'
44    
45     # Job type
46     self.job_type_name = None
47    
48     # Continuation flag
49     self.flag_continue = 0
50    
51     # quiet mode, i.e. no output on screen
52     self.flag_quiet = 0
53     # produce more output
54     self.debug_level = 0
55    
56 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
57 nsmirnov 1.1 self.scheduler_name = 'edg'
58    
59 nsmirnov 1.3 self.initialize_(opts)
60 nsmirnov 1.1
61     return
62    
63 nsmirnov 1.7 def version():
64 nsmirnov 1.1 return common.prog_version_str
65    
66 nsmirnov 1.7 version = staticmethod(version)
67    
68 nsmirnov 1.3 def initialize_(self, opts):
69 nsmirnov 1.1
70     # Process the '-continue' option first because
71     # in the case of continuation the CRAB configuration
72     # parameters are loaded from already existing Working Space.
73 nsmirnov 1.3 self.processContinueOption_(opts)
74 nsmirnov 1.1
75     # Process ini-file first, then command line options
76     # because they override ini-file settings.
77    
78 nsmirnov 1.3 self.processIniFile_(opts)
79 nsmirnov 1.1
80 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
81 nsmirnov 1.1
82 nsmirnov 1.3 self.processOptions_(opts)
83 nsmirnov 1.1
84     if not self.flag_continue:
85 nsmirnov 1.3 self.createWorkingSpace_()
86 slacapra 1.9 optsToBeSaved={}
87     for it in opts.keys():
88     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
89     pass
90     else:
91     optsToBeSaved[it]=opts[it]
92     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
93 nsmirnov 1.1 pass
94    
95     # At this point all configuration options have been read.
96    
97     args = string.join(sys.argv,' ')
98 slacapra 1.11
99 nsmirnov 1.3 self.updateHistory_(args)
100 slacapra 1.11
101 nsmirnov 1.3 self.createLogger_(args)
102 slacapra 1.11
103 nsmirnov 1.1 common.jobDB = JobDB()
104 slacapra 1.11
105 nsmirnov 1.3 if self.flag_continue:
106 slacapra 1.12 try:
107     common.jobDB.load()
108     common.logger.debug(6, str(common.jobDB))
109     except DBException,e:
110     pass
111 nsmirnov 1.3 pass
112 slacapra 1.11
113 nsmirnov 1.3 self.createScheduler_()
114 slacapra 1.11
115 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
116     common.logger.debug(6, 'Used properties:')
117     keys = self.cfg_params.keys()
118     keys.sort()
119     for k in keys:
120     if self.cfg_params[k]:
121     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
122     pass
123     else:
124     common.logger.debug(6, ' '+k+' : ')
125     pass
126     pass
127     common.logger.debug(6, 'End of used properties.\n')
128     pass
129     self.initializeActions_(opts)
130 nsmirnov 1.1 return
131    
132 nsmirnov 1.3 def processContinueOption_(self,opts):
133 nsmirnov 1.1
134     continue_dir = None
135 nsmirnov 1.4
136     # Look for the '-continue' option.
137    
138 nsmirnov 1.1 for opt in opts.keys():
139     if ( opt in ('-continue','-c') ):
140     self.flag_continue = 1
141     val = opts[opt]
142     if val:
143     if val[0] == '/': continue_dir = val # abs path
144     else: continue_dir = self.cwd + val # rel path
145     pass
146 nsmirnov 1.4 break
147     pass
148    
149     # Look for actions which has sense only with '-continue'
150    
151     if not self.flag_continue:
152     for opt in opts.keys():
153 slacapra 1.6 if ( opt in (self.aux_actions) ):
154 nsmirnov 1.4 self.flag_continue = 1
155     break
156 nsmirnov 1.1 pass
157     pass
158 slacapra 1.6 submit_flag=0
159     create_flag=0
160     for opt in opts.keys():
161     if opt == "-submit": submit_flag=1
162     if opt == "-create": create_flag=1
163     pass
164     if (submit_flag and not create_flag):
165 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
166     raise CrabException(msg)
167 slacapra 1.6 pass
168 nsmirnov 1.1
169     if not self.flag_continue: return
170    
171     if not continue_dir:
172     prefix = common.prog_name + '_' + self.name + '_'
173     continue_dir = findLastWorkDir(prefix)
174     pass
175    
176     if not continue_dir:
177     raise CrabException('Cannot find last working directory.')
178    
179     if not os.path.exists(continue_dir):
180     msg = 'Cannot continue because the working directory <'
181     msg += continue_dir
182     msg += '> does not exist.'
183     raise CrabException(msg)
184    
185     # Instantiate WorkSpace
186     common.work_space = WorkSpace(continue_dir)
187    
188     return
189    
190 nsmirnov 1.3 def processIniFile_(self, opts):
191 nsmirnov 1.1 """
192     Processes a configuration INI-file.
193     """
194    
195     # Extract cfg-file name from the cmd-line options.
196    
197     for opt in opts.keys():
198     if ( opt == '-cfg' ):
199     if self.flag_continue:
200     raise CrabException('-continue and -cfg cannot coexist.')
201     if opts[opt] : self.cfg_fname = opts[opt]
202     else : usage()
203     pass
204    
205     elif ( opt == '-name' ):
206     self.name = opts[opt]
207     pass
208    
209     pass
210    
211     # Set default cfg-fname
212    
213     if self.cfg_fname == None:
214     if self.flag_continue:
215     self.cfg_fname = common.work_space.cfgFileName()
216     else:
217     self.cfg_fname = common.prog_name+'.cfg'
218     pass
219     pass
220    
221     # Load cfg-file
222    
223     if string.lower(self.cfg_fname) != 'none':
224     if os.path.exists(self.cfg_fname):
225     self.cfg_params = loadConfig(self.cfg_fname)
226     pass
227     else:
228     msg = 'cfg-file '+self.cfg_fname+' not found.'
229     raise CrabException(msg)
230     pass
231     pass
232    
233     # process the [CRAB] section
234    
235     lhp = len('CRAB.')
236     for k in self.cfg_params.keys():
237     if len(k) >= lhp and k[:lhp] == 'CRAB.':
238     opt = '-'+k[lhp:]
239     if len(opt) >= 3 and opt[:3] == '-__': continue
240     if opt not in opts.keys():
241     opts[opt] = self.cfg_params[k]
242     pass
243     pass
244     pass
245    
246     return
247    
248 nsmirnov 1.3 def processOptions_(self, opts):
249 nsmirnov 1.1 """
250     Processes the command-line options.
251     """
252    
253     for opt in opts.keys():
254     val = opts[opt]
255    
256 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
257     if opt in self.main_actions:
258     self.cfg_params['CRAB.'+opt[1:]] = val
259     continue
260     if opt in self.aux_actions:
261     self.cfg_params['CRAB.'+opt[1:]] = val
262     continue
263 nsmirnov 1.1
264    
265     elif ( opt == '-cfg' ):
266     pass
267    
268     elif ( opt in ('-continue', '-c') ):
269 nsmirnov 1.4 # Already processed in processContinueOption_()
270 nsmirnov 1.1 pass
271    
272     elif ( opt == '-jobtype' ):
273     if val : self.job_type_name = string.upper(val)
274     else : usage()
275     pass
276    
277     elif ( opt == '-Q' ):
278     self.flag_quiet = 1
279     pass
280    
281     elif ( opt == '-debug' ):
282 slacapra 1.6 if val: self.debug_level = int(val)
283     else: self.debug_level = 1
284 nsmirnov 1.1 pass
285    
286     elif ( opt == '-scheduler' ):
287     if val: self.scheduler_name = val
288     else:
289     print common.prog_name+". No value for '-scheduler'."
290     usage()
291     pass
292     pass
293    
294 nsmirnov 1.3 elif string.find(opt,'.') == -1:
295     print common.prog_name+'. Unrecognized option '+opt
296     usage()
297     pass
298 nsmirnov 1.1
299 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
300     if string.find(opt,'.') == -1 :
301     self.cfg_params['CRAB.'+opt[1:]] = val
302 nsmirnov 1.1 pass
303 nsmirnov 1.3 else:
304 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
305     self.cfg_params[opt[1:]] = val
306     pass
307     pass
308     return
309    
310 slacapra 1.8 def parseRange_(self, aRange):
311 nsmirnov 1.4 """
312 slacapra 1.8 Takes as the input a string with a range defined in any of the following
313     way, including combination, and return a tuple with the ints defined
314     according to following table. A consistency check is done.
315     NB: the first job is "1", not "0".
316     'all' -> [1,2,..., NJobs]
317     '' -> [1,2,..., NJobs]
318     'n1' -> [n1]
319     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
320     'n1,n2' -> [n1, n2]
321     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
322     """
323     result = []
324    
325 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
326     if aRange=='all' or aRange==None or aRange=='':
327 slacapra 1.8 result=range(0,common.jobDB.nJobs())
328     return result
329 slacapra 1.9 elif aRange=='0':
330     return result
331 slacapra 1.8
332     subRanges = string.split(aRange, ',')
333     for subRange in subRanges:
334     result = result+self.parseSimpleRange_(subRange)
335    
336     if self.checkUniqueness_(result):
337     return result
338     else:
339     print "Error ", result
340     return []
341    
342     def checkUniqueness_(self, list):
343     """
344 slacapra 1.9 check if a list contains only unique elements
345 slacapra 1.8 """
346    
347     uniqueList = []
348     # use a list comprehension statement (takes a while to understand)
349    
350     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
351    
352     return (len(list)==len(uniqueList))
353    
354     def parseSimpleRange_(self, aRange):
355     """
356     Takes as the input a string with two integers separated by
357     the minus sign and returns the tuple with these numbers:
358     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
359     'n1' -> [n1]
360     """
361     (start, end) = (None, None)
362    
363     result = []
364     minus = string.find(aRange, '-')
365     if ( minus < 0 ):
366     if isInt(aRange) and int(aRange)>0:
367     result.append(int(aRange)-1)
368     else:
369 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
370 nsmirnov 1.4 pass
371     else:
372 slacapra 1.8 (start, end) = string.split(aRange, '-')
373     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
374     result=range(int(start)-1, int(end))
375     else:
376     print "ERROR ", start, end
377    
378     return result
379 nsmirnov 1.4
380 nsmirnov 1.3 def initializeActions_(self, opts):
381 nsmirnov 1.1 """
382     For each user action instantiate a corresponding
383     object and put it in the action dictionary.
384     """
385    
386     for opt in opts.keys():
387     val = opts[opt]
388    
389     if ( opt == '-create' ):
390     if val:
391     if ( isInt(val) ):
392     ncjobs = int(val)
393     elif ( val == 'all'):
394     ncjobs = val
395     else:
396 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
397     msg += ' Must be an integer or "all"'
398 slacapra 1.8 msg += ' Generic range is not allowed"'
399 nsmirnov 1.5 raise CrabException(msg)
400 nsmirnov 1.1 pass
401     else: ncjobs = 'all'
402    
403     if ncjobs != 0:
404 nsmirnov 1.3 # Instantiate Creator object
405 nsmirnov 1.1 creator = Creator(self.job_type_name,
406     self.cfg_params,
407     ncjobs)
408     self.actions[opt] = creator
409 nsmirnov 1.3
410     # Initialize the JobDB object if needed
411 nsmirnov 1.1 if not self.flag_continue:
412     common.jobDB.create(creator.nJobs())
413     pass
414 nsmirnov 1.3
415     # Create and initialize JobList
416    
417     common.job_list = JobList(common.jobDB.nJobs(),
418     creator.jobType())
419    
420     common.job_list.setScriptNames(self.job_type_name+'.sh')
421     common.job_list.setJDLNames(self.job_type_name+'.jdl')
422 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
423 slacapra 1.9
424     creator.writeJobsSpecsToDB()
425 nsmirnov 1.1 pass
426 nsmirnov 1.3 pass
427 nsmirnov 1.1
428     elif ( opt == '-submit' ):
429 nsmirnov 1.7 nj_list = self.parseRange_(val)
430 nsmirnov 1.5
431     if len(nj_list) != 0:
432 nsmirnov 1.3 # Instantiate Submitter object
433 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
434 nsmirnov 1.3
435     # Create and initialize JobList
436    
437     if len(common.job_list) == 0 :
438     common.job_list = JobList(common.jobDB.nJobs(),
439     None)
440     common.job_list.setJDLNames(self.job_type_name+'.jdl')
441     pass
442 nsmirnov 1.1 pass
443 nsmirnov 1.3 pass
444 nsmirnov 1.1
445 nsmirnov 1.4 elif ( opt == '-list' ):
446 slacapra 1.8 jobs = self.parseRange_(val)
447    
448     common.jobDB.dump(jobs)
449 nsmirnov 1.4 pass
450    
451     elif ( opt == '-status' ):
452 slacapra 1.8 jobs = self.parseRange_(val)
453    
454 slacapra 1.9 if len(jobs) != 0:
455     # Instantiate Submitter object
456     self.actions[opt] = Status(self.cfg_params, jobs)
457 nsmirnov 1.1 pass
458     pass
459    
460 nsmirnov 1.4 elif ( opt == '-kill' ):
461 slacapra 1.9 if val:
462     jobs = self.parseRange_(val)
463 slacapra 1.8
464 slacapra 1.9 for nj in jobs:
465     st = common.jobDB.status(nj)
466     if st == 'S':
467     jid = common.jobDB.jobId(nj)
468     common.logger.message("Killing job # "+`(nj+1)`)
469     common.scheduler.cancel(jid)
470     common.jobDB.setStatus(nj, 'K')
471     pass
472 nsmirnov 1.4 pass
473 slacapra 1.9
474     common.jobDB.save()
475 nsmirnov 1.1 pass
476 slacapra 1.9 else:
477     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
478 nsmirnov 1.1
479 slacapra 1.8 elif ( opt == '-getoutput' ):
480     jobs = self.parseRange_(val)
481    
482 spiga 1.13 fileCODE1 = open(common.work_space.logDir()+"/.code","r")
483     array = fileCODE1.read().split('::')
484     self.ID1 = array[0]
485     self.NJC = array[1]
486     self.dataset = array[2]
487     self.owner = array[3]
488     fileCODE1.close()
489    
490    
491 slacapra 1.12 jobs_done = []
492 slacapra 1.8 for nj in jobs:
493 nsmirnov 1.4 st = common.jobDB.status(nj)
494 slacapra 1.12 if st == 'D':
495     jobs_done.append(nj)
496     pass
497     elif st == 'S':
498 nsmirnov 1.4 jid = common.jobDB.jobId(nj)
499 slacapra 1.9 currStatus = common.scheduler.queryStatus(jid)
500     if currStatus=="Done":
501 slacapra 1.12 jobs_done.append(nj)
502 slacapra 1.9 else:
503     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
504 nsmirnov 1.4 common.logger.message(msg)
505     pass
506 slacapra 1.8 else:
507 slacapra 1.9 common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
508 slacapra 1.12 pass
509     pass
510    
511     for nj in jobs_done:
512     jid = common.jobDB.jobId(nj)
513     dir = common.scheduler.getOutput(jid)
514     common.jobDB.setStatus(nj, 'Y')
515    
516     # Rename the directory with results to smth readable
517     new_dir = common.work_space.resDir()
518     try:
519     files = os.listdir(dir)
520     for file in files:
521     os.rename(dir+'/'+file, new_dir+'/'+file)
522     os.rmdir(dir)
523     except OSError, e:
524     msg = 'rename files from '+dir+' to '+new_dir+' error: '
525     msg += str(e)
526     common.logger.message(msg)
527     # ignore error
528     pass
529    
530 spiga 1.13 destination = common.scheduler.queryDest(jid).split(":")[0]
531     ID3 = jid.split("/")[3]
532     broker = jid.split("/")[2].split(":")[0]
533     resFlag = 0
534     exCode = common.scheduler.getExitStatus(jid)
535     Statistic.notify('retrieved',resFlag,exCode,self.dataset,self.owner,destination,broker,ID3,self.ID1,self.NJC)
536 slacapra 1.12
537     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
538     common.logger.message(msg)
539 nsmirnov 1.4 pass
540    
541     common.jobDB.save()
542     pass
543    
544     elif ( opt == '-resubmit' ):
545 slacapra 1.11 if val:
546     jobs = self.parseRange_(val)
547 slacapra 1.8
548 slacapra 1.11 # create a list of jobs to be resubmitted.
549 slacapra 1.8
550 slacapra 1.11 nj_list = []
551     for nj in jobs:
552     st = common.jobDB.status(nj)
553 slacapra 1.8
554 slacapra 1.11 if st in ['K','A']:
555     nj_list.append(nj)
556     elif st == 'Y':
557     # here I would like to move the old output to a new place
558     # I would need the name of the output files.
559     # these infos are in the jobType class, which is not accessible from here!
560     outSandbox = common.jobDB.outputSandbox(nj)
561    
562     resDir = common.work_space.resDir()
563     resDirSave = resDir + self.current_time
564     os.mkdir(resDirSave)
565    
566     for file in outSandbox:
567     if os.path.exists(resDir+'/'+file):
568     os.rename(resDir+'/'+file, resDirSave+'/'+file)
569     common.logger.message('Output file '+file+' moved to '+resDirSave)
570     pass
571     nj_list.append(nj)
572 spiga 1.13 st = common.jobDB.setStatus(nj,'RC')
573 slacapra 1.11 elif st == 'D':
574     ## Done but not yet retrieved
575     ## retrieve job, then go to previous ('Y') case
576     ## TODO
577     pass
578     else:
579     common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
580 slacapra 1.9 pass
581 slacapra 1.8
582 slacapra 1.11 if len(nj_list) != 0:
583     # Instantiate Submitter object
584     self.actions[opt] = Submitter(self.cfg_params, nj_list)
585    
586     # Create and initialize JobList
587    
588     if len(common.job_list) == 0 :
589     common.job_list = JobList(common.jobDB.nJobs(),
590     None)
591     common.job_list.setJDLNames(self.job_type_name+'.jdl')
592     pass
593 slacapra 1.8 pass
594     pass
595 slacapra 1.11 else:
596     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
597     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
598     pass
599 slacapra 1.8 pass
600    
601     elif ( opt == '-cancelAndResubmit' ):
602     jobs = self.parseRange_(val)
603 nsmirnov 1.5
604 nsmirnov 1.7 # Cancel submitted jobs.
605 nsmirnov 1.5
606 slacapra 1.8 nj_list = []
607     for nj in jobs:
608 nsmirnov 1.5 st = common.jobDB.status(nj)
609     if st == 'S':
610     jid = common.jobDB.jobId(nj)
611     common.scheduler.cancel(jid)
612     st = 'K'
613     common.jobDB.setStatus(nj, st)
614     pass
615 nsmirnov 1.7 pass
616 nsmirnov 1.5
617 slacapra 1.8 if st != 'X': nj_list.append(nj)
618     pass
619 nsmirnov 1.5
620     if len(nj_list) != 0:
621     # Instantiate Submitter object
622     self.actions[opt] = Submitter(self.cfg_params, nj_list)
623    
624     # Create and initialize JobList
625    
626     if len(common.job_list) == 0 :
627     common.job_list = JobList(common.jobDB.nJobs(),
628     None)
629     common.job_list.setJDLNames(self.job_type_name+'.jdl')
630     pass
631     pass
632 nsmirnov 1.4 pass
633    
634 slacapra 1.8 elif ( opt == '-check' ):
635     jobs = self.parseRange_(val)
636     nj_list = []
637     for nj in jobs:
638     st = common.jobDB.status(nj)
639 slacapra 1.12 if st == 'C': nj_list.append(nj)
640 slacapra 1.8 pass
641    
642     if len(nj_list) != 0:
643     # Instantiate Submitter object
644     self.actions[opt] = Checker(self.cfg_params, nj_list)
645    
646     # Create and initialize JobList
647    
648     if len(common.job_list) == 0 :
649     common.job_list = JobList(common.jobDB.nJobs(), None)
650     common.job_list.setJDLNames(self.job_type_name+'.jdl')
651     pass
652     pass
653    
654 slacapra 1.9 elif ( opt == '-postMortem' ):
655     jobs = self.parseRange_(val)
656     nj_list = []
657     for nj in jobs:
658     st = common.jobDB.status(nj)
659 slacapra 1.11 if st in ['Y','A','D']: nj_list.append(nj)
660 slacapra 1.9 pass
661    
662     if len(nj_list) != 0:
663     # Instantiate Submitter object
664     self.actions[opt] = PostMortem(self.cfg_params, nj_list)
665    
666     # Create and initialize JobList
667    
668     if len(common.job_list) == 0 :
669     common.job_list = JobList(common.jobDB.nJobs(), None)
670     common.job_list.setJDLNames(self.job_type_name+'.jdl')
671     pass
672     pass
673    
674     elif ( opt == '-clean' ):
675     if val != None:
676     raise CrabException("No range allowed for '-clean'")
677    
678     submittedJobs=0
679 slacapra 1.11 doneJobs=0
680 slacapra 1.12 try:
681     for nj in range(0,common.jobDB.nJobs()):
682     st = common.jobDB.status(nj)
683     if st == 'S':
684     submittedJobs = submittedJobs+1
685     if st == 'D':
686     doneJobs = doneJobs+1
687     pass
688     pass
689     except DBException:
690     common.logger.debug(5,'DB not found, so delete all')
691 nsmirnov 1.1 pass
692 slacapra 1.9
693 slacapra 1.11 if submittedJobs or doneJobs:
694     msg = "There are still "
695     if submittedJobs:
696     msg= msg+str(submittedJobs)+" jobs submitted. Kill them '-kill' before '-clean'"
697     if (submittedJobs and doneJobs):
698     msg = msg + "and \nalso"
699     if doneJobs:
700     msg= msg+str(doneJobs)+" jobs Done. Get their outputs '-getoutput' before '-clean'"
701     raise CrabException(msg)
702 slacapra 1.9
703     msg = 'directory '+common.work_space.topDir()+' removed'
704     common.work_space.delete()
705     common.logger.message(msg)
706    
707 nsmirnov 1.1 pass
708 slacapra 1.9
709 nsmirnov 1.1 pass
710     return
711    
712 nsmirnov 1.3 def createWorkingSpace_(self):
713 slacapra 1.9 new_dir = ''
714    
715     try:
716     new_dir = self.cfg_params['USER.ui_working_dir']
717     except KeyError:
718     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
719     new_dir = self.cwd + new_dir
720     pass
721    
722 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
723     common.work_space.create()
724     return
725    
726 nsmirnov 1.3 def loadConfiguration_(self, opts):
727 nsmirnov 1.1
728     save_opts = common.work_space.loadSavedOptions()
729    
730     # Override saved options with new command-line options
731    
732     for k in opts.keys():
733     save_opts[k] = opts[k]
734     pass
735    
736     # Return updated options
737     return save_opts
738    
739 nsmirnov 1.3 def createLogger_(self, args):
740 nsmirnov 1.1
741     log = Logger()
742     log.quiet(self.flag_quiet)
743     log.setDebugLevel(self.debug_level)
744     log.write(args+'\n')
745 nsmirnov 1.3 log.message(self.headerString_())
746 nsmirnov 1.1 log.flush()
747     common.logger = log
748     return
749    
750 nsmirnov 1.3 def updateHistory_(self, args):
751 nsmirnov 1.1 history_fname = common.prog_name+'.history'
752     history_file = open(history_fname, 'a')
753     history_file.write(self.current_time+': '+args+'\n')
754     history_file.close()
755     return
756    
757 nsmirnov 1.3 def headerString_(self):
758 nsmirnov 1.1 """
759     Creates a string describing program options either given in
760     the command line or their default values.
761     """
762     header = common.prog_name + ' (version ' + common.prog_version_str + \
763     ') running on ' + \
764     time.ctime(time.time())+'\n\n' + \
765     common.prog_name+'. Working options:\n'
766     header = header +\
767     ' scheduler ' + self.scheduler_name + '\n'+\
768     ' job type ' + self.job_type_name + '\n'+\
769     ' working directory ' + common.work_space.topDir()\
770     + '\n'
771     return header
772    
773 nsmirnov 1.3 def createScheduler_(self):
774 nsmirnov 1.1 """
775     Creates a scheduler object instantiated by its name.
776     """
777     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
778     file_name = klass_name
779     try:
780     klass = importName(file_name, klass_name)
781     except KeyError:
782     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
783     raise CrabException(msg)
784     except ImportError, e:
785     msg = 'Cannot create scheduler '+self.scheduler_name
786     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
787     msg += str(e)
788     raise CrabException(msg)
789    
790     common.scheduler = klass()
791     common.scheduler.configure(self.cfg_params)
792     return
793    
794     def run(self):
795     """
796     For each
797     """
798    
799     for act in self.main_actions:
800     if act in self.actions.keys(): self.actions[act].run()
801     pass
802    
803     for act in self.aux_actions:
804     if act in self.actions.keys(): self.actions[act].run()
805     pass
806     return
807    
808     ###########################################################################
809     def processHelpOptions(opts):
810    
811 slacapra 1.11 if len(opts):
812     for opt in opts.keys():
813     if opt in ('-v', '-version', '--version') :
814     print Crab.version()
815     return 1
816     if opt in ('-h','-help','--help') :
817     if opts[opt] : help(opts[opt])
818     else: help()
819     return 1
820     else:
821     usage()
822 nsmirnov 1.1
823     return 0
824    
825     ###########################################################################
826     if __name__ == '__main__':
827    
828     # Parse command-line options and create a dictionary with
829     # key-value pairs.
830    
831     options = parseOptions(sys.argv[1:])
832    
833     # Process "help" options, such as '-help', '-version'
834    
835 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
836 nsmirnov 1.1
837     # Create, initialize, and run a Crab object
838    
839     try:
840 nsmirnov 1.3 crab = Crab(options)
841 nsmirnov 1.1 crab.run()
842     except CrabException, e:
843     print '\n' + common.prog_name + ': ' + str(e) + '\n'
844     if common.logger:
845     common.logger.write('ERROR: '+str(e)+'\n')
846     pass
847     pass
848    
849     pass