ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.20
Committed: Sun Oct 9 17:32:21 2005 UTC (19 years, 6 months ago) by spiga
Content type: text/x-python
Branch: MAIN
Changes since 1.19: +45 -44 lines
Log Message: 
boss_getoutput

File Contents

# User Rev Content
1 slacapra 1.9 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 nsmirnov 1.1 import common
16 spiga 1.13 import Statistic
17 nsmirnov 1.1
18     import sys, os, time, string
19    
20     ###########################################################################
21     class Crab:
22 nsmirnov 1.3 def __init__(self, opts):
23 nsmirnov 1.1
24     # The order of main_actions is important !
25 slacapra 1.9 self.main_actions = [ '-create', '-submit', '-monitor' ]
26 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
27 slacapra 1.9 '-resubmit' , '-cancelAndResubmit', '-check', '-postMortem', '-clean']
28 nsmirnov 1.1
29     # Dictionary of actions, e.g. '-create' -> object of class Creator
30     self.actions = {}
31    
32     # Configuration file
33     self.cfg_fname = None
34     # Dictionary with configuration parameters
35     self.cfg_params = {}
36    
37     # Current working directory
38     self.cwd = os.getcwd()+'/'
39     # Current time in format 'yymmdd_hhmmss'
40     self.current_time = time.strftime('%y%m%d_%H%M%S',
41     time.localtime(time.time()))
42    
43 nsmirnov 1.3 # Session name (?) Do we need this ?
44 nsmirnov 1.1 self.name = '0'
45    
46     # Job type
47     self.job_type_name = None
48    
49     # Continuation flag
50     self.flag_continue = 0
51    
52     # quiet mode, i.e. no output on screen
53     self.flag_quiet = 0
54     # produce more output
55     self.debug_level = 0
56    
57 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
58 fanzago 1.14 self.scheduler_name = ''
59 nsmirnov 1.1
60 nsmirnov 1.3 self.initialize_(opts)
61 nsmirnov 1.1
62     return
63    
64 nsmirnov 1.7 def version():
65 nsmirnov 1.1 return common.prog_version_str
66    
67 nsmirnov 1.7 version = staticmethod(version)
68    
69 nsmirnov 1.3 def initialize_(self, opts):
70 nsmirnov 1.1
71     # Process the '-continue' option first because
72     # in the case of continuation the CRAB configuration
73     # parameters are loaded from already existing Working Space.
74 nsmirnov 1.3 self.processContinueOption_(opts)
75 nsmirnov 1.1
76     # Process ini-file first, then command line options
77     # because they override ini-file settings.
78    
79 nsmirnov 1.3 self.processIniFile_(opts)
80 nsmirnov 1.1
81 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
82 nsmirnov 1.1
83 nsmirnov 1.3 self.processOptions_(opts)
84 nsmirnov 1.1
85     if not self.flag_continue:
86 nsmirnov 1.3 self.createWorkingSpace_()
87 slacapra 1.9 optsToBeSaved={}
88     for it in opts.keys():
89     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
90     pass
91     else:
92     optsToBeSaved[it]=opts[it]
93     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
94 nsmirnov 1.1 pass
95    
96     # At this point all configuration options have been read.
97    
98     args = string.join(sys.argv,' ')
99 slacapra 1.11
100 nsmirnov 1.3 self.updateHistory_(args)
101 slacapra 1.11
102 nsmirnov 1.3 self.createLogger_(args)
103 slacapra 1.11
104 nsmirnov 1.1 common.jobDB = JobDB()
105 slacapra 1.11
106 nsmirnov 1.3 if self.flag_continue:
107 slacapra 1.12 try:
108     common.jobDB.load()
109     common.logger.debug(6, str(common.jobDB))
110     except DBException,e:
111     pass
112 nsmirnov 1.3 pass
113 slacapra 1.11
114 nsmirnov 1.3 self.createScheduler_()
115 slacapra 1.11
116 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
117     common.logger.debug(6, 'Used properties:')
118     keys = self.cfg_params.keys()
119     keys.sort()
120     for k in keys:
121     if self.cfg_params[k]:
122     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
123     pass
124     else:
125     common.logger.debug(6, ' '+k+' : ')
126     pass
127     pass
128     common.logger.debug(6, 'End of used properties.\n')
129     pass
130     self.initializeActions_(opts)
131 nsmirnov 1.1 return
132    
133 nsmirnov 1.3 def processContinueOption_(self,opts):
134 nsmirnov 1.1
135     continue_dir = None
136 nsmirnov 1.4
137     # Look for the '-continue' option.
138    
139 nsmirnov 1.1 for opt in opts.keys():
140     if ( opt in ('-continue','-c') ):
141     self.flag_continue = 1
142     val = opts[opt]
143     if val:
144     if val[0] == '/': continue_dir = val # abs path
145     else: continue_dir = self.cwd + val # rel path
146     pass
147 nsmirnov 1.4 break
148     pass
149    
150     # Look for actions which has sense only with '-continue'
151    
152     if not self.flag_continue:
153     for opt in opts.keys():
154 slacapra 1.6 if ( opt in (self.aux_actions) ):
155 nsmirnov 1.4 self.flag_continue = 1
156     break
157 nsmirnov 1.1 pass
158     pass
159 slacapra 1.6 submit_flag=0
160     create_flag=0
161     for opt in opts.keys():
162     if opt == "-submit": submit_flag=1
163     if opt == "-create": create_flag=1
164     pass
165     if (submit_flag and not create_flag):
166 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
167     raise CrabException(msg)
168 slacapra 1.6 pass
169 nsmirnov 1.1
170     if not self.flag_continue: return
171    
172     if not continue_dir:
173     prefix = common.prog_name + '_' + self.name + '_'
174     continue_dir = findLastWorkDir(prefix)
175     pass
176    
177     if not continue_dir:
178     raise CrabException('Cannot find last working directory.')
179    
180     if not os.path.exists(continue_dir):
181     msg = 'Cannot continue because the working directory <'
182     msg += continue_dir
183     msg += '> does not exist.'
184     raise CrabException(msg)
185    
186     # Instantiate WorkSpace
187     common.work_space = WorkSpace(continue_dir)
188    
189     return
190    
191 nsmirnov 1.3 def processIniFile_(self, opts):
192 nsmirnov 1.1 """
193     Processes a configuration INI-file.
194     """
195    
196     # Extract cfg-file name from the cmd-line options.
197    
198     for opt in opts.keys():
199     if ( opt == '-cfg' ):
200     if self.flag_continue:
201     raise CrabException('-continue and -cfg cannot coexist.')
202     if opts[opt] : self.cfg_fname = opts[opt]
203     else : usage()
204     pass
205    
206     elif ( opt == '-name' ):
207     self.name = opts[opt]
208     pass
209    
210     pass
211    
212     # Set default cfg-fname
213    
214     if self.cfg_fname == None:
215     if self.flag_continue:
216     self.cfg_fname = common.work_space.cfgFileName()
217     else:
218     self.cfg_fname = common.prog_name+'.cfg'
219     pass
220     pass
221    
222     # Load cfg-file
223    
224     if string.lower(self.cfg_fname) != 'none':
225     if os.path.exists(self.cfg_fname):
226     self.cfg_params = loadConfig(self.cfg_fname)
227     pass
228     else:
229     msg = 'cfg-file '+self.cfg_fname+' not found.'
230     raise CrabException(msg)
231     pass
232     pass
233    
234     # process the [CRAB] section
235    
236     lhp = len('CRAB.')
237     for k in self.cfg_params.keys():
238     if len(k) >= lhp and k[:lhp] == 'CRAB.':
239     opt = '-'+k[lhp:]
240     if len(opt) >= 3 and opt[:3] == '-__': continue
241     if opt not in opts.keys():
242     opts[opt] = self.cfg_params[k]
243     pass
244     pass
245     pass
246    
247     return
248    
249 nsmirnov 1.3 def processOptions_(self, opts):
250 nsmirnov 1.1 """
251     Processes the command-line options.
252     """
253    
254     for opt in opts.keys():
255     val = opts[opt]
256    
257 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
258     if opt in self.main_actions:
259     self.cfg_params['CRAB.'+opt[1:]] = val
260     continue
261     if opt in self.aux_actions:
262     self.cfg_params['CRAB.'+opt[1:]] = val
263     continue
264 nsmirnov 1.1
265    
266     elif ( opt == '-cfg' ):
267     pass
268    
269     elif ( opt in ('-continue', '-c') ):
270 nsmirnov 1.4 # Already processed in processContinueOption_()
271 nsmirnov 1.1 pass
272    
273     elif ( opt == '-jobtype' ):
274     if val : self.job_type_name = string.upper(val)
275     else : usage()
276     pass
277    
278     elif ( opt == '-Q' ):
279     self.flag_quiet = 1
280     pass
281    
282     elif ( opt == '-debug' ):
283 slacapra 1.6 if val: self.debug_level = int(val)
284     else: self.debug_level = 1
285 nsmirnov 1.1 pass
286    
287     elif ( opt == '-scheduler' ):
288     if val: self.scheduler_name = val
289     else:
290     print common.prog_name+". No value for '-scheduler'."
291     usage()
292     pass
293     pass
294 fanzago 1.18 ### FEDE
295     # elif ( opt == '-use_boss' ):
296     # self.scheduler_name = 'boss'
297     # pass
298    
299     # elif ( opt == '-copy_data' or opt == '-copy' ):
300     # if ( val == '0' or val == '1' ):
301     # common.flag_copy_data = int(val)
302     # else:
303     # print common.prog_name+'. Bad flag for -copy_data option:',\
304     # val,'Possible values are 0(=No) or 1(=Yes)'
305     # usage()
306     # pass
307    
308     # elif ( opt == '-register_data' or opt == '-register' ):
309     # if ( val == '0' or val == '1' ):
310     # common.flag_register_data = int(val)
311     # else:
312     # print common.prog_name+'. Bad flag for -register_data option:',\
313     # val,'Possible values are 0(=No) or 1(=Yes#)'
314     # usage()
315     # pass
316    
317     # elif ( opt == '-return_data' or opt == '-return' ):
318     # if ( val == '0' or val == '1' ):
319     # common.flag_return_data = int(val)
320     # else:
321     # print common.prog_name+'. Bad flag for -return_data option:',\
322     # val,'Possible values are 0(=No) or 1(=Yes)'
323     # usage()
324     # pass
325    
326     elif ( opt in ('-use_boss', '-useboss') ):
327     if ( val == '1' ):
328     self.scheduler_name = 'boss'
329     pass
330     elif ( val == '0' ):
331     pass
332     else:
333     print common.prog_name+'. Bad flag for -use_boss option:',\
334     val,'Possible values are 0(=No) or 1(=Yes)'
335     usage()
336     pass
337 fanzago 1.14 pass
338 fanzago 1.18 ######
339 fanzago 1.14
340 nsmirnov 1.3 elif string.find(opt,'.') == -1:
341     print common.prog_name+'. Unrecognized option '+opt
342     usage()
343     pass
344 nsmirnov 1.1
345 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
346     if string.find(opt,'.') == -1 :
347     self.cfg_params['CRAB.'+opt[1:]] = val
348 nsmirnov 1.1 pass
349 nsmirnov 1.3 else:
350 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
351     self.cfg_params[opt[1:]] = val
352     pass
353     pass
354     return
355    
356 slacapra 1.8 def parseRange_(self, aRange):
357 nsmirnov 1.4 """
358 slacapra 1.8 Takes as the input a string with a range defined in any of the following
359     way, including combination, and return a tuple with the ints defined
360     according to following table. A consistency check is done.
361     NB: the first job is "1", not "0".
362     'all' -> [1,2,..., NJobs]
363     '' -> [1,2,..., NJobs]
364     'n1' -> [n1]
365     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
366     'n1,n2' -> [n1, n2]
367     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
368     """
369     result = []
370    
371 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
372     if aRange=='all' or aRange==None or aRange=='':
373 slacapra 1.8 result=range(0,common.jobDB.nJobs())
374     return result
375 slacapra 1.9 elif aRange=='0':
376     return result
377 slacapra 1.8
378     subRanges = string.split(aRange, ',')
379     for subRange in subRanges:
380     result = result+self.parseSimpleRange_(subRange)
381    
382     if self.checkUniqueness_(result):
383     return result
384     else:
385     print "Error ", result
386     return []
387    
388     def checkUniqueness_(self, list):
389     """
390 slacapra 1.9 check if a list contains only unique elements
391 slacapra 1.8 """
392    
393     uniqueList = []
394     # use a list comprehension statement (takes a while to understand)
395    
396     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
397    
398     return (len(list)==len(uniqueList))
399    
400     def parseSimpleRange_(self, aRange):
401     """
402     Takes as the input a string with two integers separated by
403     the minus sign and returns the tuple with these numbers:
404     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
405     'n1' -> [n1]
406     """
407     (start, end) = (None, None)
408    
409     result = []
410     minus = string.find(aRange, '-')
411     if ( minus < 0 ):
412     if isInt(aRange) and int(aRange)>0:
413     result.append(int(aRange)-1)
414     else:
415 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
416 nsmirnov 1.4 pass
417     else:
418 slacapra 1.8 (start, end) = string.split(aRange, '-')
419     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
420     result=range(int(start)-1, int(end))
421     else:
422     print "ERROR ", start, end
423    
424     return result
425 nsmirnov 1.4
426 nsmirnov 1.3 def initializeActions_(self, opts):
427 nsmirnov 1.1 """
428     For each user action instantiate a corresponding
429     object and put it in the action dictionary.
430     """
431     for opt in opts.keys():
432 spiga 1.16 if ( opt == '-use_boss'):
433 spiga 1.15 self.flag_useboss = 1
434     else:
435     self.flag_useboss = 0
436     pass
437    
438     for opt in opts.keys():
439    
440 nsmirnov 1.1 val = opts[opt]
441 spiga 1.15
442    
443     if ( opt == '-create' ):
444 nsmirnov 1.1 if val:
445     if ( isInt(val) ):
446     ncjobs = int(val)
447     elif ( val == 'all'):
448     ncjobs = val
449     else:
450 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
451     msg += ' Must be an integer or "all"'
452 slacapra 1.8 msg += ' Generic range is not allowed"'
453 nsmirnov 1.5 raise CrabException(msg)
454 nsmirnov 1.1 pass
455     else: ncjobs = 'all'
456    
457     if ncjobs != 0:
458 nsmirnov 1.3 # Instantiate Creator object
459 nsmirnov 1.1 creator = Creator(self.job_type_name,
460     self.cfg_params,
461     ncjobs)
462     self.actions[opt] = creator
463 nsmirnov 1.3
464     # Initialize the JobDB object if needed
465 nsmirnov 1.1 if not self.flag_continue:
466     common.jobDB.create(creator.nJobs())
467     pass
468 nsmirnov 1.3
469     # Create and initialize JobList
470    
471     common.job_list = JobList(common.jobDB.nJobs(),
472     creator.jobType())
473    
474     common.job_list.setScriptNames(self.job_type_name+'.sh')
475     common.job_list.setJDLNames(self.job_type_name+'.jdl')
476 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
477 slacapra 1.9
478     creator.writeJobsSpecsToDB()
479 nsmirnov 1.1 pass
480 nsmirnov 1.3 pass
481 nsmirnov 1.1
482     elif ( opt == '-submit' ):
483 nsmirnov 1.7 nj_list = self.parseRange_(val)
484 nsmirnov 1.5
485     if len(nj_list) != 0:
486 nsmirnov 1.3 # Instantiate Submitter object
487 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
488 nsmirnov 1.3
489     # Create and initialize JobList
490    
491     if len(common.job_list) == 0 :
492     common.job_list = JobList(common.jobDB.nJobs(),
493     None)
494     common.job_list.setJDLNames(self.job_type_name+'.jdl')
495     pass
496 nsmirnov 1.1 pass
497 nsmirnov 1.3 pass
498 nsmirnov 1.1
499 nsmirnov 1.4 elif ( opt == '-list' ):
500 slacapra 1.8 jobs = self.parseRange_(val)
501    
502     common.jobDB.dump(jobs)
503 nsmirnov 1.4 pass
504    
505     elif ( opt == '-status' ):
506 slacapra 1.8 jobs = self.parseRange_(val)
507    
508 slacapra 1.9 if len(jobs) != 0:
509 spiga 1.16 if ( self.flag_useboss == 1 ):
510     self.actions[opt] = StatusBoss(self.cfg_params, jobs)
511 spiga 1.15 else:
512     # Instantiate Submitter object
513     self.actions[opt] = Status(self.cfg_params, jobs)
514     pass
515 nsmirnov 1.1 pass
516     pass
517 nsmirnov 1.4 elif ( opt == '-kill' ):
518 slacapra 1.9 if val:
519     jobs = self.parseRange_(val)
520 slacapra 1.8
521 slacapra 1.9 for nj in jobs:
522     st = common.jobDB.status(nj)
523     if st == 'S':
524     jid = common.jobDB.jobId(nj)
525     common.logger.message("Killing job # "+`(nj+1)`)
526     common.scheduler.cancel(jid)
527     common.jobDB.setStatus(nj, 'K')
528     pass
529 nsmirnov 1.4 pass
530 slacapra 1.9
531     common.jobDB.save()
532 nsmirnov 1.1 pass
533 slacapra 1.9 else:
534     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
535 nsmirnov 1.1
536 slacapra 1.8 elif ( opt == '-getoutput' ):
537    
538 spiga 1.20 if ( self.flag_useboss == 1 ):
539     if val=='all' or val==None or val=='':
540     jobs = listBoss()
541     else:
542     jobs = self.parseRange_(val)
543    
544     common.scheduler.getOutput(jobs)
545     else:
546     jobs = self.parseRange_(val)
547     fileCODE1 = open(common.work_space.logDir()+"/.code","r")
548     array = fileCODE1.read().split('::')
549     self.ID1 = array[0]
550     self.NJC = array[1]
551     self.dataset = array[2]
552     self.owner = array[3]
553     fileCODE1.close()
554 spiga 1.13
555    
556 spiga 1.20 jobs_done = []
557     for nj in jobs:
558     st = common.jobDB.status(nj)
559     if st == 'D':
560 slacapra 1.12 jobs_done.append(nj)
561 spiga 1.20 pass
562     elif st == 'S':
563     jid = common.jobDB.jobId(nj)
564     print "jid", jid
565     currStatus = common.scheduler.queryStatus(jid)
566     if currStatus=="Done":
567     jobs_done.append(nj)
568     else:
569     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
570     common.logger.message(msg)
571     pass
572 slacapra 1.9 else:
573 spiga 1.20 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
574     pass
575 slacapra 1.12 pass
576    
577 spiga 1.20 for nj in jobs_done:
578 spiga 1.15 jid = common.jobDB.jobId(nj)
579     dir = common.scheduler.getOutput(jid)
580     common.jobDB.setStatus(nj, 'Y')
581 slacapra 1.12
582     # Rename the directory with results to smth readable
583     new_dir = common.work_space.resDir()
584 spiga 1.20 try:
585     files = os.listdir(dir)
586     for file in files:
587     os.rename(dir+'/'+file, new_dir+'/'+file)
588     os.rmdir(dir)
589     except OSError, e:
590     msg = 'rename files from '+dir+' to '+new_dir+' error: '
591     msg += str(e)
592     common.logger.message(msg)
593     # ignore error
594 slacapra 1.12 pass
595 spiga 1.20 pass
596 spiga 1.13 destination = common.scheduler.queryDest(jid).split(":")[0]
597     ID3 = jid.split("/")[3]
598     broker = jid.split("/")[2].split(":")[0]
599     resFlag = 0
600     exCode = common.scheduler.getExitStatus(jid)
601     Statistic.notify('retrieved',resFlag,exCode,self.dataset,self.owner,destination,broker,ID3,self.ID1,self.NJC)
602 slacapra 1.12
603     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
604     common.logger.message(msg)
605 nsmirnov 1.4 pass
606    
607     common.jobDB.save()
608     pass
609    
610     elif ( opt == '-resubmit' ):
611 slacapra 1.11 if val:
612     jobs = self.parseRange_(val)
613 slacapra 1.8
614 slacapra 1.11 # create a list of jobs to be resubmitted.
615 slacapra 1.8
616 slacapra 1.11 nj_list = []
617     for nj in jobs:
618     st = common.jobDB.status(nj)
619 slacapra 1.8
620 slacapra 1.11 if st in ['K','A']:
621     nj_list.append(nj)
622     elif st == 'Y':
623     # here I would like to move the old output to a new place
624     # I would need the name of the output files.
625     # these infos are in the jobType class, which is not accessible from here!
626     outSandbox = common.jobDB.outputSandbox(nj)
627    
628     resDir = common.work_space.resDir()
629     resDirSave = resDir + self.current_time
630     os.mkdir(resDirSave)
631    
632     for file in outSandbox:
633     if os.path.exists(resDir+'/'+file):
634     os.rename(resDir+'/'+file, resDirSave+'/'+file)
635     common.logger.message('Output file '+file+' moved to '+resDirSave)
636     pass
637     nj_list.append(nj)
638 spiga 1.13 st = common.jobDB.setStatus(nj,'RC')
639 slacapra 1.11 elif st == 'D':
640     ## Done but not yet retrieved
641     ## retrieve job, then go to previous ('Y') case
642     ## TODO
643     pass
644     else:
645     common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
646 slacapra 1.9 pass
647 slacapra 1.8
648 slacapra 1.11 if len(nj_list) != 0:
649     # Instantiate Submitter object
650     self.actions[opt] = Submitter(self.cfg_params, nj_list)
651    
652     # Create and initialize JobList
653    
654     if len(common.job_list) == 0 :
655     common.job_list = JobList(common.jobDB.nJobs(),
656     None)
657     common.job_list.setJDLNames(self.job_type_name+'.jdl')
658     pass
659 slacapra 1.8 pass
660     pass
661 slacapra 1.11 else:
662     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
663     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
664     pass
665 slacapra 1.8 pass
666    
667     elif ( opt == '-cancelAndResubmit' ):
668     jobs = self.parseRange_(val)
669 nsmirnov 1.5
670 nsmirnov 1.7 # Cancel submitted jobs.
671 nsmirnov 1.5
672 slacapra 1.8 nj_list = []
673     for nj in jobs:
674 nsmirnov 1.5 st = common.jobDB.status(nj)
675     if st == 'S':
676     jid = common.jobDB.jobId(nj)
677     common.scheduler.cancel(jid)
678     st = 'K'
679     common.jobDB.setStatus(nj, st)
680     pass
681 nsmirnov 1.7 pass
682 nsmirnov 1.5
683 slacapra 1.8 if st != 'X': nj_list.append(nj)
684     pass
685 nsmirnov 1.5
686     if len(nj_list) != 0:
687     # Instantiate Submitter object
688     self.actions[opt] = Submitter(self.cfg_params, nj_list)
689    
690     # Create and initialize JobList
691    
692     if len(common.job_list) == 0 :
693     common.job_list = JobList(common.jobDB.nJobs(),
694     None)
695     common.job_list.setJDLNames(self.job_type_name+'.jdl')
696     pass
697     pass
698 nsmirnov 1.4 pass
699    
700 slacapra 1.8 elif ( opt == '-check' ):
701     jobs = self.parseRange_(val)
702     nj_list = []
703     for nj in jobs:
704     st = common.jobDB.status(nj)
705 slacapra 1.12 if st == 'C': nj_list.append(nj)
706 slacapra 1.8 pass
707    
708     if len(nj_list) != 0:
709     # Instantiate Submitter object
710     self.actions[opt] = Checker(self.cfg_params, nj_list)
711    
712     # Create and initialize JobList
713    
714     if len(common.job_list) == 0 :
715     common.job_list = JobList(common.jobDB.nJobs(), None)
716     common.job_list.setJDLNames(self.job_type_name+'.jdl')
717     pass
718     pass
719    
720 slacapra 1.9 elif ( opt == '-postMortem' ):
721     jobs = self.parseRange_(val)
722     nj_list = []
723     for nj in jobs:
724     st = common.jobDB.status(nj)
725 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
726 slacapra 1.9 pass
727    
728     if len(nj_list) != 0:
729     # Instantiate Submitter object
730     self.actions[opt] = PostMortem(self.cfg_params, nj_list)
731    
732     # Create and initialize JobList
733    
734     if len(common.job_list) == 0 :
735     common.job_list = JobList(common.jobDB.nJobs(), None)
736     common.job_list.setJDLNames(self.job_type_name+'.jdl')
737     pass
738     pass
739    
740     elif ( opt == '-clean' ):
741     if val != None:
742     raise CrabException("No range allowed for '-clean'")
743    
744     submittedJobs=0
745 slacapra 1.11 doneJobs=0
746 slacapra 1.12 try:
747     for nj in range(0,common.jobDB.nJobs()):
748     st = common.jobDB.status(nj)
749     if st == 'S':
750     submittedJobs = submittedJobs+1
751     if st == 'D':
752     doneJobs = doneJobs+1
753     pass
754     pass
755     except DBException:
756     common.logger.debug(5,'DB not found, so delete all')
757 nsmirnov 1.1 pass
758 slacapra 1.9
759 slacapra 1.11 if submittedJobs or doneJobs:
760     msg = "There are still "
761     if submittedJobs:
762     msg= msg+str(submittedJobs)+" jobs submitted. Kill them '-kill' before '-clean'"
763     if (submittedJobs and doneJobs):
764     msg = msg + "and \nalso"
765     if doneJobs:
766     msg= msg+str(doneJobs)+" jobs Done. Get their outputs '-getoutput' before '-clean'"
767     raise CrabException(msg)
768 slacapra 1.9
769     msg = 'directory '+common.work_space.topDir()+' removed'
770     common.work_space.delete()
771     common.logger.message(msg)
772    
773 nsmirnov 1.1 pass
774 slacapra 1.9
775 nsmirnov 1.1 pass
776     return
777    
778 nsmirnov 1.3 def createWorkingSpace_(self):
779 slacapra 1.9 new_dir = ''
780    
781     try:
782     new_dir = self.cfg_params['USER.ui_working_dir']
783     except KeyError:
784     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
785     new_dir = self.cwd + new_dir
786     pass
787    
788 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
789     common.work_space.create()
790     return
791    
792 nsmirnov 1.3 def loadConfiguration_(self, opts):
793 nsmirnov 1.1
794     save_opts = common.work_space.loadSavedOptions()
795    
796     # Override saved options with new command-line options
797    
798     for k in opts.keys():
799     save_opts[k] = opts[k]
800     pass
801    
802     # Return updated options
803     return save_opts
804    
805 nsmirnov 1.3 def createLogger_(self, args):
806 nsmirnov 1.1
807     log = Logger()
808     log.quiet(self.flag_quiet)
809     log.setDebugLevel(self.debug_level)
810     log.write(args+'\n')
811 nsmirnov 1.3 log.message(self.headerString_())
812 nsmirnov 1.1 log.flush()
813     common.logger = log
814     return
815    
816 nsmirnov 1.3 def updateHistory_(self, args):
817 nsmirnov 1.1 history_fname = common.prog_name+'.history'
818     history_file = open(history_fname, 'a')
819     history_file.write(self.current_time+': '+args+'\n')
820     history_file.close()
821     return
822    
823 nsmirnov 1.3 def headerString_(self):
824 nsmirnov 1.1 """
825     Creates a string describing program options either given in
826     the command line or their default values.
827     """
828     header = common.prog_name + ' (version ' + common.prog_version_str + \
829     ') running on ' + \
830     time.ctime(time.time())+'\n\n' + \
831     common.prog_name+'. Working options:\n'
832     header = header +\
833     ' scheduler ' + self.scheduler_name + '\n'+\
834     ' job type ' + self.job_type_name + '\n'+\
835     ' working directory ' + common.work_space.topDir()\
836     + '\n'
837     return header
838    
839 nsmirnov 1.3 def createScheduler_(self):
840 nsmirnov 1.1 """
841     Creates a scheduler object instantiated by its name.
842     """
843     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
844     file_name = klass_name
845     try:
846     klass = importName(file_name, klass_name)
847     except KeyError:
848     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
849     raise CrabException(msg)
850     except ImportError, e:
851     msg = 'Cannot create scheduler '+self.scheduler_name
852     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
853     msg += str(e)
854     raise CrabException(msg)
855    
856     common.scheduler = klass()
857     common.scheduler.configure(self.cfg_params)
858     return
859    
860     def run(self):
861     """
862     For each
863     """
864    
865     for act in self.main_actions:
866     if act in self.actions.keys(): self.actions[act].run()
867     pass
868    
869     for act in self.aux_actions:
870     if act in self.actions.keys(): self.actions[act].run()
871     pass
872     return
873    
874     ###########################################################################
875     def processHelpOptions(opts):
876    
877 slacapra 1.11 if len(opts):
878     for opt in opts.keys():
879     if opt in ('-v', '-version', '--version') :
880     print Crab.version()
881     return 1
882     if opt in ('-h','-help','--help') :
883     if opts[opt] : help(opts[opt])
884     else: help()
885     return 1
886     else:
887     usage()
888 nsmirnov 1.1
889     return 0
890    
891     ###########################################################################
892     if __name__ == '__main__':
893    
894 fanzago 1.18
895 fanzago 1.19 # # Initial settings for Python modules. Avoid appending manually lib paths.
896     # try:
897     # path=os.environ['EDG_WL_LOCATION']
898     # except:
899     # print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
900     # sys.exit(1)
901     #
902     # libPath=os.path.join(path, "lib")
903     # sys.path.append(libPath)
904     # libPath=os.path.join(path, "lib", "python")
905     # sys.path.append(libPath)
906 fanzago 1.18
907    
908 nsmirnov 1.1 # Parse command-line options and create a dictionary with
909     # key-value pairs.
910    
911     options = parseOptions(sys.argv[1:])
912    
913     # Process "help" options, such as '-help', '-version'
914    
915 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
916 nsmirnov 1.1
917     # Create, initialize, and run a Crab object
918    
919     try:
920 nsmirnov 1.3 crab = Crab(options)
921 nsmirnov 1.1 crab.run()
922     except CrabException, e:
923     print '\n' + common.prog_name + ': ' + str(e) + '\n'
924     if common.logger:
925     common.logger.write('ERROR: '+str(e)+'\n')
926     pass
927     pass
928    
929     pass