ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.18
Committed: Tue Oct 4 18:19:33 2005 UTC (19 years, 6 months ago) by fanzago
Content type: text/x-python
Branch: MAIN
Changes since 1.17: +58 -3 lines
Log Message:
changed for copy and register data

File Contents

# User Rev Content
1 slacapra 1.9 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 nsmirnov 1.1 import common
16 spiga 1.13 import Statistic
17 nsmirnov 1.1
18     import sys, os, time, string
19    
20     ###########################################################################
21     class Crab:
22 nsmirnov 1.3 def __init__(self, opts):
23 nsmirnov 1.1
24     # The order of main_actions is important !
25 slacapra 1.9 self.main_actions = [ '-create', '-submit', '-monitor' ]
26 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
27 slacapra 1.9 '-resubmit' , '-cancelAndResubmit', '-check', '-postMortem', '-clean']
28 nsmirnov 1.1
29     # Dictionary of actions, e.g. '-create' -> object of class Creator
30     self.actions = {}
31    
32     # Configuration file
33     self.cfg_fname = None
34     # Dictionary with configuration parameters
35     self.cfg_params = {}
36    
37     # Current working directory
38     self.cwd = os.getcwd()+'/'
39     # Current time in format 'yymmdd_hhmmss'
40     self.current_time = time.strftime('%y%m%d_%H%M%S',
41     time.localtime(time.time()))
42    
43 nsmirnov 1.3 # Session name (?) Do we need this ?
44 nsmirnov 1.1 self.name = '0'
45    
46     # Job type
47     self.job_type_name = None
48    
49     # Continuation flag
50     self.flag_continue = 0
51    
52     # quiet mode, i.e. no output on screen
53     self.flag_quiet = 0
54     # produce more output
55     self.debug_level = 0
56    
57 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
58 fanzago 1.14 self.scheduler_name = ''
59 nsmirnov 1.1
60 nsmirnov 1.3 self.initialize_(opts)
61 nsmirnov 1.1
62     return
63    
64 nsmirnov 1.7 def version():
65 nsmirnov 1.1 return common.prog_version_str
66    
67 nsmirnov 1.7 version = staticmethod(version)
68    
69 nsmirnov 1.3 def initialize_(self, opts):
70 nsmirnov 1.1
71     # Process the '-continue' option first because
72     # in the case of continuation the CRAB configuration
73     # parameters are loaded from already existing Working Space.
74 nsmirnov 1.3 self.processContinueOption_(opts)
75 nsmirnov 1.1
76     # Process ini-file first, then command line options
77     # because they override ini-file settings.
78    
79 nsmirnov 1.3 self.processIniFile_(opts)
80 nsmirnov 1.1
81 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
82 nsmirnov 1.1
83 nsmirnov 1.3 self.processOptions_(opts)
84 nsmirnov 1.1
85     if not self.flag_continue:
86 nsmirnov 1.3 self.createWorkingSpace_()
87 slacapra 1.9 optsToBeSaved={}
88     for it in opts.keys():
89     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
90     pass
91     else:
92     optsToBeSaved[it]=opts[it]
93     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
94 nsmirnov 1.1 pass
95    
96     # At this point all configuration options have been read.
97    
98     args = string.join(sys.argv,' ')
99 slacapra 1.11
100 nsmirnov 1.3 self.updateHistory_(args)
101 slacapra 1.11
102 nsmirnov 1.3 self.createLogger_(args)
103 slacapra 1.11
104 nsmirnov 1.1 common.jobDB = JobDB()
105 slacapra 1.11
106 nsmirnov 1.3 if self.flag_continue:
107 slacapra 1.12 try:
108     common.jobDB.load()
109     common.logger.debug(6, str(common.jobDB))
110     except DBException,e:
111     pass
112 nsmirnov 1.3 pass
113 slacapra 1.11
114 nsmirnov 1.3 self.createScheduler_()
115 slacapra 1.11
116 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
117     common.logger.debug(6, 'Used properties:')
118     keys = self.cfg_params.keys()
119     keys.sort()
120     for k in keys:
121     if self.cfg_params[k]:
122     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
123     pass
124     else:
125     common.logger.debug(6, ' '+k+' : ')
126     pass
127     pass
128     common.logger.debug(6, 'End of used properties.\n')
129     pass
130     self.initializeActions_(opts)
131 nsmirnov 1.1 return
132    
133 nsmirnov 1.3 def processContinueOption_(self,opts):
134 nsmirnov 1.1
135     continue_dir = None
136 nsmirnov 1.4
137     # Look for the '-continue' option.
138    
139 nsmirnov 1.1 for opt in opts.keys():
140     if ( opt in ('-continue','-c') ):
141     self.flag_continue = 1
142     val = opts[opt]
143     if val:
144     if val[0] == '/': continue_dir = val # abs path
145     else: continue_dir = self.cwd + val # rel path
146     pass
147 nsmirnov 1.4 break
148     pass
149    
150     # Look for actions which has sense only with '-continue'
151    
152     if not self.flag_continue:
153     for opt in opts.keys():
154 slacapra 1.6 if ( opt in (self.aux_actions) ):
155 nsmirnov 1.4 self.flag_continue = 1
156     break
157 nsmirnov 1.1 pass
158     pass
159 slacapra 1.6 submit_flag=0
160     create_flag=0
161     for opt in opts.keys():
162     if opt == "-submit": submit_flag=1
163     if opt == "-create": create_flag=1
164     pass
165     if (submit_flag and not create_flag):
166 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
167     raise CrabException(msg)
168 slacapra 1.6 pass
169 nsmirnov 1.1
170     if not self.flag_continue: return
171    
172     if not continue_dir:
173     prefix = common.prog_name + '_' + self.name + '_'
174     continue_dir = findLastWorkDir(prefix)
175     pass
176    
177     if not continue_dir:
178     raise CrabException('Cannot find last working directory.')
179    
180     if not os.path.exists(continue_dir):
181     msg = 'Cannot continue because the working directory <'
182     msg += continue_dir
183     msg += '> does not exist.'
184     raise CrabException(msg)
185    
186     # Instantiate WorkSpace
187     common.work_space = WorkSpace(continue_dir)
188    
189     return
190    
191 nsmirnov 1.3 def processIniFile_(self, opts):
192 nsmirnov 1.1 """
193     Processes a configuration INI-file.
194     """
195    
196     # Extract cfg-file name from the cmd-line options.
197    
198     for opt in opts.keys():
199     if ( opt == '-cfg' ):
200     if self.flag_continue:
201     raise CrabException('-continue and -cfg cannot coexist.')
202     if opts[opt] : self.cfg_fname = opts[opt]
203     else : usage()
204     pass
205    
206     elif ( opt == '-name' ):
207     self.name = opts[opt]
208     pass
209    
210     pass
211    
212     # Set default cfg-fname
213    
214     if self.cfg_fname == None:
215     if self.flag_continue:
216     self.cfg_fname = common.work_space.cfgFileName()
217     else:
218     self.cfg_fname = common.prog_name+'.cfg'
219     pass
220     pass
221    
222     # Load cfg-file
223    
224     if string.lower(self.cfg_fname) != 'none':
225     if os.path.exists(self.cfg_fname):
226     self.cfg_params = loadConfig(self.cfg_fname)
227     pass
228     else:
229     msg = 'cfg-file '+self.cfg_fname+' not found.'
230     raise CrabException(msg)
231     pass
232     pass
233    
234     # process the [CRAB] section
235    
236     lhp = len('CRAB.')
237     for k in self.cfg_params.keys():
238     if len(k) >= lhp and k[:lhp] == 'CRAB.':
239     opt = '-'+k[lhp:]
240     if len(opt) >= 3 and opt[:3] == '-__': continue
241     if opt not in opts.keys():
242     opts[opt] = self.cfg_params[k]
243     pass
244     pass
245     pass
246    
247     return
248    
249 nsmirnov 1.3 def processOptions_(self, opts):
250 nsmirnov 1.1 """
251     Processes the command-line options.
252     """
253    
254     for opt in opts.keys():
255     val = opts[opt]
256    
257 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
258     if opt in self.main_actions:
259     self.cfg_params['CRAB.'+opt[1:]] = val
260     continue
261     if opt in self.aux_actions:
262     self.cfg_params['CRAB.'+opt[1:]] = val
263     continue
264 nsmirnov 1.1
265    
266     elif ( opt == '-cfg' ):
267     pass
268    
269     elif ( opt in ('-continue', '-c') ):
270 nsmirnov 1.4 # Already processed in processContinueOption_()
271 nsmirnov 1.1 pass
272    
273     elif ( opt == '-jobtype' ):
274     if val : self.job_type_name = string.upper(val)
275     else : usage()
276     pass
277    
278     elif ( opt == '-Q' ):
279     self.flag_quiet = 1
280     pass
281    
282     elif ( opt == '-debug' ):
283 slacapra 1.6 if val: self.debug_level = int(val)
284     else: self.debug_level = 1
285 nsmirnov 1.1 pass
286    
287     elif ( opt == '-scheduler' ):
288     if val: self.scheduler_name = val
289     else:
290     print common.prog_name+". No value for '-scheduler'."
291     usage()
292     pass
293     pass
294 fanzago 1.18 ### FEDE
295     # elif ( opt == '-use_boss' ):
296     # self.scheduler_name = 'boss'
297     # pass
298    
299     # elif ( opt == '-copy_data' or opt == '-copy' ):
300     # if ( val == '0' or val == '1' ):
301     # common.flag_copy_data = int(val)
302     # else:
303     # print common.prog_name+'. Bad flag for -copy_data option:',\
304     # val,'Possible values are 0(=No) or 1(=Yes)'
305     # usage()
306     # pass
307    
308     # elif ( opt == '-register_data' or opt == '-register' ):
309     # if ( val == '0' or val == '1' ):
310     # common.flag_register_data = int(val)
311     # else:
312     # print common.prog_name+'. Bad flag for -register_data option:',\
313     # val,'Possible values are 0(=No) or 1(=Yes#)'
314     # usage()
315     # pass
316    
317     # elif ( opt == '-return_data' or opt == '-return' ):
318     # if ( val == '0' or val == '1' ):
319     # common.flag_return_data = int(val)
320     # else:
321     # print common.prog_name+'. Bad flag for -return_data option:',\
322     # val,'Possible values are 0(=No) or 1(=Yes)'
323     # usage()
324     # pass
325    
326     elif ( opt in ('-use_boss', '-useboss') ):
327     if ( val == '1' ):
328     self.scheduler_name = 'boss'
329     pass
330     elif ( val == '0' ):
331     pass
332     else:
333     print common.prog_name+'. Bad flag for -use_boss option:',\
334     val,'Possible values are 0(=No) or 1(=Yes)'
335     usage()
336     pass
337 fanzago 1.14 pass
338 fanzago 1.18 ######
339 fanzago 1.14
340 nsmirnov 1.3 elif string.find(opt,'.') == -1:
341     print common.prog_name+'. Unrecognized option '+opt
342     usage()
343     pass
344 nsmirnov 1.1
345 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
346     if string.find(opt,'.') == -1 :
347     self.cfg_params['CRAB.'+opt[1:]] = val
348 nsmirnov 1.1 pass
349 nsmirnov 1.3 else:
350 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
351     self.cfg_params[opt[1:]] = val
352     pass
353     pass
354     return
355    
356 slacapra 1.8 def parseRange_(self, aRange):
357 nsmirnov 1.4 """
358 slacapra 1.8 Takes as the input a string with a range defined in any of the following
359     way, including combination, and return a tuple with the ints defined
360     according to following table. A consistency check is done.
361     NB: the first job is "1", not "0".
362     'all' -> [1,2,..., NJobs]
363     '' -> [1,2,..., NJobs]
364     'n1' -> [n1]
365     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
366     'n1,n2' -> [n1, n2]
367     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
368     """
369     result = []
370    
371 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
372     if aRange=='all' or aRange==None or aRange=='':
373 slacapra 1.8 result=range(0,common.jobDB.nJobs())
374     return result
375 slacapra 1.9 elif aRange=='0':
376     return result
377 slacapra 1.8
378     subRanges = string.split(aRange, ',')
379     for subRange in subRanges:
380     result = result+self.parseSimpleRange_(subRange)
381    
382     if self.checkUniqueness_(result):
383     return result
384     else:
385     print "Error ", result
386     return []
387    
388     def checkUniqueness_(self, list):
389     """
390 slacapra 1.9 check if a list contains only unique elements
391 slacapra 1.8 """
392    
393     uniqueList = []
394     # use a list comprehension statement (takes a while to understand)
395    
396     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
397    
398     return (len(list)==len(uniqueList))
399    
400     def parseSimpleRange_(self, aRange):
401     """
402     Takes as the input a string with two integers separated by
403     the minus sign and returns the tuple with these numbers:
404     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
405     'n1' -> [n1]
406     """
407     (start, end) = (None, None)
408    
409     result = []
410     minus = string.find(aRange, '-')
411     if ( minus < 0 ):
412     if isInt(aRange) and int(aRange)>0:
413     result.append(int(aRange)-1)
414     else:
415 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
416 nsmirnov 1.4 pass
417     else:
418 slacapra 1.8 (start, end) = string.split(aRange, '-')
419     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
420     result=range(int(start)-1, int(end))
421     else:
422     print "ERROR ", start, end
423    
424     return result
425 nsmirnov 1.4
426 nsmirnov 1.3 def initializeActions_(self, opts):
427 nsmirnov 1.1 """
428     For each user action instantiate a corresponding
429     object and put it in the action dictionary.
430     """
431     for opt in opts.keys():
432 spiga 1.16 if ( opt == '-use_boss'):
433 spiga 1.15 self.flag_useboss = 1
434     else:
435     self.flag_useboss = 0
436     pass
437    
438     for opt in opts.keys():
439    
440 nsmirnov 1.1 val = opts[opt]
441 spiga 1.15
442    
443     if ( opt == '-create' ):
444 nsmirnov 1.1 if val:
445     if ( isInt(val) ):
446     ncjobs = int(val)
447     elif ( val == 'all'):
448     ncjobs = val
449     else:
450 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
451     msg += ' Must be an integer or "all"'
452 slacapra 1.8 msg += ' Generic range is not allowed"'
453 nsmirnov 1.5 raise CrabException(msg)
454 nsmirnov 1.1 pass
455     else: ncjobs = 'all'
456    
457     if ncjobs != 0:
458 nsmirnov 1.3 # Instantiate Creator object
459 nsmirnov 1.1 creator = Creator(self.job_type_name,
460     self.cfg_params,
461     ncjobs)
462     self.actions[opt] = creator
463 nsmirnov 1.3
464     # Initialize the JobDB object if needed
465 nsmirnov 1.1 if not self.flag_continue:
466     common.jobDB.create(creator.nJobs())
467     pass
468 nsmirnov 1.3
469     # Create and initialize JobList
470    
471     common.job_list = JobList(common.jobDB.nJobs(),
472     creator.jobType())
473    
474     common.job_list.setScriptNames(self.job_type_name+'.sh')
475     common.job_list.setJDLNames(self.job_type_name+'.jdl')
476 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
477 slacapra 1.9
478     creator.writeJobsSpecsToDB()
479 nsmirnov 1.1 pass
480 nsmirnov 1.3 pass
481 nsmirnov 1.1
482     elif ( opt == '-submit' ):
483 nsmirnov 1.7 nj_list = self.parseRange_(val)
484 nsmirnov 1.5
485     if len(nj_list) != 0:
486 nsmirnov 1.3 # Instantiate Submitter object
487 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
488 nsmirnov 1.3
489     # Create and initialize JobList
490    
491     if len(common.job_list) == 0 :
492     common.job_list = JobList(common.jobDB.nJobs(),
493     None)
494     common.job_list.setJDLNames(self.job_type_name+'.jdl')
495     pass
496 nsmirnov 1.1 pass
497 nsmirnov 1.3 pass
498 nsmirnov 1.1
499 nsmirnov 1.4 elif ( opt == '-list' ):
500 slacapra 1.8 jobs = self.parseRange_(val)
501    
502     common.jobDB.dump(jobs)
503 nsmirnov 1.4 pass
504    
505     elif ( opt == '-status' ):
506 slacapra 1.8 jobs = self.parseRange_(val)
507    
508 slacapra 1.9 if len(jobs) != 0:
509 spiga 1.16 if ( self.flag_useboss == 1 ):
510     self.actions[opt] = StatusBoss(self.cfg_params, jobs)
511 spiga 1.15 else:
512     # Instantiate Submitter object
513     self.actions[opt] = Status(self.cfg_params, jobs)
514     pass
515 nsmirnov 1.1 pass
516     pass
517 nsmirnov 1.4 elif ( opt == '-kill' ):
518 slacapra 1.9 if val:
519     jobs = self.parseRange_(val)
520 slacapra 1.8
521 slacapra 1.9 for nj in jobs:
522     st = common.jobDB.status(nj)
523     if st == 'S':
524     jid = common.jobDB.jobId(nj)
525     common.logger.message("Killing job # "+`(nj+1)`)
526     common.scheduler.cancel(jid)
527     common.jobDB.setStatus(nj, 'K')
528     pass
529 nsmirnov 1.4 pass
530 slacapra 1.9
531     common.jobDB.save()
532 nsmirnov 1.1 pass
533 slacapra 1.9 else:
534     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
535 nsmirnov 1.1
536 slacapra 1.8 elif ( opt == '-getoutput' ):
537     jobs = self.parseRange_(val)
538    
539 spiga 1.13 fileCODE1 = open(common.work_space.logDir()+"/.code","r")
540     array = fileCODE1.read().split('::')
541     self.ID1 = array[0]
542     self.NJC = array[1]
543     self.dataset = array[2]
544     self.owner = array[3]
545     fileCODE1.close()
546    
547    
548 slacapra 1.12 jobs_done = []
549 slacapra 1.8 for nj in jobs:
550 nsmirnov 1.4 st = common.jobDB.status(nj)
551 spiga 1.16 if st == 'D':
552 slacapra 1.12 jobs_done.append(nj)
553     pass
554     elif st == 'S':
555 nsmirnov 1.4 jid = common.jobDB.jobId(nj)
556 fanzago 1.14 print "jid", jid
557 slacapra 1.9 currStatus = common.scheduler.queryStatus(jid)
558     if currStatus=="Done":
559 slacapra 1.12 jobs_done.append(nj)
560 slacapra 1.9 else:
561     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
562 nsmirnov 1.4 common.logger.message(msg)
563     pass
564 slacapra 1.8 else:
565 spiga 1.16 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
566 slacapra 1.12 pass
567     pass
568    
569     for nj in jobs_done:
570 spiga 1.16 if ( self.flag_useboss == 1 ):
571 spiga 1.15 id = common.jobDB.bossId(nj)
572     dir = common.scheduler.getOutput(id)
573     common.jobDB.setStatus(nj, 'Y')
574     jid = common.jobDB.jobId(nj)
575     else:
576     jid = common.jobDB.jobId(nj)
577     dir = common.scheduler.getOutput(jid)
578     common.jobDB.setStatus(nj, 'Y')
579 slacapra 1.12
580     # Rename the directory with results to smth readable
581     new_dir = common.work_space.resDir()
582 spiga 1.16 if ( self.flag_useboss == 0 ):
583 spiga 1.15 try:
584     files = os.listdir(dir)
585     for file in files:
586     os.rename(dir+'/'+file, new_dir+'/'+file)
587     os.rmdir(dir)
588     except OSError, e:
589     msg = 'rename files from '+dir+' to '+new_dir+' error: '
590     msg += str(e)
591     common.logger.message(msg)
592     # ignore error
593     pass
594 slacapra 1.12 pass
595 spiga 1.13 destination = common.scheduler.queryDest(jid).split(":")[0]
596     ID3 = jid.split("/")[3]
597     broker = jid.split("/")[2].split(":")[0]
598     resFlag = 0
599     exCode = common.scheduler.getExitStatus(jid)
600     Statistic.notify('retrieved',resFlag,exCode,self.dataset,self.owner,destination,broker,ID3,self.ID1,self.NJC)
601 slacapra 1.12
602     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
603     common.logger.message(msg)
604 nsmirnov 1.4 pass
605    
606     common.jobDB.save()
607     pass
608    
609     elif ( opt == '-resubmit' ):
610 slacapra 1.11 if val:
611     jobs = self.parseRange_(val)
612 slacapra 1.8
613 slacapra 1.11 # create a list of jobs to be resubmitted.
614 slacapra 1.8
615 slacapra 1.11 nj_list = []
616     for nj in jobs:
617     st = common.jobDB.status(nj)
618 slacapra 1.8
619 slacapra 1.11 if st in ['K','A']:
620     nj_list.append(nj)
621     elif st == 'Y':
622     # here I would like to move the old output to a new place
623     # I would need the name of the output files.
624     # these infos are in the jobType class, which is not accessible from here!
625     outSandbox = common.jobDB.outputSandbox(nj)
626    
627     resDir = common.work_space.resDir()
628     resDirSave = resDir + self.current_time
629     os.mkdir(resDirSave)
630    
631     for file in outSandbox:
632     if os.path.exists(resDir+'/'+file):
633     os.rename(resDir+'/'+file, resDirSave+'/'+file)
634     common.logger.message('Output file '+file+' moved to '+resDirSave)
635     pass
636     nj_list.append(nj)
637 spiga 1.13 st = common.jobDB.setStatus(nj,'RC')
638 slacapra 1.11 elif st == 'D':
639     ## Done but not yet retrieved
640     ## retrieve job, then go to previous ('Y') case
641     ## TODO
642     pass
643     else:
644     common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
645 slacapra 1.9 pass
646 slacapra 1.8
647 slacapra 1.11 if len(nj_list) != 0:
648     # Instantiate Submitter object
649     self.actions[opt] = Submitter(self.cfg_params, nj_list)
650    
651     # Create and initialize JobList
652    
653     if len(common.job_list) == 0 :
654     common.job_list = JobList(common.jobDB.nJobs(),
655     None)
656     common.job_list.setJDLNames(self.job_type_name+'.jdl')
657     pass
658 slacapra 1.8 pass
659     pass
660 slacapra 1.11 else:
661     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
662     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
663     pass
664 slacapra 1.8 pass
665    
666     elif ( opt == '-cancelAndResubmit' ):
667     jobs = self.parseRange_(val)
668 nsmirnov 1.5
669 nsmirnov 1.7 # Cancel submitted jobs.
670 nsmirnov 1.5
671 slacapra 1.8 nj_list = []
672     for nj in jobs:
673 nsmirnov 1.5 st = common.jobDB.status(nj)
674     if st == 'S':
675     jid = common.jobDB.jobId(nj)
676     common.scheduler.cancel(jid)
677     st = 'K'
678     common.jobDB.setStatus(nj, st)
679     pass
680 nsmirnov 1.7 pass
681 nsmirnov 1.5
682 slacapra 1.8 if st != 'X': nj_list.append(nj)
683     pass
684 nsmirnov 1.5
685     if len(nj_list) != 0:
686     # Instantiate Submitter object
687     self.actions[opt] = Submitter(self.cfg_params, nj_list)
688    
689     # Create and initialize JobList
690    
691     if len(common.job_list) == 0 :
692     common.job_list = JobList(common.jobDB.nJobs(),
693     None)
694     common.job_list.setJDLNames(self.job_type_name+'.jdl')
695     pass
696     pass
697 nsmirnov 1.4 pass
698    
699 slacapra 1.8 elif ( opt == '-check' ):
700     jobs = self.parseRange_(val)
701     nj_list = []
702     for nj in jobs:
703     st = common.jobDB.status(nj)
704 slacapra 1.12 if st == 'C': nj_list.append(nj)
705 slacapra 1.8 pass
706    
707     if len(nj_list) != 0:
708     # Instantiate Submitter object
709     self.actions[opt] = Checker(self.cfg_params, nj_list)
710    
711     # Create and initialize JobList
712    
713     if len(common.job_list) == 0 :
714     common.job_list = JobList(common.jobDB.nJobs(), None)
715     common.job_list.setJDLNames(self.job_type_name+'.jdl')
716     pass
717     pass
718    
719 slacapra 1.9 elif ( opt == '-postMortem' ):
720     jobs = self.parseRange_(val)
721     nj_list = []
722     for nj in jobs:
723     st = common.jobDB.status(nj)
724 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
725 slacapra 1.9 pass
726    
727     if len(nj_list) != 0:
728     # Instantiate Submitter object
729     self.actions[opt] = PostMortem(self.cfg_params, nj_list)
730    
731     # Create and initialize JobList
732    
733     if len(common.job_list) == 0 :
734     common.job_list = JobList(common.jobDB.nJobs(), None)
735     common.job_list.setJDLNames(self.job_type_name+'.jdl')
736     pass
737     pass
738    
739     elif ( opt == '-clean' ):
740     if val != None:
741     raise CrabException("No range allowed for '-clean'")
742    
743     submittedJobs=0
744 slacapra 1.11 doneJobs=0
745 slacapra 1.12 try:
746     for nj in range(0,common.jobDB.nJobs()):
747     st = common.jobDB.status(nj)
748     if st == 'S':
749     submittedJobs = submittedJobs+1
750     if st == 'D':
751     doneJobs = doneJobs+1
752     pass
753     pass
754     except DBException:
755     common.logger.debug(5,'DB not found, so delete all')
756 nsmirnov 1.1 pass
757 slacapra 1.9
758 slacapra 1.11 if submittedJobs or doneJobs:
759     msg = "There are still "
760     if submittedJobs:
761     msg= msg+str(submittedJobs)+" jobs submitted. Kill them '-kill' before '-clean'"
762     if (submittedJobs and doneJobs):
763     msg = msg + "and \nalso"
764     if doneJobs:
765     msg= msg+str(doneJobs)+" jobs Done. Get their outputs '-getoutput' before '-clean'"
766     raise CrabException(msg)
767 slacapra 1.9
768     msg = 'directory '+common.work_space.topDir()+' removed'
769     common.work_space.delete()
770     common.logger.message(msg)
771    
772 nsmirnov 1.1 pass
773 slacapra 1.9
774 nsmirnov 1.1 pass
775     return
776    
777 nsmirnov 1.3 def createWorkingSpace_(self):
778 slacapra 1.9 new_dir = ''
779    
780     try:
781     new_dir = self.cfg_params['USER.ui_working_dir']
782     except KeyError:
783     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
784     new_dir = self.cwd + new_dir
785     pass
786    
787 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
788     common.work_space.create()
789     return
790    
791 nsmirnov 1.3 def loadConfiguration_(self, opts):
792 nsmirnov 1.1
793     save_opts = common.work_space.loadSavedOptions()
794    
795     # Override saved options with new command-line options
796    
797     for k in opts.keys():
798     save_opts[k] = opts[k]
799     pass
800    
801     # Return updated options
802     return save_opts
803    
804 nsmirnov 1.3 def createLogger_(self, args):
805 nsmirnov 1.1
806     log = Logger()
807     log.quiet(self.flag_quiet)
808     log.setDebugLevel(self.debug_level)
809     log.write(args+'\n')
810 nsmirnov 1.3 log.message(self.headerString_())
811 nsmirnov 1.1 log.flush()
812     common.logger = log
813     return
814    
815 nsmirnov 1.3 def updateHistory_(self, args):
816 nsmirnov 1.1 history_fname = common.prog_name+'.history'
817     history_file = open(history_fname, 'a')
818     history_file.write(self.current_time+': '+args+'\n')
819     history_file.close()
820     return
821    
822 nsmirnov 1.3 def headerString_(self):
823 nsmirnov 1.1 """
824     Creates a string describing program options either given in
825     the command line or their default values.
826     """
827     header = common.prog_name + ' (version ' + common.prog_version_str + \
828     ') running on ' + \
829     time.ctime(time.time())+'\n\n' + \
830     common.prog_name+'. Working options:\n'
831     header = header +\
832     ' scheduler ' + self.scheduler_name + '\n'+\
833     ' job type ' + self.job_type_name + '\n'+\
834     ' working directory ' + common.work_space.topDir()\
835     + '\n'
836     return header
837    
838 nsmirnov 1.3 def createScheduler_(self):
839 nsmirnov 1.1 """
840     Creates a scheduler object instantiated by its name.
841     """
842     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
843     file_name = klass_name
844     try:
845     klass = importName(file_name, klass_name)
846     except KeyError:
847     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
848     raise CrabException(msg)
849     except ImportError, e:
850     msg = 'Cannot create scheduler '+self.scheduler_name
851     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
852     msg += str(e)
853     raise CrabException(msg)
854    
855     common.scheduler = klass()
856     common.scheduler.configure(self.cfg_params)
857     return
858    
859     def run(self):
860     """
861     For each
862     """
863    
864     for act in self.main_actions:
865     if act in self.actions.keys(): self.actions[act].run()
866     pass
867    
868     for act in self.aux_actions:
869     if act in self.actions.keys(): self.actions[act].run()
870     pass
871     return
872    
873     ###########################################################################
874     def processHelpOptions(opts):
875    
876 slacapra 1.11 if len(opts):
877     for opt in opts.keys():
878     if opt in ('-v', '-version', '--version') :
879     print Crab.version()
880     return 1
881     if opt in ('-h','-help','--help') :
882     if opts[opt] : help(opts[opt])
883     else: help()
884     return 1
885     else:
886     usage()
887 nsmirnov 1.1
888     return 0
889    
890     ###########################################################################
891     if __name__ == '__main__':
892    
893 fanzago 1.18
894     # Initial settings for Python modules. Avoid appending manually lib paths.
895     try:
896     path=os.environ['EDG_WL_LOCATION']
897     except:
898     print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
899     sys.exit(1)
900    
901     libPath=os.path.join(path, "lib")
902     sys.path.append(libPath)
903     libPath=os.path.join(path, "lib", "python")
904     sys.path.append(libPath)
905    
906    
907 nsmirnov 1.1 # Parse command-line options and create a dictionary with
908     # key-value pairs.
909    
910     options = parseOptions(sys.argv[1:])
911    
912     # Process "help" options, such as '-help', '-version'
913    
914 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
915 nsmirnov 1.1
916     # Create, initialize, and run a Crab object
917    
918     try:
919 nsmirnov 1.3 crab = Crab(options)
920 nsmirnov 1.1 crab.run()
921     except CrabException, e:
922     print '\n' + common.prog_name + ': ' + str(e) + '\n'
923     if common.logger:
924     common.logger.write('ERROR: '+str(e)+'\n')
925     pass
926     pass
927    
928     pass