ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.42
Committed: Thu Dec 15 15:03:05 2005 UTC (19 years, 4 months ago) by corvo
Content type: text/x-python
Branch: MAIN
Changes since 1.41: +7 -2 lines
Log Message: 
Apmon is instanciated here now

File Contents

# User Rev Content
1 slacapra 1.27 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 corvo 1.42 from ApmonIf import ApmonIf
16 slacapra 1.36 from Cleaner import Cleaner
17 nsmirnov 1.1 import common
18 spiga 1.13 import Statistic
19 nsmirnov 1.1
20     import sys, os, time, string
21    
22     ###########################################################################
23     class Crab:
24 nsmirnov 1.3 def __init__(self, opts):
25 nsmirnov 1.1
26     # The order of main_actions is important !
27 slacapra 1.28 self.main_actions = [ '-create', '-submit' ]
28 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
29 slacapra 1.28 '-resubmit' , '-cancelAndResubmit', '-testJdl', '-postMortem', '-clean']
30 nsmirnov 1.1
31     # Dictionary of actions, e.g. '-create' -> object of class Creator
32     self.actions = {}
33    
34     # Configuration file
35     self.cfg_fname = None
36     # Dictionary with configuration parameters
37     self.cfg_params = {}
38    
39     # Current working directory
40     self.cwd = os.getcwd()+'/'
41     # Current time in format 'yymmdd_hhmmss'
42     self.current_time = time.strftime('%y%m%d_%H%M%S',
43     time.localtime(time.time()))
44    
45 nsmirnov 1.3 # Session name (?) Do we need this ?
46 nsmirnov 1.1 self.name = '0'
47    
48     # Job type
49     self.job_type_name = None
50    
51     # Continuation flag
52     self.flag_continue = 0
53    
54     # quiet mode, i.e. no output on screen
55     self.flag_quiet = 0
56     # produce more output
57     self.debug_level = 0
58    
59 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
60 fanzago 1.14 self.scheduler_name = ''
61 nsmirnov 1.1
62 nsmirnov 1.3 self.initialize_(opts)
63 nsmirnov 1.1
64     return
65    
66 nsmirnov 1.7 def version():
67 nsmirnov 1.1 return common.prog_version_str
68    
69 nsmirnov 1.7 version = staticmethod(version)
70    
71 nsmirnov 1.3 def initialize_(self, opts):
72 nsmirnov 1.1
73     # Process the '-continue' option first because
74     # in the case of continuation the CRAB configuration
75     # parameters are loaded from already existing Working Space.
76 nsmirnov 1.3 self.processContinueOption_(opts)
77 nsmirnov 1.1
78     # Process ini-file first, then command line options
79     # because they override ini-file settings.
80    
81 nsmirnov 1.3 self.processIniFile_(opts)
82 nsmirnov 1.1
83 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
84 nsmirnov 1.1
85 nsmirnov 1.3 self.processOptions_(opts)
86 nsmirnov 1.1
87     if not self.flag_continue:
88 nsmirnov 1.3 self.createWorkingSpace_()
89 slacapra 1.9 optsToBeSaved={}
90     for it in opts.keys():
91     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
92     pass
93     else:
94     optsToBeSaved[it]=opts[it]
95     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
96 nsmirnov 1.1 pass
97    
98     # At this point all configuration options have been read.
99    
100     args = string.join(sys.argv,' ')
101 slacapra 1.11
102 nsmirnov 1.3 self.updateHistory_(args)
103 slacapra 1.11
104 nsmirnov 1.3 self.createLogger_(args)
105 slacapra 1.11
106 nsmirnov 1.1 common.jobDB = JobDB()
107 slacapra 1.11
108 nsmirnov 1.3 if self.flag_continue:
109 slacapra 1.12 try:
110     common.jobDB.load()
111     common.logger.debug(6, str(common.jobDB))
112     except DBException,e:
113     pass
114 nsmirnov 1.3 pass
115 slacapra 1.11
116 nsmirnov 1.3 self.createScheduler_()
117 slacapra 1.11
118 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
119     common.logger.debug(6, 'Used properties:')
120     keys = self.cfg_params.keys()
121     keys.sort()
122     for k in keys:
123     if self.cfg_params[k]:
124 corvo 1.42 common.logger.debug(6, ' '+k+' : '+str(self.cfg_params[k]))
125 nsmirnov 1.3 pass
126     else:
127     common.logger.debug(6, ' '+k+' : ')
128     pass
129     pass
130     common.logger.debug(6, 'End of used properties.\n')
131     pass
132     self.initializeActions_(opts)
133 nsmirnov 1.1 return
134    
135 nsmirnov 1.3 def processContinueOption_(self,opts):
136 nsmirnov 1.1
137     continue_dir = None
138 nsmirnov 1.4
139     # Look for the '-continue' option.
140    
141 nsmirnov 1.1 for opt in opts.keys():
142     if ( opt in ('-continue','-c') ):
143     self.flag_continue = 1
144     val = opts[opt]
145     if val:
146     if val[0] == '/': continue_dir = val # abs path
147     else: continue_dir = self.cwd + val # rel path
148     pass
149 nsmirnov 1.4 break
150     pass
151    
152     # Look for actions which has sense only with '-continue'
153    
154     if not self.flag_continue:
155     for opt in opts.keys():
156 slacapra 1.6 if ( opt in (self.aux_actions) ):
157 nsmirnov 1.4 self.flag_continue = 1
158     break
159 nsmirnov 1.1 pass
160     pass
161 slacapra 1.6 submit_flag=0
162     create_flag=0
163     for opt in opts.keys():
164     if opt == "-submit": submit_flag=1
165     if opt == "-create": create_flag=1
166     pass
167     if (submit_flag and not create_flag):
168 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
169     raise CrabException(msg)
170 slacapra 1.6 pass
171 nsmirnov 1.1
172     if not self.flag_continue: return
173    
174     if not continue_dir:
175     prefix = common.prog_name + '_' + self.name + '_'
176     continue_dir = findLastWorkDir(prefix)
177     pass
178    
179     if not continue_dir:
180     raise CrabException('Cannot find last working directory.')
181    
182     if not os.path.exists(continue_dir):
183     msg = 'Cannot continue because the working directory <'
184     msg += continue_dir
185     msg += '> does not exist.'
186     raise CrabException(msg)
187    
188     # Instantiate WorkSpace
189     common.work_space = WorkSpace(continue_dir)
190    
191     return
192    
193 nsmirnov 1.3 def processIniFile_(self, opts):
194 nsmirnov 1.1 """
195     Processes a configuration INI-file.
196     """
197    
198     # Extract cfg-file name from the cmd-line options.
199    
200     for opt in opts.keys():
201     if ( opt == '-cfg' ):
202     if self.flag_continue:
203     raise CrabException('-continue and -cfg cannot coexist.')
204     if opts[opt] : self.cfg_fname = opts[opt]
205     else : usage()
206     pass
207    
208     elif ( opt == '-name' ):
209     self.name = opts[opt]
210     pass
211    
212     pass
213    
214     # Set default cfg-fname
215    
216     if self.cfg_fname == None:
217     if self.flag_continue:
218     self.cfg_fname = common.work_space.cfgFileName()
219     else:
220     self.cfg_fname = common.prog_name+'.cfg'
221     pass
222     pass
223    
224     # Load cfg-file
225    
226     if string.lower(self.cfg_fname) != 'none':
227     if os.path.exists(self.cfg_fname):
228     self.cfg_params = loadConfig(self.cfg_fname)
229 corvo 1.42 # Better idea on where to put user info for ML?
230     self.cfg_params['user'] = os.getlogin()
231     # Better idea on where to put ML???? Looks crap here, but had no better thought!
232     if int(self.cfg_params['USER.activate_monalisa']): self.cfg_params['apmon'] = ApmonIf()
233 nsmirnov 1.1 pass
234     else:
235     msg = 'cfg-file '+self.cfg_fname+' not found.'
236     raise CrabException(msg)
237     pass
238     pass
239    
240     # process the [CRAB] section
241    
242     lhp = len('CRAB.')
243     for k in self.cfg_params.keys():
244     if len(k) >= lhp and k[:lhp] == 'CRAB.':
245     opt = '-'+k[lhp:]
246     if len(opt) >= 3 and opt[:3] == '-__': continue
247     if opt not in opts.keys():
248     opts[opt] = self.cfg_params[k]
249     pass
250     pass
251     pass
252    
253     return
254    
255 nsmirnov 1.3 def processOptions_(self, opts):
256 nsmirnov 1.1 """
257     Processes the command-line options.
258     """
259    
260     for opt in opts.keys():
261     val = opts[opt]
262    
263 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
264     if opt in self.main_actions:
265     self.cfg_params['CRAB.'+opt[1:]] = val
266     continue
267     if opt in self.aux_actions:
268     self.cfg_params['CRAB.'+opt[1:]] = val
269     continue
270 nsmirnov 1.1
271    
272     elif ( opt == '-cfg' ):
273     pass
274    
275     elif ( opt in ('-continue', '-c') ):
276 nsmirnov 1.4 # Already processed in processContinueOption_()
277 nsmirnov 1.1 pass
278    
279     elif ( opt == '-jobtype' ):
280     if val : self.job_type_name = string.upper(val)
281     else : usage()
282     pass
283    
284     elif ( opt == '-Q' ):
285     self.flag_quiet = 1
286     pass
287    
288     elif ( opt == '-debug' ):
289 slacapra 1.6 if val: self.debug_level = int(val)
290     else: self.debug_level = 1
291 nsmirnov 1.1 pass
292    
293     elif ( opt == '-scheduler' ):
294     if val: self.scheduler_name = val
295     else:
296     print common.prog_name+". No value for '-scheduler'."
297     usage()
298     pass
299     pass
300 slacapra 1.22
301 fanzago 1.18 elif ( opt in ('-use_boss', '-useboss') ):
302     if ( val == '1' ):
303     self.scheduler_name = 'boss'
304     pass
305     elif ( val == '0' ):
306     pass
307     else:
308     print common.prog_name+'. Bad flag for -use_boss option:',\
309     val,'Possible values are 0(=No) or 1(=Yes)'
310     usage()
311     pass
312 fanzago 1.14 pass
313    
314 nsmirnov 1.3 elif string.find(opt,'.') == -1:
315     print common.prog_name+'. Unrecognized option '+opt
316     usage()
317     pass
318 nsmirnov 1.1
319 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
320     if string.find(opt,'.') == -1 :
321     self.cfg_params['CRAB.'+opt[1:]] = val
322 nsmirnov 1.1 pass
323 nsmirnov 1.3 else:
324 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
325     self.cfg_params[opt[1:]] = val
326     pass
327     pass
328     return
329    
330 slacapra 1.8 def parseRange_(self, aRange):
331 nsmirnov 1.4 """
332 slacapra 1.8 Takes as the input a string with a range defined in any of the following
333     way, including combination, and return a tuple with the ints defined
334     according to following table. A consistency check is done.
335     NB: the first job is "1", not "0".
336     'all' -> [1,2,..., NJobs]
337     '' -> [1,2,..., NJobs]
338     'n1' -> [n1]
339     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
340     'n1,n2' -> [n1, n2]
341     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
342     """
343     result = []
344    
345 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
346     if aRange=='all' or aRange==None or aRange=='':
347 slacapra 1.8 result=range(0,common.jobDB.nJobs())
348     return result
349 slacapra 1.9 elif aRange=='0':
350     return result
351 slacapra 1.8
352     subRanges = string.split(aRange, ',')
353     for subRange in subRanges:
354     result = result+self.parseSimpleRange_(subRange)
355    
356     if self.checkUniqueness_(result):
357     return result
358     else:
359 slacapra 1.33 common.logger.message("Error "+result)
360 slacapra 1.8 return []
361    
362     def checkUniqueness_(self, list):
363     """
364 slacapra 1.9 check if a list contains only unique elements
365 slacapra 1.8 """
366    
367     uniqueList = []
368     # use a list comprehension statement (takes a while to understand)
369    
370     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
371    
372     return (len(list)==len(uniqueList))
373    
374     def parseSimpleRange_(self, aRange):
375     """
376     Takes as the input a string with two integers separated by
377     the minus sign and returns the tuple with these numbers:
378     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
379     'n1' -> [n1]
380     """
381     (start, end) = (None, None)
382    
383     result = []
384     minus = string.find(aRange, '-')
385     if ( minus < 0 ):
386     if isInt(aRange) and int(aRange)>0:
387 fanzago 1.37 # FEDE
388     #result.append(int(aRange)-1)
389     ###
390     result.append(aRange)
391 slacapra 1.8 else:
392 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
393 nsmirnov 1.4 pass
394     else:
395 slacapra 1.8 (start, end) = string.split(aRange, '-')
396     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
397 spiga 1.38 #result=range(int(start)-1, int(end))
398     result=range(int(start), int(end)+1) #Daniele
399 slacapra 1.8 else:
400 slacapra 1.33 common.logger.message("parseSimpleRange_ ERROR "+start+end)
401 slacapra 1.8
402     return result
403 nsmirnov 1.4
404 nsmirnov 1.3 def initializeActions_(self, opts):
405 nsmirnov 1.1 """
406     For each user action instantiate a corresponding
407     object and put it in the action dictionary.
408     """
409     for opt in opts.keys():
410 spiga 1.32 self.flag_useboss = 0
411 spiga 1.16 if ( opt == '-use_boss'):
412 slacapra 1.25 val = opts[opt]
413     if ( val == '1' ):
414     self.flag_useboss = 1
415     common.logger.message('Using BOSS')
416     pass
417     else:
418     self.flag_useboss = 0
419     pass
420 spiga 1.15
421     for opt in opts.keys():
422    
423 nsmirnov 1.1 val = opts[opt]
424 spiga 1.15
425    
426     if ( opt == '-create' ):
427 slacapra 1.22 ncjobs = 0
428 nsmirnov 1.1 if val:
429     if ( isInt(val) ):
430     ncjobs = int(val)
431     elif ( val == 'all'):
432     ncjobs = val
433     else:
434 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
435     msg += ' Must be an integer or "all"'
436 slacapra 1.8 msg += ' Generic range is not allowed"'
437 nsmirnov 1.5 raise CrabException(msg)
438 nsmirnov 1.1 pass
439     else: ncjobs = 'all'
440    
441     if ncjobs != 0:
442 nsmirnov 1.3 # Instantiate Creator object
443 nsmirnov 1.1 creator = Creator(self.job_type_name,
444     self.cfg_params,
445     ncjobs)
446     self.actions[opt] = creator
447 nsmirnov 1.3
448     # Initialize the JobDB object if needed
449 nsmirnov 1.1 if not self.flag_continue:
450     common.jobDB.create(creator.nJobs())
451     pass
452 nsmirnov 1.3
453     # Create and initialize JobList
454    
455     common.job_list = JobList(common.jobDB.nJobs(),
456     creator.jobType())
457    
458     common.job_list.setScriptNames(self.job_type_name+'.sh')
459     common.job_list.setJDLNames(self.job_type_name+'.jdl')
460 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
461 slacapra 1.9
462     creator.writeJobsSpecsToDB()
463 nsmirnov 1.1 pass
464 nsmirnov 1.3 pass
465 nsmirnov 1.1
466     elif ( opt == '-submit' ):
467 slacapra 1.23
468     # total jobs
469     # get the first not already submitted
470 slacapra 1.24 common.logger.debug(5,'Total jobs '+str(common.jobDB.nJobs()))
471     lastSubmittedJob=0
472     for nj in range(common.jobDB.nJobs()):
473 spiga 1.41 if (common.jobDB.status(nj)=='S') or (common.jobDB.status(nj)=='K'): #Da
474 slacapra 1.24 lastSubmittedJob +=1
475 slacapra 1.30 else: break
476 slacapra 1.24 # count job from 1
477 slacapra 1.30 totalJobsSubmittable = common.jobDB.nJobs()-lastSubmittedJob
478 slacapra 1.24 common.logger.debug(5,'lastSubmittedJob '+str(lastSubmittedJob))
479     common.logger.debug(5,'totalJobsSubmittable '+str(totalJobsSubmittable))
480 slacapra 1.23
481 slacapra 1.24 nsjobs = lastSubmittedJob+totalJobsSubmittable
482 slacapra 1.23 # get user request
483 slacapra 1.22 if val:
484     if ( isInt(val) ):
485 slacapra 1.24 tmp = int(val)
486     if (tmp >= totalJobsSubmittable):
487     common.logger.message('asking to submit '+str(tmp)+' jobs, but only '+str(totalJobsSubmittable)+' left: submitting those')
488     pass
489     else:
490     nsjobs=lastSubmittedJob+int(val)
491 slacapra 1.23 elif (val=='all'):
492     pass
493 slacapra 1.22 else:
494     msg = 'Bad submission option <'+str(val)+'>\n'
495     msg += ' Must be an integer or "all"'
496     msg += ' Generic range is not allowed"'
497     raise CrabException(msg)
498     pass
499 slacapra 1.24 common.logger.debug(5,'nsjobs '+str(nsjobs))
500 slacapra 1.23
501     # submit N from last submitted job
502 slacapra 1.24 nj_list = range(lastSubmittedJob, nsjobs)
503     common.logger.debug(5,'nj_list '+str(nj_list))
504 nsmirnov 1.5
505     if len(nj_list) != 0:
506 nsmirnov 1.3 # Instantiate Submitter object
507 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
508 nsmirnov 1.3
509     # Create and initialize JobList
510     if len(common.job_list) == 0 :
511     common.job_list = JobList(common.jobDB.nJobs(),
512     None)
513     common.job_list.setJDLNames(self.job_type_name+'.jdl')
514     pass
515 nsmirnov 1.1 pass
516 slacapra 1.30 pass
517 nsmirnov 1.1
518 nsmirnov 1.4 elif ( opt == '-list' ):
519 slacapra 1.8 jobs = self.parseRange_(val)
520    
521     common.jobDB.dump(jobs)
522 nsmirnov 1.4 pass
523    
524     elif ( opt == '-status' ):
525 slacapra 1.8 jobs = self.parseRange_(val)
526    
527 slacapra 1.9 if len(jobs) != 0:
528 spiga 1.16 if ( self.flag_useboss == 1 ):
529 corvo 1.42 self.actions[opt] = StatusBoss(self.cfg_params)
530 spiga 1.15 else:
531 slacapra 1.36 self.actions[opt] = Status(jobs)
532 spiga 1.15 pass
533 nsmirnov 1.1 pass
534     pass
535 slacapra 1.22
536 nsmirnov 1.4 elif ( opt == '-kill' ):
537 slacapra 1.8
538 spiga 1.31 if ( self.flag_useboss == 1 ):
539     if val:
540     if val =='all':
541 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
542     jobs = allBoss_id.keys()
543 spiga 1.31 else:
544     jobs = self.parseRange_(val)
545     common.scheduler.cancel(jobs)
546     else:
547     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
548     else:
549     if val:
550     jobs = self.parseRange_(val)
551    
552     for nj in jobs:
553     st = common.jobDB.status(nj)
554     if st == 'S':
555     jid = common.jobDB.jobId(nj)
556     common.logger.message("Killing job # "+`(nj+1)`)
557     common.scheduler.cancel(jid)
558     common.jobDB.setStatus(nj, 'K')
559     pass
560 slacapra 1.9 pass
561 spiga 1.31
562     common.jobDB.save()
563 nsmirnov 1.4 pass
564 spiga 1.31 else:
565     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
566 nsmirnov 1.1
567 slacapra 1.8 elif ( opt == '-getoutput' ):
568    
569 spiga 1.20 if ( self.flag_useboss == 1 ):
570     if val=='all' or val==None or val=='':
571 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
572     jobs = allBoss_id.keys()
573 spiga 1.20 else:
574     jobs = self.parseRange_(val)
575     common.scheduler.getOutput(jobs)
576     else:
577     jobs = self.parseRange_(val)
578 spiga 1.13
579 slacapra 1.22 ## also this: create a ActorClass (GetOutput)
580 spiga 1.20 jobs_done = []
581     for nj in jobs:
582     st = common.jobDB.status(nj)
583     if st == 'D':
584 slacapra 1.12 jobs_done.append(nj)
585 spiga 1.20 pass
586     elif st == 'S':
587     jid = common.jobDB.jobId(nj)
588     currStatus = common.scheduler.queryStatus(jid)
589     if currStatus=="Done":
590     jobs_done.append(nj)
591     else:
592     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
593     common.logger.message(msg)
594     pass
595 slacapra 1.9 else:
596 spiga 1.20 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
597     pass
598 slacapra 1.12 pass
599    
600 spiga 1.20 for nj in jobs_done:
601 spiga 1.15 jid = common.jobDB.jobId(nj)
602     dir = common.scheduler.getOutput(jid)
603     common.jobDB.setStatus(nj, 'Y')
604 slacapra 1.12
605     # Rename the directory with results to smth readable
606     new_dir = common.work_space.resDir()
607 spiga 1.20 try:
608     files = os.listdir(dir)
609     for file in files:
610     os.rename(dir+'/'+file, new_dir+'/'+file)
611     os.rmdir(dir)
612     except OSError, e:
613     msg = 'rename files from '+dir+' to '+new_dir+' error: '
614     msg += str(e)
615     common.logger.message(msg)
616     # ignore error
617 slacapra 1.12 pass
618 spiga 1.20 pass
619 slacapra 1.22 ###
620    
621 spiga 1.13 resFlag = 0
622     exCode = common.scheduler.getExitStatus(jid)
623 spiga 1.34 Statistic.Monitor('retrieved',resFlag,jid,exCode)
624 slacapra 1.12
625     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
626     common.logger.message(msg)
627 nsmirnov 1.4 pass
628    
629     common.jobDB.save()
630     pass
631    
632     elif ( opt == '-resubmit' ):
633 fanzago 1.37 if ( self.flag_useboss == 1 ):
634     if val=='all' or val==None or val=='':
635     allBoss_id = common.scheduler.listBoss()
636     # sono gli internal_id di BOSS
637     jobs = allBoss_id.keys()
638     else:
639     jobs = self.parseRange_(val)
640     else:
641     if val:
642     jobs = self.parseRange_(val)
643    
644 slacapra 1.11 if val:
645     # create a list of jobs to be resubmitted.
646 slacapra 1.8
647 slacapra 1.22 ### as before, create a Resubmittter Class
648 slacapra 1.11 nj_list = []
649     for nj in jobs:
650 fanzago 1.37 st = common.jobDB.status(int(nj)-1)
651     #print "stato del job = ", st
652 slacapra 1.11 if st in ['K','A']:
653 fanzago 1.37 nj_list.append(int(nj)-1)
654     common.jobDB.setStatus(int(nj)-1,'C')
655 slacapra 1.11 elif st == 'Y':
656 fanzago 1.39 common.scheduler.moveOutput(nj)
657 fanzago 1.37 nj_list.append(int(nj)-1)
658     st = common.jobDB.setStatus(int(nj)-1,'RC')
659     elif st in ['D','C','X']:
660     # Nel caso DONE bisogna prima ritirare l'output
661     # Nel caso C va prima sottomesso
662     # Nel caso X va prima creato
663 slacapra 1.11 pass
664     else:
665 fanzago 1.37 #common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
666     common.logger.message('Job #'+nj+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
667 slacapra 1.9 pass
668 slacapra 1.8
669 fanzago 1.37 if len(common.job_list) == 0 :
670     #print "common.job_list = ", common.job_list
671     common.job_list = JobList(common.jobDB.nJobs(),None)
672     common.job_list.setJDLNames(self.job_type_name+'.jdl')
673     pass
674    
675 slacapra 1.11 if len(nj_list) != 0:
676 fanzago 1.37 #print "nj_list = ", nj_list
677     common.scheduler.resubmit(nj_list)
678 slacapra 1.11 # Instantiate Submitter object
679     self.actions[opt] = Submitter(self.cfg_params, nj_list)
680    
681 slacapra 1.8 pass
682     pass
683 slacapra 1.11 else:
684     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
685     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
686     pass
687 fanzago 1.37 common.jobDB.save()
688 slacapra 1.8 pass
689    
690     elif ( opt == '-cancelAndResubmit' ):
691     jobs = self.parseRange_(val)
692 nsmirnov 1.5
693 nsmirnov 1.7 # Cancel submitted jobs.
694 nsmirnov 1.5
695 slacapra 1.8 nj_list = []
696     for nj in jobs:
697 nsmirnov 1.5 st = common.jobDB.status(nj)
698     if st == 'S':
699     jid = common.jobDB.jobId(nj)
700     common.scheduler.cancel(jid)
701     st = 'K'
702     common.jobDB.setStatus(nj, st)
703     pass
704 nsmirnov 1.7 pass
705 nsmirnov 1.5
706 slacapra 1.8 if st != 'X': nj_list.append(nj)
707     pass
708 nsmirnov 1.5
709     if len(nj_list) != 0:
710     # Instantiate Submitter object
711     self.actions[opt] = Submitter(self.cfg_params, nj_list)
712    
713     # Create and initialize JobList
714    
715     if len(common.job_list) == 0 :
716     common.job_list = JobList(common.jobDB.nJobs(),
717     None)
718     common.job_list.setJDLNames(self.job_type_name+'.jdl')
719     pass
720     pass
721 nsmirnov 1.4 pass
722    
723 slacapra 1.28 elif ( opt == '-testJdl' ):
724 slacapra 1.8 jobs = self.parseRange_(val)
725     nj_list = []
726     for nj in jobs:
727     st = common.jobDB.status(nj)
728 slacapra 1.12 if st == 'C': nj_list.append(nj)
729 slacapra 1.8 pass
730    
731     if len(nj_list) != 0:
732     # Instantiate Submitter object
733     self.actions[opt] = Checker(self.cfg_params, nj_list)
734    
735     # Create and initialize JobList
736    
737     if len(common.job_list) == 0 :
738     common.job_list = JobList(common.jobDB.nJobs(), None)
739     common.job_list.setJDLNames(self.job_type_name+'.jdl')
740     pass
741     pass
742    
743 slacapra 1.9 elif ( opt == '-postMortem' ):
744     jobs = self.parseRange_(val)
745     nj_list = []
746     for nj in jobs:
747     st = common.jobDB.status(nj)
748 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
749 slacapra 1.9 pass
750    
751     if len(nj_list) != 0:
752     # Instantiate Submitter object
753 spiga 1.35 self.actions[opt] = PostMortem(self.cfg_params, nj_list,self.flag_useboss)
754 slacapra 1.9
755     # Create and initialize JobList
756    
757     if len(common.job_list) == 0 :
758     common.job_list = JobList(common.jobDB.nJobs(), None)
759     common.job_list.setJDLNames(self.job_type_name+'.jdl')
760     pass
761     pass
762 slacapra 1.33 else:
763     common.logger.message("No jobs to analyze")
764 slacapra 1.9
765     elif ( opt == '-clean' ):
766     if val != None:
767     raise CrabException("No range allowed for '-clean'")
768    
769 slacapra 1.36 theCleaner = Cleaner(self.scheduler_name == 'boss')
770     theCleaner.clean()
771 slacapra 1.9
772 nsmirnov 1.1 pass
773     return
774    
775 nsmirnov 1.3 def createWorkingSpace_(self):
776 slacapra 1.9 new_dir = ''
777    
778     try:
779     new_dir = self.cfg_params['USER.ui_working_dir']
780     except KeyError:
781     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
782     new_dir = self.cwd + new_dir
783     pass
784    
785 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
786     common.work_space.create()
787     return
788    
789 nsmirnov 1.3 def loadConfiguration_(self, opts):
790 nsmirnov 1.1
791     save_opts = common.work_space.loadSavedOptions()
792    
793     # Override saved options with new command-line options
794    
795     for k in opts.keys():
796     save_opts[k] = opts[k]
797     pass
798    
799     # Return updated options
800     return save_opts
801    
802 nsmirnov 1.3 def createLogger_(self, args):
803 nsmirnov 1.1
804     log = Logger()
805     log.quiet(self.flag_quiet)
806     log.setDebugLevel(self.debug_level)
807     log.write(args+'\n')
808 nsmirnov 1.3 log.message(self.headerString_())
809 nsmirnov 1.1 log.flush()
810     common.logger = log
811     return
812    
813 nsmirnov 1.3 def updateHistory_(self, args):
814 nsmirnov 1.1 history_fname = common.prog_name+'.history'
815     history_file = open(history_fname, 'a')
816     history_file.write(self.current_time+': '+args+'\n')
817     history_file.close()
818     return
819    
820 nsmirnov 1.3 def headerString_(self):
821 nsmirnov 1.1 """
822     Creates a string describing program options either given in
823     the command line or their default values.
824     """
825     header = common.prog_name + ' (version ' + common.prog_version_str + \
826     ') running on ' + \
827     time.ctime(time.time())+'\n\n' + \
828     common.prog_name+'. Working options:\n'
829     header = header +\
830     ' scheduler ' + self.scheduler_name + '\n'+\
831     ' job type ' + self.job_type_name + '\n'+\
832     ' working directory ' + common.work_space.topDir()\
833     + '\n'
834     return header
835    
836 nsmirnov 1.3 def createScheduler_(self):
837 nsmirnov 1.1 """
838     Creates a scheduler object instantiated by its name.
839     """
840     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
841     file_name = klass_name
842     try:
843     klass = importName(file_name, klass_name)
844     except KeyError:
845     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
846     raise CrabException(msg)
847     except ImportError, e:
848     msg = 'Cannot create scheduler '+self.scheduler_name
849     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
850     msg += str(e)
851     raise CrabException(msg)
852    
853     common.scheduler = klass()
854     common.scheduler.configure(self.cfg_params)
855     return
856    
857     def run(self):
858     """
859     For each
860     """
861    
862     for act in self.main_actions:
863     if act in self.actions.keys(): self.actions[act].run()
864     pass
865    
866     for act in self.aux_actions:
867     if act in self.actions.keys(): self.actions[act].run()
868     pass
869     return
870    
871     ###########################################################################
872     def processHelpOptions(opts):
873    
874 slacapra 1.11 if len(opts):
875     for opt in opts.keys():
876     if opt in ('-v', '-version', '--version') :
877     print Crab.version()
878     return 1
879     if opt in ('-h','-help','--help') :
880     if opts[opt] : help(opts[opt])
881     else: help()
882     return 1
883     else:
884     usage()
885 nsmirnov 1.1
886     return 0
887    
888     ###########################################################################
889     if __name__ == '__main__':
890    
891 fanzago 1.18
892 fanzago 1.19 # # Initial settings for Python modules. Avoid appending manually lib paths.
893     # try:
894     # path=os.environ['EDG_WL_LOCATION']
895     # except:
896     # print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
897     # sys.exit(1)
898     #
899     # libPath=os.path.join(path, "lib")
900     # sys.path.append(libPath)
901     # libPath=os.path.join(path, "lib", "python")
902     # sys.path.append(libPath)
903 fanzago 1.18
904    
905 nsmirnov 1.1 # Parse command-line options and create a dictionary with
906     # key-value pairs.
907    
908     options = parseOptions(sys.argv[1:])
909    
910     # Process "help" options, such as '-help', '-version'
911    
912 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
913 nsmirnov 1.1
914     # Create, initialize, and run a Crab object
915    
916     try:
917 nsmirnov 1.3 crab = Crab(options)
918 nsmirnov 1.1 crab.run()
919     except CrabException, e:
920     print '\n' + common.prog_name + ': ' + str(e) + '\n'
921     if common.logger:
922     common.logger.write('ERROR: '+str(e)+'\n')
923     pass
924     pass
925    
926     pass