ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.155
Committed: Thu Mar 20 11:28:42 2008 UTC (17 years, 1 month ago) by spiga
Content type: text/x-python
Branch: MAIN
Changes since 1.154: +0 -14 lines
Log Message: 
minor changes for ServerConfig usage... able to accept any capitalization

File Contents

# User Rev Content
1 elmer 1.65 #!/usr/bin/env python
2 spiga 1.151 import sys, os, time, string
3 farinafa 1.133
4 spiga 1.151 ### Hopefully will be removed soon!!! DS
5 farinafa 1.133 ## pre-import env configuratin steps
6 spiga 1.151 #def dropOutPy23dynLoads():
7     # # FEDE added -publish
8     # #if not ('-create' in sys.argv or '-publish' in sys.argv):
9     # if not ('-create' in sys.argv or '-publish' in sys.argv):
10     # return
11     # tmp = []
12     # for p in sys.path:
13     # if p.find( "python2.3/lib-dynload" ) == -1 :
14     # tmp.append(p)
15     # pass
16     #
17     # sys.path=tmp
18     # pass
19 slacapra 1.144
20 farinafa 1.133 # this is needed to remove interferences between LCG and CMSSW envs
21 spiga 1.151 #dropOutPy23dynLoads()
22 farinafa 1.133
23     ## actual import session
24 nsmirnov 1.1 from crab_util import *
25     from crab_exceptions import *
26     from crab_logger import Logger
27     from WorkSpace import WorkSpace
28 spiga 1.151 from DBinterface import DBinterface ## added to interface with DB BL--DS
29 nsmirnov 1.3 from JobList import JobList
30 corvo 1.42 from ApmonIf import ApmonIf
31 nsmirnov 1.1 import common
32 spiga 1.107
33 nsmirnov 1.1 ###########################################################################
34     class Crab:
35 nsmirnov 1.3 def __init__(self, opts):
36 fanzago 1.76 ## test_tag
37 nsmirnov 1.1 # The order of main_actions is important !
38 fanzago 1.111 self.main_actions = [ '-create', '-submit' ]
39     ### FEDE new option "-publish" FOR DBS OUTPUT PUBLICATION
40 slacapra 1.70 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput','-get',
41 slacapra 1.67 '-resubmit' , '-cancelAndResubmit', '-testJdl', '-postMortem', '-clean',
42 fanzago 1.111 '-printId', '-publish' ]
43 nsmirnov 1.1
44     # Dictionary of actions, e.g. '-create' -> object of class Creator
45     self.actions = {}
46 fanzago 1.76
47 nsmirnov 1.1 # Configuration file
48     self.cfg_fname = None
49     # Dictionary with configuration parameters
50     self.cfg_params = {}
51    
52     # Current working directory
53     self.cwd = os.getcwd()+'/'
54     # Current time in format 'yymmdd_hhmmss'
55     self.current_time = time.strftime('%y%m%d_%H%M%S',
56     time.localtime(time.time()))
57    
58 nsmirnov 1.3 # Session name (?) Do we need this ?
59 nsmirnov 1.1 self.name = '0'
60    
61     # Job type
62     self.job_type_name = None
63    
64     # Continuation flag
65     self.flag_continue = 0
66    
67     # quiet mode, i.e. no output on screen
68     self.flag_quiet = 0
69     # produce more output
70     self.debug_level = 0
71    
72    
73 nsmirnov 1.3 self.initialize_(opts)
74 nsmirnov 1.1
75     return
76    
77 nsmirnov 1.7 def version():
78 nsmirnov 1.1 return common.prog_version_str
79    
80 nsmirnov 1.7 version = staticmethod(version)
81    
82 nsmirnov 1.3 def initialize_(self, opts):
83 nsmirnov 1.1
84     # Process the '-continue' option first because
85     # in the case of continuation the CRAB configuration
86     # parameters are loaded from already existing Working Space.
87 nsmirnov 1.3 self.processContinueOption_(opts)
88 nsmirnov 1.1
89     # Process ini-file first, then command line options
90     # because they override ini-file settings.
91    
92 nsmirnov 1.3 self.processIniFile_(opts)
93 nsmirnov 1.1
94 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
95 nsmirnov 1.1
96 nsmirnov 1.3 self.processOptions_(opts)
97 nsmirnov 1.1
98 slacapra 1.77
99 farinafa 1.154 # self.UseServer=int(self.cfg_params.get('CRAB.server_mode',0))
100     #
101     srvName = opts.get('-server_name', None)
102     if srvName == 'None':
103     srvName = None
104     self.UseServer = int( srvName is not None ) # cast bool to int
105    
106 spiga 1.152 common._db = DBinterface(self.cfg_params) #BL--DS
107 slacapra 1.101
108 nsmirnov 1.1 if not self.flag_continue:
109 nsmirnov 1.3 self.createWorkingSpace_()
110 spiga 1.152 common._db.configureDB()
111 slacapra 1.9 optsToBeSaved={}
112 spiga 1.151 optsToBeSavedDB={}
113 slacapra 1.9 for it in opts.keys():
114     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
115     pass
116     else:
117 spiga 1.151 optsToBeSavedDB[it[1:]]=opts[it]
118 slacapra 1.9 optsToBeSaved[it]=opts[it]
119 slacapra 1.101 # store in taskDB the opts
120 farinafa 1.154
121    
122 spiga 1.151 common._db.createTask_(optsToBeSavedDB) #BL--DS
123 slacapra 1.9 common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
124 spiga 1.152 else:
125     common._db.loadDB()
126 nsmirnov 1.1
127     # At this point all configuration options have been read.
128     args = string.join(sys.argv,' ')
129 slacapra 1.11
130 nsmirnov 1.3 self.updateHistory_(args)
131 slacapra 1.11
132 nsmirnov 1.3 self.createLogger_(args)
133 corvo 1.51
134 slacapra 1.144 common.apmon = ApmonIf()
135 slacapra 1.11
136 nsmirnov 1.3 self.createScheduler_()
137 slacapra 1.11
138 slacapra 1.106 common.logger.debug(6, 'Used properties:')
139     if (common.logger.debugLevel()<6 ):
140     common.logger.write('Used properties:')
141     keys = self.cfg_params.keys()
142     keys.sort()
143     for k in keys:
144     if self.cfg_params[k]:
145     common.logger.debug(6, ' '+k+' : '+str(self.cfg_params[k]))
146     if (common.logger.debugLevel()<6 ):
147     common.logger.write(' '+k+' : '+str(self.cfg_params[k]))
148     pass
149     else:
150     common.logger.debug(6, ' '+k+' : ')
151     if (common.logger.debugLevel()<6 ):
152     common.logger.write(' '+k+' : ')
153 nsmirnov 1.3 pass
154     pass
155 slacapra 1.106 common.logger.debug(6, 'End of used properties.\n')
156     if (common.logger.debugLevel()<6 ):
157     common.logger.write('End of used properties.\n')
158    
159 nsmirnov 1.3 self.initializeActions_(opts)
160 nsmirnov 1.1 return
161    
162 nsmirnov 1.3 def processContinueOption_(self,opts):
163 nsmirnov 1.1
164     continue_dir = None
165 nsmirnov 1.4
166     # Look for the '-continue' option.
167    
168 nsmirnov 1.1 for opt in opts.keys():
169     if ( opt in ('-continue','-c') ):
170     self.flag_continue = 1
171     val = opts[opt]
172     if val:
173     if val[0] == '/': continue_dir = val # abs path
174     else: continue_dir = self.cwd + val # rel path
175     pass
176 nsmirnov 1.4 break
177     pass
178    
179     # Look for actions which has sense only with '-continue'
180    
181 slacapra 1.144 if "-create" not in opts.keys() :
182     self.flag_continue = 1
183 nsmirnov 1.1
184     if not self.flag_continue: return
185    
186     if not continue_dir:
187     prefix = common.prog_name + '_' + self.name + '_'
188     continue_dir = findLastWorkDir(prefix)
189     pass
190    
191     if not continue_dir:
192     raise CrabException('Cannot find last working directory.')
193    
194     if not os.path.exists(continue_dir):
195     msg = 'Cannot continue because the working directory <'
196     msg += continue_dir
197     msg += '> does not exist.'
198     raise CrabException(msg)
199    
200     # Instantiate WorkSpace
201 fanzago 1.49 common.work_space = WorkSpace(continue_dir, self.cfg_params)
202 nsmirnov 1.1
203     return
204    
205 nsmirnov 1.3 def processIniFile_(self, opts):
206 nsmirnov 1.1 """
207     Processes a configuration INI-file.
208     """
209    
210     # Extract cfg-file name from the cmd-line options.
211    
212     for opt in opts.keys():
213     if ( opt == '-cfg' ):
214     if self.flag_continue:
215     raise CrabException('-continue and -cfg cannot coexist.')
216     if opts[opt] : self.cfg_fname = opts[opt]
217 slacapra 1.144 else : processHelpOptions()
218 nsmirnov 1.1 pass
219    
220     elif ( opt == '-name' ):
221     self.name = opts[opt]
222     pass
223    
224     pass
225    
226     # Set default cfg-fname
227    
228     if self.cfg_fname == None:
229     if self.flag_continue:
230     self.cfg_fname = common.work_space.cfgFileName()
231     else:
232     self.cfg_fname = common.prog_name+'.cfg'
233     pass
234     pass
235    
236     # Load cfg-file
237    
238     if string.lower(self.cfg_fname) != 'none':
239     if os.path.exists(self.cfg_fname):
240     self.cfg_params = loadConfig(self.cfg_fname)
241 corvo 1.64 self.cfg_params['user'] = os.environ['USER']
242 nsmirnov 1.1 pass
243     else:
244     msg = 'cfg-file '+self.cfg_fname+' not found.'
245     raise CrabException(msg)
246     pass
247     pass
248    
249     # process the [CRAB] section
250    
251     lhp = len('CRAB.')
252     for k in self.cfg_params.keys():
253     if len(k) >= lhp and k[:lhp] == 'CRAB.':
254     opt = '-'+k[lhp:]
255     if len(opt) >= 3 and opt[:3] == '-__': continue
256     if opt not in opts.keys():
257     opts[opt] = self.cfg_params[k]
258     pass
259     pass
260     pass
261    
262     return
263    
264 nsmirnov 1.3 def processOptions_(self, opts):
265 nsmirnov 1.1 """
266     Processes the command-line options.
267     """
268    
269     for opt in opts.keys():
270     val = opts[opt]
271    
272 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
273     if opt in self.main_actions:
274     self.cfg_params['CRAB.'+opt[1:]] = val
275     continue
276     if opt in self.aux_actions:
277     self.cfg_params['CRAB.'+opt[1:]] = val
278     continue
279 nsmirnov 1.1
280 spiga 1.107 elif ( opt == '-server_name' ):
281 farinafa 1.154 # TODO
282 spiga 1.107 pass
283 nsmirnov 1.1
284     elif ( opt == '-cfg' ):
285     pass
286    
287     elif ( opt in ('-continue', '-c') ):
288 nsmirnov 1.4 # Already processed in processContinueOption_()
289 nsmirnov 1.1 pass
290    
291     elif ( opt == '-jobtype' ):
292     if val : self.job_type_name = string.upper(val)
293 slacapra 1.144 else : processHelpOptions()
294 nsmirnov 1.1 pass
295    
296     elif ( opt == '-Q' ):
297     self.flag_quiet = 1
298     pass
299    
300     elif ( opt == '-debug' ):
301 slacapra 1.6 if val: self.debug_level = int(val)
302     else: self.debug_level = 1
303 nsmirnov 1.1 pass
304    
305     elif ( opt == '-scheduler' ):
306     pass
307 slacapra 1.22
308 nsmirnov 1.3 elif string.find(opt,'.') == -1:
309     print common.prog_name+'. Unrecognized option '+opt
310 slacapra 1.144 processHelpOptions()
311 nsmirnov 1.3 pass
312 nsmirnov 1.1
313 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
314     if string.find(opt,'.') == -1 :
315     self.cfg_params['CRAB.'+opt[1:]] = val
316 nsmirnov 1.1 pass
317 nsmirnov 1.3 else:
318 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
319     self.cfg_params[opt[1:]] = val
320     pass
321     pass
322     return
323    
324 slacapra 1.8 def parseRange_(self, aRange):
325 nsmirnov 1.4 """
326 slacapra 1.8 Takes as the input a string with a range defined in any of the following
327     way, including combination, and return a tuple with the ints defined
328     according to following table. A consistency check is done.
329     NB: the first job is "1", not "0".
330     'all' -> [1,2,..., NJobs]
331     '' -> [1,2,..., NJobs]
332     'n1' -> [n1]
333     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
334     'n1,n2' -> [n1, n2]
335     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
336     """
337     result = []
338 farinafa 1.128
339 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
340     if aRange=='all' or aRange==None or aRange=='':
341 spiga 1.151 result=range(1,common._db.nJobs()+1)## new funct. BL--DS
342 slacapra 1.8 return result
343 slacapra 1.9 elif aRange=='0':
344     return result
345 slacapra 1.8
346 farinafa 1.128 subRanges = str(aRange).split(',') # DEPRECATED # Fabio #string.split(aRange, ',')
347 slacapra 1.8 for subRange in subRanges:
348     result = result+self.parseSimpleRange_(subRange)
349    
350     if self.checkUniqueness_(result):
351     return result
352     else:
353 mcinquil 1.132 common.logger.message( "Error " +str(result) )
354 slacapra 1.8 return []
355    
356     def checkUniqueness_(self, list):
357     """
358 slacapra 1.9 check if a list contains only unique elements
359 slacapra 1.8 """
360    
361     uniqueList = []
362     # use a list comprehension statement (takes a while to understand)
363    
364     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
365    
366     return (len(list)==len(uniqueList))
367    
368 spiga 1.107
369 slacapra 1.8 def parseSimpleRange_(self, aRange):
370     """
371     Takes as the input a string with two integers separated by
372     the minus sign and returns the tuple with these numbers:
373     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
374     'n1' -> [n1]
375     """
376     (start, end) = (None, None)
377    
378     result = []
379 farinafa 1.128 minus = str(aRange).find('-') #DEPRECATED #Fabio #string.find(aRange, '-')
380 slacapra 1.8 if ( minus < 0 ):
381 farinafa 1.128 if int(aRange)>0:
382 corvo 1.97 # FEDE
383     #result.append(int(aRange)-1)
384     ###
385 slacapra 1.62 result.append(int(aRange))
386 slacapra 1.8 else:
387 spiga 1.47 common.logger.message("parseSimpleRange_ ERROR "+aRange)
388 slacapra 1.144 processHelpOptions()
389 spiga 1.47 pass
390    
391 nsmirnov 1.4 pass
392     else:
393 farinafa 1.128 (start, end) = str(aRange).split('-')
394 slacapra 1.8 if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
395 corvo 1.97 #result=range(int(start)-1, int(end))
396     result=range(int(start), int(end)+1) #Daniele
397 slacapra 1.8 else:
398 slacapra 1.33 common.logger.message("parseSimpleRange_ ERROR "+start+end)
399 slacapra 1.8
400     return result
401 nsmirnov 1.4
402 nsmirnov 1.3 def initializeActions_(self, opts):
403 nsmirnov 1.1 """
404     For each user action instantiate a corresponding
405     object and put it in the action dictionary.
406     """
407 spiga 1.15
408     for opt in opts.keys():
409    
410 nsmirnov 1.1 val = opts[opt]
411 spiga 1.15
412    
413     if ( opt == '-create' ):
414 gutsche 1.83 if val and val != 'all':
415     msg = 'Per default, CRAB will create all jobs as specified in the crab.cfg file, not the command line!'
416     common.logger.message(msg)
417     msg = 'Submission will still take into account the number of jobs specified on the command line!\n'
418     common.logger.message(msg)
419 slacapra 1.82 ncjobs = 'all'
420 farinafa 1.138
421 slacapra 1.144 from Creator import Creator
422 slacapra 1.82 # Instantiate Creator object
423     self.creator = Creator(self.job_type_name,
424     self.cfg_params,
425     ncjobs)
426     self.actions[opt] = self.creator
427    
428 spiga 1.151 ### ToBeRempoved BL--DS
429     # # ############################
430     # # environmet swapping
431     # # N.B. CRAB is expected to works coherently with python version and lib
432     # # used by the UI, and it is able to manage the user env. re-organizing
433     # # the paths according to its needed.
434     #
435     # # Since the job creation require the CMSSW python API usage
436     # # which need the python2.4 (shipped with CMSSW), at the moment,
437     # # and just for this functionality, if creation and submition are required together
438     # # CRAB prioritarize the CMSSW stuff first and then re-set the environment. DS.
439     # # ############################
440     # if '-submit' in opts:
441     # os.putenv('PATH', os.environ['AUX_SUBM_PATH'])
442     # os.putenv('PYTHONPATH', os.environ['AUX_SUBM_PY'])
443     # #
444     # os.environ['PATH'] = os.environ['AUX_SUBM_PATH']
445     # os.environ['PYTHONPATH'] = os.environ['AUX_SUBM_PY']
446     # #
447     # dropOutPy23dynLoads()
448     # pass
449 spiga 1.139 # ############################
450 farinafa 1.138
451 slacapra 1.82 # Initialize the JobDB object if needed
452     if not self.flag_continue:
453 spiga 1.151 common._db.createJobs_(self.creator.nJobs()) ## new BL--DS
454 nsmirnov 1.1 pass
455 nsmirnov 1.3
456 slacapra 1.82 # Create and initialize JobList
457 spiga 1.151 common.job_list = JobList(common._db.nJobs(), ## new BL--DS
458 slacapra 1.82 self.creator.jobType())
459 spiga 1.151 ### ToBeRemoved BL--DS
460     # common.taskDB.setDict('ScriptName',common.work_space.jobDir()+"/"+self.job_type_name+'.sh')
461     # common.taskDB.setDict('JdlName',common.work_space.jobDir()+"/"+self.job_type_name+'.jdl')
462     # common.taskDB.setDict('CfgName',common.work_space.jobDir()+"/"+self.creator.jobType().configFilename())
463    
464     taskinfo={} ## new BL--DS
465     taskinfo['scriptName'] = common.work_space.jobDir()+"/"+self.job_type_name+'.sh' ## new BL--DS
466     taskinfo['cfgName'] = common.work_space.jobDir()+"/"+self.creator.jobType().configFilename() ## new BL--DS
467 nsmirnov 1.3
468 spiga 1.151
469    
470    
471 slacapra 1.82 common.job_list.setScriptNames(self.job_type_name+'.sh')
472 spiga 1.151 ## Obsolete
473     ## common.job_list.setJDLNames(self.job_type_name+'.jdl')
474 slacapra 1.82 common.job_list.setCfgNames(self.creator.jobType().configFilename())
475 slacapra 1.9
476 slacapra 1.82 self.creator.writeJobsSpecsToDB()
477 spiga 1.151 common._db.updateTask_(taskinfo) ## New BL--DS
478 nsmirnov 1.3 pass
479 nsmirnov 1.1
480     elif ( opt == '-submit' ):
481 slacapra 1.150 ## Dealt with val == int so that -submit N means submit N jobs and not job # N
482 spiga 1.107 # modified to support server mode
483     if (self.UseServer== 1):
484 mcinquil 1.119 from SubmitterServer import SubmitterServer
485 farinafa 1.125 self.actions[opt] = SubmitterServer(self.cfg_params, self.parseRange_(val), val)
486 spiga 1.107 else:
487 slacapra 1.146 from Submitter import Submitter
488     # Instantiate Submitter object
489     self.actions[opt] = Submitter(self.cfg_params, self.parseRange_(val), val)
490     # Create and initialize JobList
491     if len(common.job_list) == 0 :
492 spiga 1.151 common.job_list = JobList(common._db.nJobs(), ## New BL--DS
493 slacapra 1.146 None)
494 spiga 1.151 # common.job_list.setJDLNames(self.job_type_name+'.jdl')
495 slacapra 1.23 pass
496 nsmirnov 1.1 pass
497    
498 nsmirnov 1.4 elif ( opt == '-list' ):
499 spiga 1.151 '''
500     Print the relevant infos of a range-all jobs/task
501     '''
502 slacapra 1.8 jobs = self.parseRange_(val)
503    
504 spiga 1.151 common._db.dump(jobs) ## New BL--DS
505 nsmirnov 1.4 pass
506    
507 slacapra 1.67 elif ( opt == '-printId' ):
508 spiga 1.151 '''
509     Print the unique name of the task if crab is used as client
510     Print the SID list of all the jobs
511     '''
512 spiga 1.115 # modified to support server mode
513 spiga 1.151 common._db.printId(self.UseServer) ## New BL--DS
514 slacapra 1.67
515 nsmirnov 1.4 elif ( opt == '-status' ):
516 spiga 1.153 from Status import Status
517 spiga 1.107 # modified to support server mode
518     if (self.UseServer== 1):
519 mcinquil 1.119 from StatusServer import StatusServer
520 spiga 1.107 self.actions[opt] = StatusServer(self.cfg_params)
521     else:
522     jobs = self.parseRange_(val)
523    
524     if len(jobs) != 0:
525 spiga 1.153 self.actions[opt] = Status(self.cfg_params)
526 spiga 1.107 pass
527 slacapra 1.22
528 nsmirnov 1.4 elif ( opt == '-kill' ):
529 slacapra 1.8
530 slacapra 1.148 if val:
531     if val =='all':
532 spiga 1.151 ## this should be NOT scheduler dependent... to move from Scheduler to _db BL--DS
533 slacapra 1.148 jobs = common.scheduler.list()
534     else:
535     jobs = self.parseRange_(val)
536     pass
537     else:
538     raise CrabException("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
539     pass
540    
541 corvo 1.109 if (self.UseServer== 1):
542 slacapra 1.148 from KillerServer import KillerServer
543     self.actions[opt] = KillerServer(self.cfg_params,val, self.parseRange_(val)) #Fabio
544 corvo 1.109 else:
545 slacapra 1.148 common.scheduler.cancel(jobs)
546 nsmirnov 1.1
547 farinafa 1.120
548 slacapra 1.70 elif ( opt == '-getoutput' or opt == '-get'):
549 spiga 1.107 # modified to support server mode
550     if (self.UseServer== 1):
551 mcinquil 1.119 from GetOutputServer import GetOutputServer
552 spiga 1.107 self.actions[opt] = GetOutputServer(self.cfg_params)
553 spiga 1.20 else:
554 spiga 1.107 if val=='all' or val==None or val=='':
555 spiga 1.151 ## this should be NOT scheduler dependent... to move from Scheduler to _db BL--DS
556 slacapra 1.147 jobs = common.scheduler.list()
557 spiga 1.107 else:
558     jobs = self.parseRange_(val)
559    
560     jobs_done = []
561     for nj in jobs:
562     jobs_done.append(nj)
563     common.scheduler.getOutput(jobs_done)
564     pass
565 nsmirnov 1.4
566     elif ( opt == '-resubmit' ):
567 slacapra 1.11 if val:
568 slacapra 1.150 if val=='all':
569     jobs = common.scheduler.list()
570     else:
571     jobs = self.parseRange_(val)
572 slacapra 1.8
573 slacapra 1.150 # Instantiate Submitter object
574     from Resubmitter import Resubmitter
575     self.actions[opt] = Resubmitter(self.cfg_params, jobs, self.UseServer)
576 corvo 1.93
577 slacapra 1.150 # if len(common.job_list) == 0 :
578     # common.job_list = JobList(common.jobDB.nJobs(),None)
579     # common.job_list.setJDLNames(self.job_type_name+'.jdl')
580     # pass
581 slacapra 1.8 pass
582 slacapra 1.11 else:
583     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
584 spiga 1.60 common.logger.message("WARNING: _all_ job specified in the range will be resubmitted!!!")
585 slacapra 1.11 pass
586 slacapra 1.8 pass
587 gutsche 1.61
588 slacapra 1.8 elif ( opt == '-cancelAndResubmit' ):
589 nsmirnov 1.5
590 slacapra 1.78 if val:
591     if val =='all':
592 slacapra 1.147 jobs = common.scheduler.list()
593 spiga 1.47 else:
594 slacapra 1.78 jobs = self.parseRange_(val)
595     # kill submitted jobs
596     common.scheduler.cancel(jobs)
597 spiga 1.47 else:
598 slacapra 1.78 common.logger.message("Warning: with '-cancelAndResubmit' you _MUST_ specify a job range or 'all'")
599 nsmirnov 1.5
600 spiga 1.47 # resubmit cancelled jobs.
601     if val:
602     nj_list = []
603     for nj in jobs:
604     st = common.jobDB.status(int(nj)-1)
605     if st in ['K','A']:
606 corvo 1.97 nj_list.append(int(nj)-1)
607     common.jobDB.setStatus(int(nj)-1,'C')
608 spiga 1.47 elif st == 'Y':
609     common.scheduler.moveOutput(nj)
610 corvo 1.97 nj_list.append(int(nj)-1)
611     st = common.jobDB.setStatus(int(nj)-1,'RC')
612 spiga 1.47 elif st in ['C','X']:
613 corvo 1.97 common.logger.message('Job #'+`int(nj)`+' has status '+crabJobStatusToString(st)+' not yet submitted!!!')
614 spiga 1.47 pass
615     elif st == 'D':
616 corvo 1.97 common.logger.message('Job #'+`int(nj)`+' has status '+crabJobStatusToString(st)+' must be retrieved before resubmission')
617 spiga 1.47 else:
618     common.logger.message('Job #'+`nj`+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
619     pass
620    
621 nsmirnov 1.5 if len(common.job_list) == 0 :
622 spiga 1.47 common.job_list = JobList(common.jobDB.nJobs(),None)
623 nsmirnov 1.5 common.job_list.setJDLNames(self.job_type_name+'.jdl')
624     pass
625 spiga 1.47
626     if len(nj_list) != 0:
627 spiga 1.86 # common.scheduler.resubmit(nj_list)
628 fanzago 1.58 self.actions[opt] = Submitter(self.cfg_params, nj_list)
629 spiga 1.47 pass
630     pass
631     else:
632     common.logger.message("WARNING: _all_ job specified in the rage will be cancelled and resubmitted!!!")
633 nsmirnov 1.5 pass
634 spiga 1.47 common.jobDB.save()
635 nsmirnov 1.4 pass
636 slacapra 1.28 elif ( opt == '-testJdl' ):
637 slacapra 1.8 jobs = self.parseRange_(val)
638     nj_list = []
639     for nj in jobs:
640 slacapra 1.62 st = common.jobDB.status(nj-1)
641 fanzago 1.129 if st != 'X': nj_list.append(nj-1)
642 slacapra 1.8 pass
643    
644     if len(nj_list) != 0:
645 slacapra 1.77 # Instantiate Checker object
646 slacapra 1.144 from Checker import Checker
647 slacapra 1.8 self.actions[opt] = Checker(self.cfg_params, nj_list)
648    
649     # Create and initialize JobList
650    
651     if len(common.job_list) == 0 :
652     common.job_list = JobList(common.jobDB.nJobs(), None)
653     common.job_list.setJDLNames(self.job_type_name+'.jdl')
654     pass
655     pass
656    
657 slacapra 1.9 elif ( opt == '-postMortem' ):
658 corvo 1.95
659 spiga 1.108 # modified to support server mode
660     if (self.UseServer== 1):
661 mcinquil 1.119 from PostMortemServer import PostMortemServer
662 spiga 1.108 self.actions[opt] = PostMortemServer(self.cfg_params)
663     else:
664     if val:
665     val = string.replace(val,'-',':')
666     else: val=''
667     nj_list = {}
668 corvo 1.97
669 spiga 1.108 try:
670 slacapra 1.147 from BossSession import ALL
671     common.scheduler.boss().task().query(ALL, val)
672 spiga 1.108 except RuntimeError,e:
673     common.logger.message( e.__str__() )
674     except ValueError,e:
675     common.logger.message("Warning : Scheduler interaction in query operation failed for jobs:")
676     common.logger.message( e.what())
677     pass
678 corvo 1.93
679 slacapra 1.148 task = common.scheduler.boss().task().jobsDict()
680 spiga 1.108
681     for c, v in task.iteritems():
682     k = int(c)
683     nj=k
684     if v['SCHED_ID']: nj_list[v['CHAIN_ID']]=v['SCHED_ID']
685     pass
686 slacapra 1.9
687 spiga 1.108 if len(nj_list) != 0:
688 corvo 1.93 # Instantiate PostMortem object
689 slacapra 1.144 from PostMortem import PostMortem
690 spiga 1.108 self.actions[opt] = PostMortem(self.cfg_params, nj_list)
691 slacapra 1.9 # Create and initialize JobList
692 spiga 1.108 if len(common.job_list) == 0 :
693     common.job_list = JobList(common.jobDB.nJobs(), None)
694     common.job_list.setJDLNames(self.job_type_name+'.jdl')
695     pass
696 slacapra 1.9 pass
697 spiga 1.108 else:
698     common.logger.message("No jobs to analyze")
699 slacapra 1.9
700     elif ( opt == '-clean' ):
701     if val != None:
702     raise CrabException("No range allowed for '-clean'")
703 mcinquil 1.118 if (self.UseServer== 1):
704 mcinquil 1.119 from CleanerServer import CleanerServer
705 mcinquil 1.118 self.actions[opt] = CleanerServer(self.cfg_params)
706     else:
707 slacapra 1.144 from Cleaner import Cleaner
708 mcinquil 1.118 self.actions[opt] = Cleaner(self.cfg_params)
709    
710 fanzago 1.111 ### FEDE DBS/DLS OUTPUT PUBLICATION
711     elif ( opt == '-publish' ):
712 slacapra 1.141 from Publisher import Publisher
713 slacapra 1.116 self.actions[opt] = Publisher(self.cfg_params)
714    
715 nsmirnov 1.1 pass
716     return
717    
718 nsmirnov 1.3 def createWorkingSpace_(self):
719 slacapra 1.9 new_dir = ''
720    
721 slacapra 1.144 if self.cfg_params.has_key('USER.ui_working_dir') :
722 slacapra 1.9 new_dir = self.cfg_params['USER.ui_working_dir']
723 slacapra 1.144 if not new_dir[1] == '/':
724 slacapra 1.122 new_dir = self.cwd + new_dir
725 slacapra 1.9 pass
726 slacapra 1.144 else:
727     new_dir = self.cwd + common.prog_name + '_' + self.name + '_' + self.current_time
728     pass
729     if os.path.exists(new_dir):
730     if os.listdir(new_dir):
731     msg = new_dir + ' already exists and is not empty. Please remove it before create new task'
732     raise CrabException(msg)
733    
734 fanzago 1.49 common.work_space = WorkSpace(new_dir, self.cfg_params)
735 nsmirnov 1.1 common.work_space.create()
736 slacapra 1.144
737 nsmirnov 1.1 return
738    
739 nsmirnov 1.3 def loadConfiguration_(self, opts):
740 nsmirnov 1.1
741     save_opts = common.work_space.loadSavedOptions()
742    
743     # Override saved options with new command-line options
744    
745     for k in opts.keys():
746     save_opts[k] = opts[k]
747     pass
748    
749     # Return updated options
750     return save_opts
751    
752 nsmirnov 1.3 def createLogger_(self, args):
753 nsmirnov 1.1
754     log = Logger()
755     log.quiet(self.flag_quiet)
756     log.setDebugLevel(self.debug_level)
757     log.write(args+'\n')
758 nsmirnov 1.3 log.message(self.headerString_())
759 nsmirnov 1.1 log.flush()
760     common.logger = log
761     return
762    
763 nsmirnov 1.3 def updateHistory_(self, args):
764 nsmirnov 1.1 history_fname = common.prog_name+'.history'
765     history_file = open(history_fname, 'a')
766     history_file.write(self.current_time+': '+args+'\n')
767     history_file.close()
768     return
769    
770 nsmirnov 1.3 def headerString_(self):
771 nsmirnov 1.1 """
772     Creates a string describing program options either given in
773     the command line or their default values.
774     """
775     header = common.prog_name + ' (version ' + common.prog_version_str + \
776     ') running on ' + \
777     time.ctime(time.time())+'\n\n' + \
778     common.prog_name+'. Working options:\n'
779 fanzago 1.59 #print self.job_type_name
780 nsmirnov 1.1 header = header +\
781 slacapra 1.85 ' scheduler ' + self.cfg_params['CRAB.scheduler'] + '\n'+\
782 nsmirnov 1.1 ' job type ' + self.job_type_name + '\n'+\
783     ' working directory ' + common.work_space.topDir()\
784     + '\n'
785     return header
786    
787 nsmirnov 1.3 def createScheduler_(self):
788 nsmirnov 1.1 """
789     Creates a scheduler object instantiated by its name.
790     """
791 slacapra 1.147 if not self.cfg_params.has_key("CRAB.scheduler"):
792     msg = 'No real scheduler selected: edg, lsf ...'
793     msg = msg + 'Please specify a scheduler type in the crab cfg file'
794     raise CrabException(msg)
795     self.scheduler_name = self.cfg_params["CRAB.scheduler"]
796    
797     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
798 nsmirnov 1.1 file_name = klass_name
799     try:
800     klass = importName(file_name, klass_name)
801     except KeyError:
802     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
803     raise CrabException(msg)
804     except ImportError, e:
805 slacapra 1.101 msg = 'Cannot create scheduler Boss'
806 nsmirnov 1.1 msg += ' (file: '+file_name+', class '+klass_name+'):\n'
807     msg += str(e)
808     raise CrabException(msg)
809    
810     common.scheduler = klass()
811     common.scheduler.configure(self.cfg_params)
812     return
813    
814     def run(self):
815     """
816     For each
817     """
818    
819     for act in self.main_actions:
820     if act in self.actions.keys(): self.actions[act].run()
821     pass
822    
823     for act in self.aux_actions:
824     if act in self.actions.keys(): self.actions[act].run()
825     pass
826     return
827    
828     ###########################################################################
829 slacapra 1.144 def processHelpOptions(opts={}):
830     from crab_help import usage, help
831 nsmirnov 1.1
832 slacapra 1.11 if len(opts):
833     for opt in opts.keys():
834     if opt in ('-v', '-version', '--version') :
835 slacapra 1.144 print 'CRAB version: ',Crab.version()
836 slacapra 1.11 return 1
837     if opt in ('-h','-help','--help') :
838     if opts[opt] : help(opts[opt])
839     else: help()
840     return 1
841     else:
842     usage()
843 corvo 1.93
844 nsmirnov 1.1 return 0
845    
846 corvo 1.93 ###########################################################################
847 nsmirnov 1.1 if __name__ == '__main__':
848 slacapra 1.92 ## Get rid of some useless warning
849 slacapra 1.94 try:
850     import warnings
851     warnings.simplefilter("ignore", RuntimeWarning)
852     except:
853     pass # too bad, you'll get the warning
854 spiga 1.137
855 nsmirnov 1.1 # Parse command-line options and create a dictionary with
856     # key-value pairs.
857     options = parseOptions(sys.argv[1:])
858    
859     # Process "help" options, such as '-help', '-version'
860 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
861 nsmirnov 1.1
862     # Create, initialize, and run a Crab object
863     try:
864 nsmirnov 1.3 crab = Crab(options)
865 nsmirnov 1.1 crab.run()
866 slacapra 1.144 common.apmon.free()
867 nsmirnov 1.1 except CrabException, e:
868     print '\n' + common.prog_name + ': ' + str(e) + '\n'
869     if common.logger:
870     common.logger.write('ERROR: '+str(e)+'\n')
871     pass
872     pass
873    
874     pass