ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.38
Committed: Wed Dec 14 16:16:37 2005 UTC (19 years, 4 months ago) by spiga
Content type: text/x-python
Branch: MAIN
Changes since 1.37: +2 -1 lines
Log Message: 
resubmit

File Contents

# User Rev Content
1 slacapra 1.27 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 slacapra 1.36 from Cleaner import Cleaner
16 nsmirnov 1.1 import common
17 spiga 1.13 import Statistic
18 nsmirnov 1.1
19     import sys, os, time, string
20    
21     ###########################################################################
22     class Crab:
23 nsmirnov 1.3 def __init__(self, opts):
24 nsmirnov 1.1
25     # The order of main_actions is important !
26 slacapra 1.28 self.main_actions = [ '-create', '-submit' ]
27 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
28 slacapra 1.28 '-resubmit' , '-cancelAndResubmit', '-testJdl', '-postMortem', '-clean']
29 nsmirnov 1.1
30     # Dictionary of actions, e.g. '-create' -> object of class Creator
31     self.actions = {}
32    
33     # Configuration file
34     self.cfg_fname = None
35     # Dictionary with configuration parameters
36     self.cfg_params = {}
37    
38     # Current working directory
39     self.cwd = os.getcwd()+'/'
40     # Current time in format 'yymmdd_hhmmss'
41     self.current_time = time.strftime('%y%m%d_%H%M%S',
42     time.localtime(time.time()))
43    
44 nsmirnov 1.3 # Session name (?) Do we need this ?
45 nsmirnov 1.1 self.name = '0'
46    
47     # Job type
48     self.job_type_name = None
49    
50     # Continuation flag
51     self.flag_continue = 0
52    
53     # quiet mode, i.e. no output on screen
54     self.flag_quiet = 0
55     # produce more output
56     self.debug_level = 0
57    
58 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
59 fanzago 1.14 self.scheduler_name = ''
60 nsmirnov 1.1
61 nsmirnov 1.3 self.initialize_(opts)
62 nsmirnov 1.1
63     return
64    
65 nsmirnov 1.7 def version():
66 nsmirnov 1.1 return common.prog_version_str
67    
68 nsmirnov 1.7 version = staticmethod(version)
69    
70 nsmirnov 1.3 def initialize_(self, opts):
71 nsmirnov 1.1
72     # Process the '-continue' option first because
73     # in the case of continuation the CRAB configuration
74     # parameters are loaded from already existing Working Space.
75 nsmirnov 1.3 self.processContinueOption_(opts)
76 nsmirnov 1.1
77     # Process ini-file first, then command line options
78     # because they override ini-file settings.
79    
80 nsmirnov 1.3 self.processIniFile_(opts)
81 nsmirnov 1.1
82 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
83 nsmirnov 1.1
84 nsmirnov 1.3 self.processOptions_(opts)
85 nsmirnov 1.1
86     if not self.flag_continue:
87 nsmirnov 1.3 self.createWorkingSpace_()
88 slacapra 1.9 optsToBeSaved={}
89     for it in opts.keys():
90     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
91     pass
92     else:
93     optsToBeSaved[it]=opts[it]
94     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
95 nsmirnov 1.1 pass
96    
97     # At this point all configuration options have been read.
98    
99     args = string.join(sys.argv,' ')
100 slacapra 1.11
101 nsmirnov 1.3 self.updateHistory_(args)
102 slacapra 1.11
103 nsmirnov 1.3 self.createLogger_(args)
104 slacapra 1.11
105 nsmirnov 1.1 common.jobDB = JobDB()
106 slacapra 1.11
107 nsmirnov 1.3 if self.flag_continue:
108 slacapra 1.12 try:
109     common.jobDB.load()
110     common.logger.debug(6, str(common.jobDB))
111     except DBException,e:
112     pass
113 nsmirnov 1.3 pass
114 slacapra 1.11
115 nsmirnov 1.3 self.createScheduler_()
116 slacapra 1.11
117 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
118     common.logger.debug(6, 'Used properties:')
119     keys = self.cfg_params.keys()
120     keys.sort()
121     for k in keys:
122     if self.cfg_params[k]:
123     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
124     pass
125     else:
126     common.logger.debug(6, ' '+k+' : ')
127     pass
128     pass
129     common.logger.debug(6, 'End of used properties.\n')
130     pass
131     self.initializeActions_(opts)
132 nsmirnov 1.1 return
133    
134 nsmirnov 1.3 def processContinueOption_(self,opts):
135 nsmirnov 1.1
136     continue_dir = None
137 nsmirnov 1.4
138     # Look for the '-continue' option.
139    
140 nsmirnov 1.1 for opt in opts.keys():
141     if ( opt in ('-continue','-c') ):
142     self.flag_continue = 1
143     val = opts[opt]
144     if val:
145     if val[0] == '/': continue_dir = val # abs path
146     else: continue_dir = self.cwd + val # rel path
147     pass
148 nsmirnov 1.4 break
149     pass
150    
151     # Look for actions which has sense only with '-continue'
152    
153     if not self.flag_continue:
154     for opt in opts.keys():
155 slacapra 1.6 if ( opt in (self.aux_actions) ):
156 nsmirnov 1.4 self.flag_continue = 1
157     break
158 nsmirnov 1.1 pass
159     pass
160 slacapra 1.6 submit_flag=0
161     create_flag=0
162     for opt in opts.keys():
163     if opt == "-submit": submit_flag=1
164     if opt == "-create": create_flag=1
165     pass
166     if (submit_flag and not create_flag):
167 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
168     raise CrabException(msg)
169 slacapra 1.6 pass
170 nsmirnov 1.1
171     if not self.flag_continue: return
172    
173     if not continue_dir:
174     prefix = common.prog_name + '_' + self.name + '_'
175     continue_dir = findLastWorkDir(prefix)
176     pass
177    
178     if not continue_dir:
179     raise CrabException('Cannot find last working directory.')
180    
181     if not os.path.exists(continue_dir):
182     msg = 'Cannot continue because the working directory <'
183     msg += continue_dir
184     msg += '> does not exist.'
185     raise CrabException(msg)
186    
187     # Instantiate WorkSpace
188     common.work_space = WorkSpace(continue_dir)
189    
190     return
191    
192 nsmirnov 1.3 def processIniFile_(self, opts):
193 nsmirnov 1.1 """
194     Processes a configuration INI-file.
195     """
196    
197     # Extract cfg-file name from the cmd-line options.
198    
199     for opt in opts.keys():
200     if ( opt == '-cfg' ):
201     if self.flag_continue:
202     raise CrabException('-continue and -cfg cannot coexist.')
203     if opts[opt] : self.cfg_fname = opts[opt]
204     else : usage()
205     pass
206    
207     elif ( opt == '-name' ):
208     self.name = opts[opt]
209     pass
210    
211     pass
212    
213     # Set default cfg-fname
214    
215     if self.cfg_fname == None:
216     if self.flag_continue:
217     self.cfg_fname = common.work_space.cfgFileName()
218     else:
219     self.cfg_fname = common.prog_name+'.cfg'
220     pass
221     pass
222    
223     # Load cfg-file
224    
225     if string.lower(self.cfg_fname) != 'none':
226     if os.path.exists(self.cfg_fname):
227     self.cfg_params = loadConfig(self.cfg_fname)
228     pass
229     else:
230     msg = 'cfg-file '+self.cfg_fname+' not found.'
231     raise CrabException(msg)
232     pass
233     pass
234    
235     # process the [CRAB] section
236    
237     lhp = len('CRAB.')
238     for k in self.cfg_params.keys():
239     if len(k) >= lhp and k[:lhp] == 'CRAB.':
240     opt = '-'+k[lhp:]
241     if len(opt) >= 3 and opt[:3] == '-__': continue
242     if opt not in opts.keys():
243     opts[opt] = self.cfg_params[k]
244     pass
245     pass
246     pass
247    
248     return
249    
250 nsmirnov 1.3 def processOptions_(self, opts):
251 nsmirnov 1.1 """
252     Processes the command-line options.
253     """
254    
255     for opt in opts.keys():
256     val = opts[opt]
257    
258 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
259     if opt in self.main_actions:
260     self.cfg_params['CRAB.'+opt[1:]] = val
261     continue
262     if opt in self.aux_actions:
263     self.cfg_params['CRAB.'+opt[1:]] = val
264     continue
265 nsmirnov 1.1
266    
267     elif ( opt == '-cfg' ):
268     pass
269    
270     elif ( opt in ('-continue', '-c') ):
271 nsmirnov 1.4 # Already processed in processContinueOption_()
272 nsmirnov 1.1 pass
273    
274     elif ( opt == '-jobtype' ):
275     if val : self.job_type_name = string.upper(val)
276     else : usage()
277     pass
278    
279     elif ( opt == '-Q' ):
280     self.flag_quiet = 1
281     pass
282    
283     elif ( opt == '-debug' ):
284 slacapra 1.6 if val: self.debug_level = int(val)
285     else: self.debug_level = 1
286 nsmirnov 1.1 pass
287    
288     elif ( opt == '-scheduler' ):
289     if val: self.scheduler_name = val
290     else:
291     print common.prog_name+". No value for '-scheduler'."
292     usage()
293     pass
294     pass
295 slacapra 1.22
296 fanzago 1.18 elif ( opt in ('-use_boss', '-useboss') ):
297     if ( val == '1' ):
298     self.scheduler_name = 'boss'
299     pass
300     elif ( val == '0' ):
301     pass
302     else:
303     print common.prog_name+'. Bad flag for -use_boss option:',\
304     val,'Possible values are 0(=No) or 1(=Yes)'
305     usage()
306     pass
307 fanzago 1.14 pass
308    
309 nsmirnov 1.3 elif string.find(opt,'.') == -1:
310     print common.prog_name+'. Unrecognized option '+opt
311     usage()
312     pass
313 nsmirnov 1.1
314 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
315     if string.find(opt,'.') == -1 :
316     self.cfg_params['CRAB.'+opt[1:]] = val
317 nsmirnov 1.1 pass
318 nsmirnov 1.3 else:
319 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
320     self.cfg_params[opt[1:]] = val
321     pass
322     pass
323     return
324    
325 slacapra 1.8 def parseRange_(self, aRange):
326 nsmirnov 1.4 """
327 slacapra 1.8 Takes as the input a string with a range defined in any of the following
328     way, including combination, and return a tuple with the ints defined
329     according to following table. A consistency check is done.
330     NB: the first job is "1", not "0".
331     'all' -> [1,2,..., NJobs]
332     '' -> [1,2,..., NJobs]
333     'n1' -> [n1]
334     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
335     'n1,n2' -> [n1, n2]
336     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
337     """
338     result = []
339    
340 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
341     if aRange=='all' or aRange==None or aRange=='':
342 slacapra 1.8 result=range(0,common.jobDB.nJobs())
343     return result
344 slacapra 1.9 elif aRange=='0':
345     return result
346 slacapra 1.8
347     subRanges = string.split(aRange, ',')
348     for subRange in subRanges:
349     result = result+self.parseSimpleRange_(subRange)
350    
351     if self.checkUniqueness_(result):
352     return result
353     else:
354 slacapra 1.33 common.logger.message("Error "+result)
355 slacapra 1.8 return []
356    
357     def checkUniqueness_(self, list):
358     """
359 slacapra 1.9 check if a list contains only unique elements
360 slacapra 1.8 """
361    
362     uniqueList = []
363     # use a list comprehension statement (takes a while to understand)
364    
365     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
366    
367     return (len(list)==len(uniqueList))
368    
369     def parseSimpleRange_(self, aRange):
370     """
371     Takes as the input a string with two integers separated by
372     the minus sign and returns the tuple with these numbers:
373     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
374     'n1' -> [n1]
375     """
376     (start, end) = (None, None)
377    
378     result = []
379     minus = string.find(aRange, '-')
380     if ( minus < 0 ):
381     if isInt(aRange) and int(aRange)>0:
382 fanzago 1.37 # FEDE
383     #result.append(int(aRange)-1)
384     ###
385     result.append(aRange)
386 slacapra 1.8 else:
387 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
388 nsmirnov 1.4 pass
389     else:
390 slacapra 1.8 (start, end) = string.split(aRange, '-')
391     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
392 spiga 1.38 #result=range(int(start)-1, int(end))
393     result=range(int(start), int(end)+1) #Daniele
394 slacapra 1.8 else:
395 slacapra 1.33 common.logger.message("parseSimpleRange_ ERROR "+start+end)
396 slacapra 1.8
397     return result
398 nsmirnov 1.4
399 nsmirnov 1.3 def initializeActions_(self, opts):
400 nsmirnov 1.1 """
401     For each user action instantiate a corresponding
402     object and put it in the action dictionary.
403     """
404     for opt in opts.keys():
405 spiga 1.32 self.flag_useboss = 0
406 spiga 1.16 if ( opt == '-use_boss'):
407 slacapra 1.25 val = opts[opt]
408     if ( val == '1' ):
409     self.flag_useboss = 1
410     common.logger.message('Using BOSS')
411     pass
412     else:
413     self.flag_useboss = 0
414     pass
415 spiga 1.15
416     for opt in opts.keys():
417    
418 nsmirnov 1.1 val = opts[opt]
419 spiga 1.15
420    
421     if ( opt == '-create' ):
422 slacapra 1.22 ncjobs = 0
423 nsmirnov 1.1 if val:
424     if ( isInt(val) ):
425     ncjobs = int(val)
426     elif ( val == 'all'):
427     ncjobs = val
428     else:
429 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
430     msg += ' Must be an integer or "all"'
431 slacapra 1.8 msg += ' Generic range is not allowed"'
432 nsmirnov 1.5 raise CrabException(msg)
433 nsmirnov 1.1 pass
434     else: ncjobs = 'all'
435    
436     if ncjobs != 0:
437 nsmirnov 1.3 # Instantiate Creator object
438 nsmirnov 1.1 creator = Creator(self.job_type_name,
439     self.cfg_params,
440     ncjobs)
441     self.actions[opt] = creator
442 nsmirnov 1.3
443     # Initialize the JobDB object if needed
444 nsmirnov 1.1 if not self.flag_continue:
445     common.jobDB.create(creator.nJobs())
446     pass
447 nsmirnov 1.3
448     # Create and initialize JobList
449    
450     common.job_list = JobList(common.jobDB.nJobs(),
451     creator.jobType())
452    
453     common.job_list.setScriptNames(self.job_type_name+'.sh')
454     common.job_list.setJDLNames(self.job_type_name+'.jdl')
455 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
456 slacapra 1.9
457     creator.writeJobsSpecsToDB()
458 nsmirnov 1.1 pass
459 nsmirnov 1.3 pass
460 nsmirnov 1.1
461     elif ( opt == '-submit' ):
462 slacapra 1.23
463     # total jobs
464     # get the first not already submitted
465 slacapra 1.24 common.logger.debug(5,'Total jobs '+str(common.jobDB.nJobs()))
466     lastSubmittedJob=0
467     for nj in range(common.jobDB.nJobs()):
468     if (common.jobDB.status(nj)=='S'):
469     lastSubmittedJob +=1
470 slacapra 1.30 else: break
471 slacapra 1.24 # count job from 1
472 slacapra 1.30 totalJobsSubmittable = common.jobDB.nJobs()-lastSubmittedJob
473 slacapra 1.24 common.logger.debug(5,'lastSubmittedJob '+str(lastSubmittedJob))
474     common.logger.debug(5,'totalJobsSubmittable '+str(totalJobsSubmittable))
475 slacapra 1.23
476 slacapra 1.24 nsjobs = lastSubmittedJob+totalJobsSubmittable
477 slacapra 1.23 # get user request
478 slacapra 1.22 if val:
479     if ( isInt(val) ):
480 slacapra 1.24 tmp = int(val)
481     if (tmp >= totalJobsSubmittable):
482     common.logger.message('asking to submit '+str(tmp)+' jobs, but only '+str(totalJobsSubmittable)+' left: submitting those')
483     pass
484     else:
485     nsjobs=lastSubmittedJob+int(val)
486 slacapra 1.23 elif (val=='all'):
487     pass
488 slacapra 1.22 else:
489     msg = 'Bad submission option <'+str(val)+'>\n'
490     msg += ' Must be an integer or "all"'
491     msg += ' Generic range is not allowed"'
492     raise CrabException(msg)
493     pass
494 slacapra 1.24 common.logger.debug(5,'nsjobs '+str(nsjobs))
495 slacapra 1.23
496     # submit N from last submitted job
497 slacapra 1.24 nj_list = range(lastSubmittedJob, nsjobs)
498     common.logger.debug(5,'nj_list '+str(nj_list))
499 nsmirnov 1.5
500     if len(nj_list) != 0:
501 nsmirnov 1.3 # Instantiate Submitter object
502 fanzago 1.37 print "prima di chiamare submitter, nj_list = ", nj_list
503 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
504 nsmirnov 1.3
505     # Create and initialize JobList
506     if len(common.job_list) == 0 :
507     common.job_list = JobList(common.jobDB.nJobs(),
508     None)
509     common.job_list.setJDLNames(self.job_type_name+'.jdl')
510     pass
511 nsmirnov 1.1 pass
512 slacapra 1.30 pass
513 nsmirnov 1.1
514 nsmirnov 1.4 elif ( opt == '-list' ):
515 slacapra 1.8 jobs = self.parseRange_(val)
516    
517     common.jobDB.dump(jobs)
518 nsmirnov 1.4 pass
519    
520     elif ( opt == '-status' ):
521 slacapra 1.8 jobs = self.parseRange_(val)
522    
523 slacapra 1.9 if len(jobs) != 0:
524 spiga 1.16 if ( self.flag_useboss == 1 ):
525 slacapra 1.36 self.actions[opt] = StatusBoss()
526 spiga 1.15 else:
527 slacapra 1.36 self.actions[opt] = Status(jobs)
528 spiga 1.15 pass
529 nsmirnov 1.1 pass
530     pass
531 slacapra 1.22
532 nsmirnov 1.4 elif ( opt == '-kill' ):
533 slacapra 1.8
534 spiga 1.31 if ( self.flag_useboss == 1 ):
535     if val:
536     if val =='all':
537 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
538     jobs = allBoss_id.keys()
539 spiga 1.31 else:
540     jobs = self.parseRange_(val)
541     common.scheduler.cancel(jobs)
542     else:
543     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
544     else:
545     if val:
546     jobs = self.parseRange_(val)
547    
548     for nj in jobs:
549     st = common.jobDB.status(nj)
550     if st == 'S':
551     jid = common.jobDB.jobId(nj)
552     common.logger.message("Killing job # "+`(nj+1)`)
553     common.scheduler.cancel(jid)
554     common.jobDB.setStatus(nj, 'K')
555     pass
556 slacapra 1.9 pass
557 spiga 1.31
558     common.jobDB.save()
559 nsmirnov 1.4 pass
560 spiga 1.31 else:
561     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
562 nsmirnov 1.1
563 slacapra 1.8 elif ( opt == '-getoutput' ):
564    
565 spiga 1.20 if ( self.flag_useboss == 1 ):
566     if val=='all' or val==None or val=='':
567 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
568     jobs = allBoss_id.keys()
569 spiga 1.20 else:
570     jobs = self.parseRange_(val)
571     common.scheduler.getOutput(jobs)
572     else:
573     jobs = self.parseRange_(val)
574 spiga 1.13
575 slacapra 1.22 ## also this: create a ActorClass (GetOutput)
576 spiga 1.20 jobs_done = []
577     for nj in jobs:
578     st = common.jobDB.status(nj)
579     if st == 'D':
580 slacapra 1.12 jobs_done.append(nj)
581 spiga 1.20 pass
582     elif st == 'S':
583     jid = common.jobDB.jobId(nj)
584     currStatus = common.scheduler.queryStatus(jid)
585     if currStatus=="Done":
586     jobs_done.append(nj)
587     else:
588     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
589     common.logger.message(msg)
590     pass
591 slacapra 1.9 else:
592 spiga 1.20 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
593     pass
594 slacapra 1.12 pass
595    
596 spiga 1.20 for nj in jobs_done:
597 spiga 1.15 jid = common.jobDB.jobId(nj)
598     dir = common.scheduler.getOutput(jid)
599     common.jobDB.setStatus(nj, 'Y')
600 slacapra 1.12
601     # Rename the directory with results to smth readable
602     new_dir = common.work_space.resDir()
603 spiga 1.20 try:
604     files = os.listdir(dir)
605     for file in files:
606     os.rename(dir+'/'+file, new_dir+'/'+file)
607     os.rmdir(dir)
608     except OSError, e:
609     msg = 'rename files from '+dir+' to '+new_dir+' error: '
610     msg += str(e)
611     common.logger.message(msg)
612     # ignore error
613 slacapra 1.12 pass
614 spiga 1.20 pass
615 slacapra 1.22 ###
616    
617 spiga 1.13 resFlag = 0
618     exCode = common.scheduler.getExitStatus(jid)
619 spiga 1.34 Statistic.Monitor('retrieved',resFlag,jid,exCode)
620 slacapra 1.12
621     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
622     common.logger.message(msg)
623 nsmirnov 1.4 pass
624    
625     common.jobDB.save()
626     pass
627    
628     elif ( opt == '-resubmit' ):
629 fanzago 1.37 if ( self.flag_useboss == 1 ):
630     if val=='all' or val==None or val=='':
631     allBoss_id = common.scheduler.listBoss()
632     # sono gli internal_id di BOSS
633     jobs = allBoss_id.keys()
634     else:
635     jobs = self.parseRange_(val)
636     else:
637     if val:
638     jobs = self.parseRange_(val)
639    
640 slacapra 1.11 if val:
641     # create a list of jobs to be resubmitted.
642 slacapra 1.8
643 slacapra 1.22 ### as before, create a Resubmittter Class
644 slacapra 1.11 nj_list = []
645     for nj in jobs:
646 fanzago 1.37 st = common.jobDB.status(int(nj)-1)
647     #print "stato del job = ", st
648 slacapra 1.11 if st in ['K','A']:
649 fanzago 1.37 nj_list.append(int(nj)-1)
650     common.jobDB.setStatus(int(nj)-1,'C')
651 slacapra 1.11 elif st == 'Y':
652     # here I would like to move the old output to a new place
653     # I would need the name of the output files.
654     # these infos are in the jobType class, which is not accessible from here!
655 fanzago 1.37
656     # fede...commento tutto ehehehehehehhhhh
657     #outSandbox = common.jobDB.outputSandbox(nj)
658 slacapra 1.11
659 fanzago 1.37 #resDir = common.work_space.resDir()
660     #resDirSave = resDir + self.current_time
661     #os.mkdir(resDirSave)
662 slacapra 1.11
663 fanzago 1.37 #for file in outSandbox:
664     # if os.path.exists(resDir+'/'+file):
665     # os.rename(resDir+'/'+file, resDirSave+'/'+file)
666     # common.logger.message('Output file '+file+' moved to '+resDirSave)
667     #pass
668     nj_list.append(int(nj)-1)
669     st = common.jobDB.setStatus(int(nj)-1,'RC')
670    
671     # scommentare le righe sopra....
672     elif st in ['D','C','X']:
673     # Nel caso DONE bisogna prima ritirare l'output
674     # Nel caso C va prima sottomesso
675     # Nel caso X va prima creato
676 slacapra 1.11 pass
677     else:
678 fanzago 1.37 #common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
679     common.logger.message('Job #'+nj+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
680 slacapra 1.9 pass
681 slacapra 1.8
682 fanzago 1.37 if len(common.job_list) == 0 :
683     #print "common.job_list = ", common.job_list
684     common.job_list = JobList(common.jobDB.nJobs(),None)
685     common.job_list.setJDLNames(self.job_type_name+'.jdl')
686     pass
687    
688 slacapra 1.11 if len(nj_list) != 0:
689 fanzago 1.37 #print "nj_list = ", nj_list
690     common.scheduler.resubmit(nj_list)
691 slacapra 1.11 # Instantiate Submitter object
692     self.actions[opt] = Submitter(self.cfg_params, nj_list)
693    
694     # Create and initialize JobList
695    
696 fanzago 1.37 #if len(common.job_list) == 0 :
697     # common.job_list = JobList(common.jobDB.nJobs(),
698     # None)
699     # common.job_list.setJDLNames(self.job_type_name+'.jdl')
700     # pass
701 slacapra 1.8 pass
702     pass
703 slacapra 1.11 else:
704     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
705     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
706     pass
707 fanzago 1.37 common.jobDB.save()
708 slacapra 1.8 pass
709    
710     elif ( opt == '-cancelAndResubmit' ):
711     jobs = self.parseRange_(val)
712 nsmirnov 1.5
713 nsmirnov 1.7 # Cancel submitted jobs.
714 nsmirnov 1.5
715 slacapra 1.8 nj_list = []
716     for nj in jobs:
717 nsmirnov 1.5 st = common.jobDB.status(nj)
718     if st == 'S':
719     jid = common.jobDB.jobId(nj)
720     common.scheduler.cancel(jid)
721     st = 'K'
722     common.jobDB.setStatus(nj, st)
723     pass
724 nsmirnov 1.7 pass
725 nsmirnov 1.5
726 slacapra 1.8 if st != 'X': nj_list.append(nj)
727     pass
728 nsmirnov 1.5
729     if len(nj_list) != 0:
730     # Instantiate Submitter object
731     self.actions[opt] = Submitter(self.cfg_params, nj_list)
732    
733     # Create and initialize JobList
734    
735     if len(common.job_list) == 0 :
736     common.job_list = JobList(common.jobDB.nJobs(),
737     None)
738     common.job_list.setJDLNames(self.job_type_name+'.jdl')
739     pass
740     pass
741 nsmirnov 1.4 pass
742    
743 slacapra 1.28 elif ( opt == '-testJdl' ):
744 slacapra 1.8 jobs = self.parseRange_(val)
745     nj_list = []
746     for nj in jobs:
747     st = common.jobDB.status(nj)
748 slacapra 1.12 if st == 'C': nj_list.append(nj)
749 slacapra 1.8 pass
750    
751     if len(nj_list) != 0:
752     # Instantiate Submitter object
753     self.actions[opt] = Checker(self.cfg_params, nj_list)
754    
755     # Create and initialize JobList
756    
757     if len(common.job_list) == 0 :
758     common.job_list = JobList(common.jobDB.nJobs(), None)
759     common.job_list.setJDLNames(self.job_type_name+'.jdl')
760     pass
761     pass
762    
763 slacapra 1.9 elif ( opt == '-postMortem' ):
764     jobs = self.parseRange_(val)
765     nj_list = []
766     for nj in jobs:
767     st = common.jobDB.status(nj)
768 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
769 slacapra 1.9 pass
770    
771     if len(nj_list) != 0:
772     # Instantiate Submitter object
773 spiga 1.35 self.actions[opt] = PostMortem(self.cfg_params, nj_list,self.flag_useboss)
774 slacapra 1.9
775     # Create and initialize JobList
776    
777     if len(common.job_list) == 0 :
778     common.job_list = JobList(common.jobDB.nJobs(), None)
779     common.job_list.setJDLNames(self.job_type_name+'.jdl')
780     pass
781     pass
782 slacapra 1.33 else:
783     common.logger.message("No jobs to analyze")
784 slacapra 1.9
785     elif ( opt == '-clean' ):
786     if val != None:
787     raise CrabException("No range allowed for '-clean'")
788    
789 slacapra 1.36 theCleaner = Cleaner(self.scheduler_name == 'boss')
790     theCleaner.clean()
791 slacapra 1.9
792 nsmirnov 1.1 pass
793     return
794    
795 nsmirnov 1.3 def createWorkingSpace_(self):
796 slacapra 1.9 new_dir = ''
797    
798     try:
799     new_dir = self.cfg_params['USER.ui_working_dir']
800     except KeyError:
801     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
802     new_dir = self.cwd + new_dir
803     pass
804    
805 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
806     common.work_space.create()
807     return
808    
809 nsmirnov 1.3 def loadConfiguration_(self, opts):
810 nsmirnov 1.1
811     save_opts = common.work_space.loadSavedOptions()
812    
813     # Override saved options with new command-line options
814    
815     for k in opts.keys():
816     save_opts[k] = opts[k]
817     pass
818    
819     # Return updated options
820     return save_opts
821    
822 nsmirnov 1.3 def createLogger_(self, args):
823 nsmirnov 1.1
824     log = Logger()
825     log.quiet(self.flag_quiet)
826     log.setDebugLevel(self.debug_level)
827     log.write(args+'\n')
828 nsmirnov 1.3 log.message(self.headerString_())
829 nsmirnov 1.1 log.flush()
830     common.logger = log
831     return
832    
833 nsmirnov 1.3 def updateHistory_(self, args):
834 nsmirnov 1.1 history_fname = common.prog_name+'.history'
835     history_file = open(history_fname, 'a')
836     history_file.write(self.current_time+': '+args+'\n')
837     history_file.close()
838     return
839    
840 nsmirnov 1.3 def headerString_(self):
841 nsmirnov 1.1 """
842     Creates a string describing program options either given in
843     the command line or their default values.
844     """
845     header = common.prog_name + ' (version ' + common.prog_version_str + \
846     ') running on ' + \
847     time.ctime(time.time())+'\n\n' + \
848     common.prog_name+'. Working options:\n'
849     header = header +\
850     ' scheduler ' + self.scheduler_name + '\n'+\
851     ' job type ' + self.job_type_name + '\n'+\
852     ' working directory ' + common.work_space.topDir()\
853     + '\n'
854     return header
855    
856 nsmirnov 1.3 def createScheduler_(self):
857 nsmirnov 1.1 """
858     Creates a scheduler object instantiated by its name.
859     """
860     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
861     file_name = klass_name
862     try:
863     klass = importName(file_name, klass_name)
864     except KeyError:
865     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
866     raise CrabException(msg)
867     except ImportError, e:
868     msg = 'Cannot create scheduler '+self.scheduler_name
869     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
870     msg += str(e)
871     raise CrabException(msg)
872    
873     common.scheduler = klass()
874     common.scheduler.configure(self.cfg_params)
875     return
876    
877     def run(self):
878     """
879     For each
880     """
881    
882     for act in self.main_actions:
883     if act in self.actions.keys(): self.actions[act].run()
884     pass
885    
886     for act in self.aux_actions:
887     if act in self.actions.keys(): self.actions[act].run()
888     pass
889     return
890    
891     ###########################################################################
892     def processHelpOptions(opts):
893    
894 slacapra 1.11 if len(opts):
895     for opt in opts.keys():
896     if opt in ('-v', '-version', '--version') :
897     print Crab.version()
898     return 1
899     if opt in ('-h','-help','--help') :
900     if opts[opt] : help(opts[opt])
901     else: help()
902     return 1
903     else:
904     usage()
905 nsmirnov 1.1
906     return 0
907    
908     ###########################################################################
909     if __name__ == '__main__':
910    
911 fanzago 1.18
912 fanzago 1.19 # # Initial settings for Python modules. Avoid appending manually lib paths.
913     # try:
914     # path=os.environ['EDG_WL_LOCATION']
915     # except:
916     # print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
917     # sys.exit(1)
918     #
919     # libPath=os.path.join(path, "lib")
920     # sys.path.append(libPath)
921     # libPath=os.path.join(path, "lib", "python")
922     # sys.path.append(libPath)
923 fanzago 1.18
924    
925 nsmirnov 1.1 # Parse command-line options and create a dictionary with
926     # key-value pairs.
927    
928     options = parseOptions(sys.argv[1:])
929    
930     # Process "help" options, such as '-help', '-version'
931    
932 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
933 nsmirnov 1.1
934     # Create, initialize, and run a Crab object
935    
936     try:
937 nsmirnov 1.3 crab = Crab(options)
938 nsmirnov 1.1 crab.run()
939     except CrabException, e:
940     print '\n' + common.prog_name + ': ' + str(e) + '\n'
941     if common.logger:
942     common.logger.write('ERROR: '+str(e)+'\n')
943     pass
944     pass
945    
946     pass