ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/crab.py
Revision: 1.39
Committed: Thu Dec 15 12:51:53 2005 UTC (19 years, 4 months ago) by fanzago
Content type: text/x-python
Branch: MAIN
Changes since 1.38: +1 -25 lines
Log Message:
new function to move retrieved output

File Contents

# User Rev Content
1 slacapra 1.27 #!/usr/bin/env python2.2
2 nsmirnov 1.1 from crab_help import *
3     from crab_util import *
4     from crab_exceptions import *
5     from crab_logger import Logger
6     from WorkSpace import WorkSpace
7     from JobDB import JobDB
8 nsmirnov 1.3 from JobList import JobList
9 nsmirnov 1.1 from Creator import Creator
10     from Submitter import Submitter
11 slacapra 1.8 from Checker import Checker
12 slacapra 1.9 from PostMortem import PostMortem
13     from Status import Status
14 spiga 1.16 from StatusBoss import StatusBoss
15 slacapra 1.36 from Cleaner import Cleaner
16 nsmirnov 1.1 import common
17 spiga 1.13 import Statistic
18 nsmirnov 1.1
19     import sys, os, time, string
20    
21     ###########################################################################
22     class Crab:
23 nsmirnov 1.3 def __init__(self, opts):
24 nsmirnov 1.1
25     # The order of main_actions is important !
26 slacapra 1.28 self.main_actions = [ '-create', '-submit' ]
27 slacapra 1.8 self.aux_actions = [ '-list', '-kill', '-status', '-getoutput',
28 slacapra 1.28 '-resubmit' , '-cancelAndResubmit', '-testJdl', '-postMortem', '-clean']
29 nsmirnov 1.1
30     # Dictionary of actions, e.g. '-create' -> object of class Creator
31     self.actions = {}
32    
33     # Configuration file
34     self.cfg_fname = None
35     # Dictionary with configuration parameters
36     self.cfg_params = {}
37    
38     # Current working directory
39     self.cwd = os.getcwd()+'/'
40     # Current time in format 'yymmdd_hhmmss'
41     self.current_time = time.strftime('%y%m%d_%H%M%S',
42     time.localtime(time.time()))
43    
44 nsmirnov 1.3 # Session name (?) Do we need this ?
45 nsmirnov 1.1 self.name = '0'
46    
47     # Job type
48     self.job_type_name = None
49    
50     # Continuation flag
51     self.flag_continue = 0
52    
53     # quiet mode, i.e. no output on screen
54     self.flag_quiet = 0
55     # produce more output
56     self.debug_level = 0
57    
58 nsmirnov 1.3 # Scheduler name, e.g. 'edg', 'lsf'
59 fanzago 1.14 self.scheduler_name = ''
60 nsmirnov 1.1
61 nsmirnov 1.3 self.initialize_(opts)
62 nsmirnov 1.1
63     return
64    
65 nsmirnov 1.7 def version():
66 nsmirnov 1.1 return common.prog_version_str
67    
68 nsmirnov 1.7 version = staticmethod(version)
69    
70 nsmirnov 1.3 def initialize_(self, opts):
71 nsmirnov 1.1
72     # Process the '-continue' option first because
73     # in the case of continuation the CRAB configuration
74     # parameters are loaded from already existing Working Space.
75 nsmirnov 1.3 self.processContinueOption_(opts)
76 nsmirnov 1.1
77     # Process ini-file first, then command line options
78     # because they override ini-file settings.
79    
80 nsmirnov 1.3 self.processIniFile_(opts)
81 nsmirnov 1.1
82 nsmirnov 1.3 if self.flag_continue: opts = self.loadConfiguration_(opts)
83 nsmirnov 1.1
84 nsmirnov 1.3 self.processOptions_(opts)
85 nsmirnov 1.1
86     if not self.flag_continue:
87 nsmirnov 1.3 self.createWorkingSpace_()
88 slacapra 1.9 optsToBeSaved={}
89     for it in opts.keys():
90     if (it in self.main_actions) or (it in self.aux_actions) or (it == '-debug'):
91     pass
92     else:
93     optsToBeSaved[it]=opts[it]
94     common.work_space.saveConfiguration(optsToBeSaved, self.cfg_fname)
95 nsmirnov 1.1 pass
96    
97     # At this point all configuration options have been read.
98    
99     args = string.join(sys.argv,' ')
100 slacapra 1.11
101 nsmirnov 1.3 self.updateHistory_(args)
102 slacapra 1.11
103 nsmirnov 1.3 self.createLogger_(args)
104 slacapra 1.11
105 nsmirnov 1.1 common.jobDB = JobDB()
106 slacapra 1.11
107 nsmirnov 1.3 if self.flag_continue:
108 slacapra 1.12 try:
109     common.jobDB.load()
110     common.logger.debug(6, str(common.jobDB))
111     except DBException,e:
112     pass
113 nsmirnov 1.3 pass
114 slacapra 1.11
115 nsmirnov 1.3 self.createScheduler_()
116 slacapra 1.11
117 nsmirnov 1.3 if common.logger.debugLevel() >= 6:
118     common.logger.debug(6, 'Used properties:')
119     keys = self.cfg_params.keys()
120     keys.sort()
121     for k in keys:
122     if self.cfg_params[k]:
123     common.logger.debug(6, ' '+k+' : '+self.cfg_params[k])
124     pass
125     else:
126     common.logger.debug(6, ' '+k+' : ')
127     pass
128     pass
129     common.logger.debug(6, 'End of used properties.\n')
130     pass
131     self.initializeActions_(opts)
132 nsmirnov 1.1 return
133    
134 nsmirnov 1.3 def processContinueOption_(self,opts):
135 nsmirnov 1.1
136     continue_dir = None
137 nsmirnov 1.4
138     # Look for the '-continue' option.
139    
140 nsmirnov 1.1 for opt in opts.keys():
141     if ( opt in ('-continue','-c') ):
142     self.flag_continue = 1
143     val = opts[opt]
144     if val:
145     if val[0] == '/': continue_dir = val # abs path
146     else: continue_dir = self.cwd + val # rel path
147     pass
148 nsmirnov 1.4 break
149     pass
150    
151     # Look for actions which has sense only with '-continue'
152    
153     if not self.flag_continue:
154     for opt in opts.keys():
155 slacapra 1.6 if ( opt in (self.aux_actions) ):
156 nsmirnov 1.4 self.flag_continue = 1
157     break
158 nsmirnov 1.1 pass
159     pass
160 slacapra 1.6 submit_flag=0
161     create_flag=0
162     for opt in opts.keys():
163     if opt == "-submit": submit_flag=1
164     if opt == "-create": create_flag=1
165     pass
166     if (submit_flag and not create_flag):
167 nsmirnov 1.7 msg = "'-submit' must be used with either '-create' or '-continue'."
168     raise CrabException(msg)
169 slacapra 1.6 pass
170 nsmirnov 1.1
171     if not self.flag_continue: return
172    
173     if not continue_dir:
174     prefix = common.prog_name + '_' + self.name + '_'
175     continue_dir = findLastWorkDir(prefix)
176     pass
177    
178     if not continue_dir:
179     raise CrabException('Cannot find last working directory.')
180    
181     if not os.path.exists(continue_dir):
182     msg = 'Cannot continue because the working directory <'
183     msg += continue_dir
184     msg += '> does not exist.'
185     raise CrabException(msg)
186    
187     # Instantiate WorkSpace
188     common.work_space = WorkSpace(continue_dir)
189    
190     return
191    
192 nsmirnov 1.3 def processIniFile_(self, opts):
193 nsmirnov 1.1 """
194     Processes a configuration INI-file.
195     """
196    
197     # Extract cfg-file name from the cmd-line options.
198    
199     for opt in opts.keys():
200     if ( opt == '-cfg' ):
201     if self.flag_continue:
202     raise CrabException('-continue and -cfg cannot coexist.')
203     if opts[opt] : self.cfg_fname = opts[opt]
204     else : usage()
205     pass
206    
207     elif ( opt == '-name' ):
208     self.name = opts[opt]
209     pass
210    
211     pass
212    
213     # Set default cfg-fname
214    
215     if self.cfg_fname == None:
216     if self.flag_continue:
217     self.cfg_fname = common.work_space.cfgFileName()
218     else:
219     self.cfg_fname = common.prog_name+'.cfg'
220     pass
221     pass
222    
223     # Load cfg-file
224    
225     if string.lower(self.cfg_fname) != 'none':
226     if os.path.exists(self.cfg_fname):
227     self.cfg_params = loadConfig(self.cfg_fname)
228     pass
229     else:
230     msg = 'cfg-file '+self.cfg_fname+' not found.'
231     raise CrabException(msg)
232     pass
233     pass
234    
235     # process the [CRAB] section
236    
237     lhp = len('CRAB.')
238     for k in self.cfg_params.keys():
239     if len(k) >= lhp and k[:lhp] == 'CRAB.':
240     opt = '-'+k[lhp:]
241     if len(opt) >= 3 and opt[:3] == '-__': continue
242     if opt not in opts.keys():
243     opts[opt] = self.cfg_params[k]
244     pass
245     pass
246     pass
247    
248     return
249    
250 nsmirnov 1.3 def processOptions_(self, opts):
251 nsmirnov 1.1 """
252     Processes the command-line options.
253     """
254    
255     for opt in opts.keys():
256     val = opts[opt]
257    
258 nsmirnov 1.3 # Skip actions, they are processed later in initializeActions_()
259     if opt in self.main_actions:
260     self.cfg_params['CRAB.'+opt[1:]] = val
261     continue
262     if opt in self.aux_actions:
263     self.cfg_params['CRAB.'+opt[1:]] = val
264     continue
265 nsmirnov 1.1
266    
267     elif ( opt == '-cfg' ):
268     pass
269    
270     elif ( opt in ('-continue', '-c') ):
271 nsmirnov 1.4 # Already processed in processContinueOption_()
272 nsmirnov 1.1 pass
273    
274     elif ( opt == '-jobtype' ):
275     if val : self.job_type_name = string.upper(val)
276     else : usage()
277     pass
278    
279     elif ( opt == '-Q' ):
280     self.flag_quiet = 1
281     pass
282    
283     elif ( opt == '-debug' ):
284 slacapra 1.6 if val: self.debug_level = int(val)
285     else: self.debug_level = 1
286 nsmirnov 1.1 pass
287    
288     elif ( opt == '-scheduler' ):
289     if val: self.scheduler_name = val
290     else:
291     print common.prog_name+". No value for '-scheduler'."
292     usage()
293     pass
294     pass
295 slacapra 1.22
296 fanzago 1.18 elif ( opt in ('-use_boss', '-useboss') ):
297     if ( val == '1' ):
298     self.scheduler_name = 'boss'
299     pass
300     elif ( val == '0' ):
301     pass
302     else:
303     print common.prog_name+'. Bad flag for -use_boss option:',\
304     val,'Possible values are 0(=No) or 1(=Yes)'
305     usage()
306     pass
307 fanzago 1.14 pass
308    
309 nsmirnov 1.3 elif string.find(opt,'.') == -1:
310     print common.prog_name+'. Unrecognized option '+opt
311     usage()
312     pass
313 nsmirnov 1.1
314 nsmirnov 1.3 # Override config parameters from INI-file with cmd-line params
315     if string.find(opt,'.') == -1 :
316     self.cfg_params['CRAB.'+opt[1:]] = val
317 nsmirnov 1.1 pass
318 nsmirnov 1.3 else:
319 nsmirnov 1.1 # Command line parameters in the form -SECTION.ENTRY=VALUE
320     self.cfg_params[opt[1:]] = val
321     pass
322     pass
323     return
324    
325 slacapra 1.8 def parseRange_(self, aRange):
326 nsmirnov 1.4 """
327 slacapra 1.8 Takes as the input a string with a range defined in any of the following
328     way, including combination, and return a tuple with the ints defined
329     according to following table. A consistency check is done.
330     NB: the first job is "1", not "0".
331     'all' -> [1,2,..., NJobs]
332     '' -> [1,2,..., NJobs]
333     'n1' -> [n1]
334     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
335     'n1,n2' -> [n1, n2]
336     'n1,n2-n3' -> [n1, n2, n2+1, n2+2, ..., n3-1, n3]
337     """
338     result = []
339    
340 slacapra 1.9 common.logger.debug(5,"parseRange_ "+str(aRange))
341     if aRange=='all' or aRange==None or aRange=='':
342 slacapra 1.8 result=range(0,common.jobDB.nJobs())
343     return result
344 slacapra 1.9 elif aRange=='0':
345     return result
346 slacapra 1.8
347     subRanges = string.split(aRange, ',')
348     for subRange in subRanges:
349     result = result+self.parseSimpleRange_(subRange)
350    
351     if self.checkUniqueness_(result):
352     return result
353     else:
354 slacapra 1.33 common.logger.message("Error "+result)
355 slacapra 1.8 return []
356    
357     def checkUniqueness_(self, list):
358     """
359 slacapra 1.9 check if a list contains only unique elements
360 slacapra 1.8 """
361    
362     uniqueList = []
363     # use a list comprehension statement (takes a while to understand)
364    
365     [uniqueList.append(it) for it in list if not uniqueList.count(it)]
366    
367     return (len(list)==len(uniqueList))
368    
369     def parseSimpleRange_(self, aRange):
370     """
371     Takes as the input a string with two integers separated by
372     the minus sign and returns the tuple with these numbers:
373     'n1-n2' -> [n1, n1+1, n1+2, ..., n2-1, n2]
374     'n1' -> [n1]
375     """
376     (start, end) = (None, None)
377    
378     result = []
379     minus = string.find(aRange, '-')
380     if ( minus < 0 ):
381     if isInt(aRange) and int(aRange)>0:
382 fanzago 1.37 # FEDE
383     #result.append(int(aRange)-1)
384     ###
385     result.append(aRange)
386 slacapra 1.8 else:
387 slacapra 1.9 common.logger.message("parseSimpleRange_ ERROR "+aRange)
388 nsmirnov 1.4 pass
389     else:
390 slacapra 1.8 (start, end) = string.split(aRange, '-')
391     if isInt(start) and isInt(end) and int(start)>0 and int(start)<int(end):
392 spiga 1.38 #result=range(int(start)-1, int(end))
393     result=range(int(start), int(end)+1) #Daniele
394 slacapra 1.8 else:
395 slacapra 1.33 common.logger.message("parseSimpleRange_ ERROR "+start+end)
396 slacapra 1.8
397     return result
398 nsmirnov 1.4
399 nsmirnov 1.3 def initializeActions_(self, opts):
400 nsmirnov 1.1 """
401     For each user action instantiate a corresponding
402     object and put it in the action dictionary.
403     """
404     for opt in opts.keys():
405 spiga 1.32 self.flag_useboss = 0
406 spiga 1.16 if ( opt == '-use_boss'):
407 slacapra 1.25 val = opts[opt]
408     if ( val == '1' ):
409     self.flag_useboss = 1
410     common.logger.message('Using BOSS')
411     pass
412     else:
413     self.flag_useboss = 0
414     pass
415 spiga 1.15
416     for opt in opts.keys():
417    
418 nsmirnov 1.1 val = opts[opt]
419 spiga 1.15
420    
421     if ( opt == '-create' ):
422 slacapra 1.22 ncjobs = 0
423 nsmirnov 1.1 if val:
424     if ( isInt(val) ):
425     ncjobs = int(val)
426     elif ( val == 'all'):
427     ncjobs = val
428     else:
429 nsmirnov 1.5 msg = 'Bad creation bunch size <'+str(val)+'>\n'
430     msg += ' Must be an integer or "all"'
431 slacapra 1.8 msg += ' Generic range is not allowed"'
432 nsmirnov 1.5 raise CrabException(msg)
433 nsmirnov 1.1 pass
434     else: ncjobs = 'all'
435    
436     if ncjobs != 0:
437 nsmirnov 1.3 # Instantiate Creator object
438 nsmirnov 1.1 creator = Creator(self.job_type_name,
439     self.cfg_params,
440     ncjobs)
441     self.actions[opt] = creator
442 nsmirnov 1.3
443     # Initialize the JobDB object if needed
444 nsmirnov 1.1 if not self.flag_continue:
445     common.jobDB.create(creator.nJobs())
446     pass
447 nsmirnov 1.3
448     # Create and initialize JobList
449    
450     common.job_list = JobList(common.jobDB.nJobs(),
451     creator.jobType())
452    
453     common.job_list.setScriptNames(self.job_type_name+'.sh')
454     common.job_list.setJDLNames(self.job_type_name+'.jdl')
455 slacapra 1.12 common.job_list.setCfgNames(self.job_type_name+'.orcarc')
456 slacapra 1.9
457     creator.writeJobsSpecsToDB()
458 nsmirnov 1.1 pass
459 nsmirnov 1.3 pass
460 nsmirnov 1.1
461     elif ( opt == '-submit' ):
462 slacapra 1.23
463     # total jobs
464     # get the first not already submitted
465 slacapra 1.24 common.logger.debug(5,'Total jobs '+str(common.jobDB.nJobs()))
466     lastSubmittedJob=0
467     for nj in range(common.jobDB.nJobs()):
468     if (common.jobDB.status(nj)=='S'):
469     lastSubmittedJob +=1
470 slacapra 1.30 else: break
471 slacapra 1.24 # count job from 1
472 slacapra 1.30 totalJobsSubmittable = common.jobDB.nJobs()-lastSubmittedJob
473 slacapra 1.24 common.logger.debug(5,'lastSubmittedJob '+str(lastSubmittedJob))
474     common.logger.debug(5,'totalJobsSubmittable '+str(totalJobsSubmittable))
475 slacapra 1.23
476 slacapra 1.24 nsjobs = lastSubmittedJob+totalJobsSubmittable
477 slacapra 1.23 # get user request
478 slacapra 1.22 if val:
479     if ( isInt(val) ):
480 slacapra 1.24 tmp = int(val)
481     if (tmp >= totalJobsSubmittable):
482     common.logger.message('asking to submit '+str(tmp)+' jobs, but only '+str(totalJobsSubmittable)+' left: submitting those')
483     pass
484     else:
485     nsjobs=lastSubmittedJob+int(val)
486 slacapra 1.23 elif (val=='all'):
487     pass
488 slacapra 1.22 else:
489     msg = 'Bad submission option <'+str(val)+'>\n'
490     msg += ' Must be an integer or "all"'
491     msg += ' Generic range is not allowed"'
492     raise CrabException(msg)
493     pass
494 slacapra 1.24 common.logger.debug(5,'nsjobs '+str(nsjobs))
495 slacapra 1.23
496     # submit N from last submitted job
497 slacapra 1.24 nj_list = range(lastSubmittedJob, nsjobs)
498     common.logger.debug(5,'nj_list '+str(nj_list))
499 nsmirnov 1.5
500     if len(nj_list) != 0:
501 nsmirnov 1.3 # Instantiate Submitter object
502 fanzago 1.37 print "prima di chiamare submitter, nj_list = ", nj_list
503 nsmirnov 1.5 self.actions[opt] = Submitter(self.cfg_params, nj_list)
504 nsmirnov 1.3
505     # Create and initialize JobList
506     if len(common.job_list) == 0 :
507     common.job_list = JobList(common.jobDB.nJobs(),
508     None)
509     common.job_list.setJDLNames(self.job_type_name+'.jdl')
510     pass
511 nsmirnov 1.1 pass
512 slacapra 1.30 pass
513 nsmirnov 1.1
514 nsmirnov 1.4 elif ( opt == '-list' ):
515 slacapra 1.8 jobs = self.parseRange_(val)
516    
517     common.jobDB.dump(jobs)
518 nsmirnov 1.4 pass
519    
520     elif ( opt == '-status' ):
521 slacapra 1.8 jobs = self.parseRange_(val)
522    
523 slacapra 1.9 if len(jobs) != 0:
524 spiga 1.16 if ( self.flag_useboss == 1 ):
525 slacapra 1.36 self.actions[opt] = StatusBoss()
526 spiga 1.15 else:
527 slacapra 1.36 self.actions[opt] = Status(jobs)
528 spiga 1.15 pass
529 nsmirnov 1.1 pass
530     pass
531 slacapra 1.22
532 nsmirnov 1.4 elif ( opt == '-kill' ):
533 slacapra 1.8
534 spiga 1.31 if ( self.flag_useboss == 1 ):
535     if val:
536     if val =='all':
537 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
538     jobs = allBoss_id.keys()
539 spiga 1.31 else:
540     jobs = self.parseRange_(val)
541     common.scheduler.cancel(jobs)
542     else:
543     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
544     else:
545     if val:
546     jobs = self.parseRange_(val)
547    
548     for nj in jobs:
549     st = common.jobDB.status(nj)
550     if st == 'S':
551     jid = common.jobDB.jobId(nj)
552     common.logger.message("Killing job # "+`(nj+1)`)
553     common.scheduler.cancel(jid)
554     common.jobDB.setStatus(nj, 'K')
555     pass
556 slacapra 1.9 pass
557 spiga 1.31
558     common.jobDB.save()
559 nsmirnov 1.4 pass
560 spiga 1.31 else:
561     common.logger.message("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
562 nsmirnov 1.1
563 slacapra 1.8 elif ( opt == '-getoutput' ):
564    
565 spiga 1.20 if ( self.flag_useboss == 1 ):
566     if val=='all' or val==None or val=='':
567 fanzago 1.37 allBoss_id = common.scheduler.listBoss()
568     jobs = allBoss_id.keys()
569 spiga 1.20 else:
570     jobs = self.parseRange_(val)
571     common.scheduler.getOutput(jobs)
572     else:
573     jobs = self.parseRange_(val)
574 spiga 1.13
575 slacapra 1.22 ## also this: create a ActorClass (GetOutput)
576 spiga 1.20 jobs_done = []
577     for nj in jobs:
578     st = common.jobDB.status(nj)
579     if st == 'D':
580 slacapra 1.12 jobs_done.append(nj)
581 spiga 1.20 pass
582     elif st == 'S':
583     jid = common.jobDB.jobId(nj)
584     currStatus = common.scheduler.queryStatus(jid)
585     if currStatus=="Done":
586     jobs_done.append(nj)
587     else:
588     msg = 'Job # '+`(nj+1)`+' submitted but still status '+currStatus+' not possible to get output'
589     common.logger.message(msg)
590     pass
591 slacapra 1.9 else:
592 spiga 1.20 # common.logger.message('Jobs #'+`(nj+1)`+' has status '+st+' not possible to get output')
593     pass
594 slacapra 1.12 pass
595    
596 spiga 1.20 for nj in jobs_done:
597 spiga 1.15 jid = common.jobDB.jobId(nj)
598     dir = common.scheduler.getOutput(jid)
599     common.jobDB.setStatus(nj, 'Y')
600 slacapra 1.12
601     # Rename the directory with results to smth readable
602     new_dir = common.work_space.resDir()
603 spiga 1.20 try:
604     files = os.listdir(dir)
605     for file in files:
606     os.rename(dir+'/'+file, new_dir+'/'+file)
607     os.rmdir(dir)
608     except OSError, e:
609     msg = 'rename files from '+dir+' to '+new_dir+' error: '
610     msg += str(e)
611     common.logger.message(msg)
612     # ignore error
613 slacapra 1.12 pass
614 spiga 1.20 pass
615 slacapra 1.22 ###
616    
617 spiga 1.13 resFlag = 0
618     exCode = common.scheduler.getExitStatus(jid)
619 spiga 1.34 Statistic.Monitor('retrieved',resFlag,jid,exCode)
620 slacapra 1.12
621     msg = 'Results of Job # '+`(nj+1)`+' are in '+new_dir
622     common.logger.message(msg)
623 nsmirnov 1.4 pass
624    
625     common.jobDB.save()
626     pass
627    
628     elif ( opt == '-resubmit' ):
629 fanzago 1.37 if ( self.flag_useboss == 1 ):
630     if val=='all' or val==None or val=='':
631     allBoss_id = common.scheduler.listBoss()
632     # sono gli internal_id di BOSS
633     jobs = allBoss_id.keys()
634     else:
635     jobs = self.parseRange_(val)
636     else:
637     if val:
638     jobs = self.parseRange_(val)
639    
640 slacapra 1.11 if val:
641     # create a list of jobs to be resubmitted.
642 slacapra 1.8
643 slacapra 1.22 ### as before, create a Resubmittter Class
644 slacapra 1.11 nj_list = []
645     for nj in jobs:
646 fanzago 1.37 st = common.jobDB.status(int(nj)-1)
647     #print "stato del job = ", st
648 slacapra 1.11 if st in ['K','A']:
649 fanzago 1.37 nj_list.append(int(nj)-1)
650     common.jobDB.setStatus(int(nj)-1,'C')
651 slacapra 1.11 elif st == 'Y':
652 fanzago 1.39 common.scheduler.moveOutput(nj)
653 fanzago 1.37 nj_list.append(int(nj)-1)
654     st = common.jobDB.setStatus(int(nj)-1,'RC')
655     elif st in ['D','C','X']:
656     # Nel caso DONE bisogna prima ritirare l'output
657     # Nel caso C va prima sottomesso
658     # Nel caso X va prima creato
659 slacapra 1.11 pass
660     else:
661 fanzago 1.37 #common.logger.message('Job #'+str(nj+1)+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
662     common.logger.message('Job #'+nj+' has status '+crabJobStatusToString(st)+' must be "killed" before resubmission')
663 slacapra 1.9 pass
664 slacapra 1.8
665 fanzago 1.37 if len(common.job_list) == 0 :
666     #print "common.job_list = ", common.job_list
667     common.job_list = JobList(common.jobDB.nJobs(),None)
668     common.job_list.setJDLNames(self.job_type_name+'.jdl')
669     pass
670    
671 slacapra 1.11 if len(nj_list) != 0:
672 fanzago 1.37 #print "nj_list = ", nj_list
673     common.scheduler.resubmit(nj_list)
674 slacapra 1.11 # Instantiate Submitter object
675     self.actions[opt] = Submitter(self.cfg_params, nj_list)
676    
677 slacapra 1.8 pass
678     pass
679 slacapra 1.11 else:
680     common.logger.message("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
681     common.logger.message("WARNING: _all_ job specified in the rage will be resubmitted!!!")
682     pass
683 fanzago 1.37 common.jobDB.save()
684 slacapra 1.8 pass
685    
686     elif ( opt == '-cancelAndResubmit' ):
687     jobs = self.parseRange_(val)
688 nsmirnov 1.5
689 nsmirnov 1.7 # Cancel submitted jobs.
690 nsmirnov 1.5
691 slacapra 1.8 nj_list = []
692     for nj in jobs:
693 nsmirnov 1.5 st = common.jobDB.status(nj)
694     if st == 'S':
695     jid = common.jobDB.jobId(nj)
696     common.scheduler.cancel(jid)
697     st = 'K'
698     common.jobDB.setStatus(nj, st)
699     pass
700 nsmirnov 1.7 pass
701 nsmirnov 1.5
702 slacapra 1.8 if st != 'X': nj_list.append(nj)
703     pass
704 nsmirnov 1.5
705     if len(nj_list) != 0:
706     # Instantiate Submitter object
707     self.actions[opt] = Submitter(self.cfg_params, nj_list)
708    
709     # Create and initialize JobList
710    
711     if len(common.job_list) == 0 :
712     common.job_list = JobList(common.jobDB.nJobs(),
713     None)
714     common.job_list.setJDLNames(self.job_type_name+'.jdl')
715     pass
716     pass
717 nsmirnov 1.4 pass
718    
719 slacapra 1.28 elif ( opt == '-testJdl' ):
720 slacapra 1.8 jobs = self.parseRange_(val)
721     nj_list = []
722     for nj in jobs:
723     st = common.jobDB.status(nj)
724 slacapra 1.12 if st == 'C': nj_list.append(nj)
725 slacapra 1.8 pass
726    
727     if len(nj_list) != 0:
728     # Instantiate Submitter object
729     self.actions[opt] = Checker(self.cfg_params, nj_list)
730    
731     # Create and initialize JobList
732    
733     if len(common.job_list) == 0 :
734     common.job_list = JobList(common.jobDB.nJobs(), None)
735     common.job_list.setJDLNames(self.job_type_name+'.jdl')
736     pass
737     pass
738    
739 slacapra 1.9 elif ( opt == '-postMortem' ):
740     jobs = self.parseRange_(val)
741     nj_list = []
742     for nj in jobs:
743     st = common.jobDB.status(nj)
744 fanzago 1.17 if st not in ['X', 'C']: nj_list.append(nj)
745 slacapra 1.9 pass
746    
747     if len(nj_list) != 0:
748     # Instantiate Submitter object
749 spiga 1.35 self.actions[opt] = PostMortem(self.cfg_params, nj_list,self.flag_useboss)
750 slacapra 1.9
751     # Create and initialize JobList
752    
753     if len(common.job_list) == 0 :
754     common.job_list = JobList(common.jobDB.nJobs(), None)
755     common.job_list.setJDLNames(self.job_type_name+'.jdl')
756     pass
757     pass
758 slacapra 1.33 else:
759     common.logger.message("No jobs to analyze")
760 slacapra 1.9
761     elif ( opt == '-clean' ):
762     if val != None:
763     raise CrabException("No range allowed for '-clean'")
764    
765 slacapra 1.36 theCleaner = Cleaner(self.scheduler_name == 'boss')
766     theCleaner.clean()
767 slacapra 1.9
768 nsmirnov 1.1 pass
769     return
770    
771 nsmirnov 1.3 def createWorkingSpace_(self):
772 slacapra 1.9 new_dir = ''
773    
774     try:
775     new_dir = self.cfg_params['USER.ui_working_dir']
776     except KeyError:
777     new_dir = common.prog_name + '_' + self.name + '_' + self.current_time
778     new_dir = self.cwd + new_dir
779     pass
780    
781 nsmirnov 1.1 common.work_space = WorkSpace(new_dir)
782     common.work_space.create()
783     return
784    
785 nsmirnov 1.3 def loadConfiguration_(self, opts):
786 nsmirnov 1.1
787     save_opts = common.work_space.loadSavedOptions()
788    
789     # Override saved options with new command-line options
790    
791     for k in opts.keys():
792     save_opts[k] = opts[k]
793     pass
794    
795     # Return updated options
796     return save_opts
797    
798 nsmirnov 1.3 def createLogger_(self, args):
799 nsmirnov 1.1
800     log = Logger()
801     log.quiet(self.flag_quiet)
802     log.setDebugLevel(self.debug_level)
803     log.write(args+'\n')
804 nsmirnov 1.3 log.message(self.headerString_())
805 nsmirnov 1.1 log.flush()
806     common.logger = log
807     return
808    
809 nsmirnov 1.3 def updateHistory_(self, args):
810 nsmirnov 1.1 history_fname = common.prog_name+'.history'
811     history_file = open(history_fname, 'a')
812     history_file.write(self.current_time+': '+args+'\n')
813     history_file.close()
814     return
815    
816 nsmirnov 1.3 def headerString_(self):
817 nsmirnov 1.1 """
818     Creates a string describing program options either given in
819     the command line or their default values.
820     """
821     header = common.prog_name + ' (version ' + common.prog_version_str + \
822     ') running on ' + \
823     time.ctime(time.time())+'\n\n' + \
824     common.prog_name+'. Working options:\n'
825     header = header +\
826     ' scheduler ' + self.scheduler_name + '\n'+\
827     ' job type ' + self.job_type_name + '\n'+\
828     ' working directory ' + common.work_space.topDir()\
829     + '\n'
830     return header
831    
832 nsmirnov 1.3 def createScheduler_(self):
833 nsmirnov 1.1 """
834     Creates a scheduler object instantiated by its name.
835     """
836     klass_name = 'Scheduler' + string.capitalize(self.scheduler_name)
837     file_name = klass_name
838     try:
839     klass = importName(file_name, klass_name)
840     except KeyError:
841     msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`'
842     raise CrabException(msg)
843     except ImportError, e:
844     msg = 'Cannot create scheduler '+self.scheduler_name
845     msg += ' (file: '+file_name+', class '+klass_name+'):\n'
846     msg += str(e)
847     raise CrabException(msg)
848    
849     common.scheduler = klass()
850     common.scheduler.configure(self.cfg_params)
851     return
852    
853     def run(self):
854     """
855     For each
856     """
857    
858     for act in self.main_actions:
859     if act in self.actions.keys(): self.actions[act].run()
860     pass
861    
862     for act in self.aux_actions:
863     if act in self.actions.keys(): self.actions[act].run()
864     pass
865     return
866    
867     ###########################################################################
868     def processHelpOptions(opts):
869    
870 slacapra 1.11 if len(opts):
871     for opt in opts.keys():
872     if opt in ('-v', '-version', '--version') :
873     print Crab.version()
874     return 1
875     if opt in ('-h','-help','--help') :
876     if opts[opt] : help(opts[opt])
877     else: help()
878     return 1
879     else:
880     usage()
881 nsmirnov 1.1
882     return 0
883    
884     ###########################################################################
885     if __name__ == '__main__':
886    
887 fanzago 1.18
888 fanzago 1.19 # # Initial settings for Python modules. Avoid appending manually lib paths.
889     # try:
890     # path=os.environ['EDG_WL_LOCATION']
891     # except:
892     # print "Error: Please set the EDG_WL_LOCATION environment variable pointing to the userinterface installation path"
893     # sys.exit(1)
894     #
895     # libPath=os.path.join(path, "lib")
896     # sys.path.append(libPath)
897     # libPath=os.path.join(path, "lib", "python")
898     # sys.path.append(libPath)
899 fanzago 1.18
900    
901 nsmirnov 1.1 # Parse command-line options and create a dictionary with
902     # key-value pairs.
903    
904     options = parseOptions(sys.argv[1:])
905    
906     # Process "help" options, such as '-help', '-version'
907    
908 slacapra 1.11 if processHelpOptions(options) : sys.exit(0)
909 nsmirnov 1.1
910     # Create, initialize, and run a Crab object
911    
912     try:
913 nsmirnov 1.3 crab = Crab(options)
914 nsmirnov 1.1 crab.run()
915     except CrabException, e:
916     print '\n' + common.prog_name + ': ' + str(e) + '\n'
917     if common.logger:
918     common.logger.write('ERROR: '+str(e)+'\n')
919     pass
920     pass
921    
922     pass