ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.64 by slacapra, Thu Jan 18 18:29:51 2007 UTC vs.
Revision 1.117 by fanzago, Fri Aug 17 10:45:26 2007 UTC

# Line 2 | Line 2 | from JobType import JobType
2   from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5 + from BlackWhiteListParser import BlackWhiteListParser
6   import common
6 – import PsetManipulator  
7 –
8 – import DataDiscovery
9 – import DataLocation
7   import Scram
8  
9 < import os, string, re, shutil
9 > import os, string, glob
10  
11   class Cmssw(JobType):
12      def __init__(self, cfg_params, ncjobs):
13          JobType.__init__(self, 'CMSSW')
14          common.logger.debug(3,'CMSSW::__init__')
15  
19 –        # Marco.
16          self._params = {}
17          self.cfg_params = cfg_params
18  
19 +        # init BlackWhiteListParser
20 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
21 +
22 +        try:
23 +            self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24 +        except KeyError:
25 +            self.MaxTarBallSize = 9.5
26 +
27          # number of jobs requested to be created, limit obj splitting
28          self.ncjobs = ncjobs
29  
# Line 29 | Line 33 | class Cmssw(JobType):
33          self.additional_inbox_files = []
34          self.scriptExe = ''
35          self.executable = ''
36 +        self.executable_arch = self.scram.getArch()
37          self.tgz_name = 'default.tgz'
38 +        self.additional_tgz_name = 'additional.tgz'
39          self.scriptName = 'CMSSW.sh'
40          self.pset = ''      #scrip use case Da  
41          self.datasetPath = '' #scrip use case Da
# Line 38 | Line 44 | class Cmssw(JobType):
44          self.fjrFileName = 'crab_fjr.xml'
45  
46          self.version = self.scram.getSWVersion()
47 +        
48 +        #
49 +        # Try to block creation in case of arch/version mismatch
50 +        #
51 +
52 +        a = string.split(self.version, "_")
53 +
54 +        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
55 +            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
56 +            raise CrabException(msg)
57 +        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58 +            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
59 +            raise CrabException(msg)
60 +        
61          common.taskDB.setDict('codeVersion',self.version)
62          self.setParam_('application', self.version)
63  
64          ### collect Data cards
65 +
66 +        ## get DBS mode
67 +        try:
68 +            self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69 +        except KeyError:
70 +            self.use_dbs_1 = 0
71 +            
72          try:
73              tmp =  cfg_params['CMSSW.datasetpath']
74              log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
# Line 61 | Line 88 | class Cmssw(JobType):
88              self.setParam_('dataset', 'None')
89              self.setParam_('owner', 'None')
90          else:
91 <            datasetpath_split = self.datasetPath.split("/")
92 <            self.setParam_('dataset', datasetpath_split[1])
93 <            self.setParam_('owner', datasetpath_split[-1])
94 <
91 >            try:
92 >                datasetpath_split = self.datasetPath.split("/")
93 >                # standard style
94 >                if self.use_dbs_1 == 1 :
95 >                    self.setParam_('dataset', datasetpath_split[1])
96 >                    self.setParam_('owner', datasetpath_split[-1])
97 >                else:
98 >                    self.setParam_('dataset', datasetpath_split[1])
99 >                    self.setParam_('owner', datasetpath_split[2])
100 >            except:
101 >                self.setParam_('dataset', self.datasetPath)
102 >                self.setParam_('owner', self.datasetPath)
103 >                
104          self.setTaskid_()
105          self.setParam_('taskId', self.cfg_params['taskId'])
106  
# Line 114 | Line 150 | class Cmssw(JobType):
150                      self.output_file.append(tmp)
151                      pass
152              else:
153 <                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
153 >                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
154                  pass
155              pass
156          except KeyError:
157 <            log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
157 >            log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
158              pass
159  
160          # script_exe file as additional file in inputSandbox
# Line 131 | Line 167 | class Cmssw(JobType):
167                 self.additional_inbox_files.append(string.strip(self.scriptExe))
168          except KeyError:
169              self.scriptExe = ''
170 +
171          #CarlosDaniele
172          if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
173 <           msg ="WARNING. script_exe  not defined"
173 >           msg ="Error. script_exe  not defined"
174             raise CrabException(msg)
175  
176          ## additional input files
177          try:
178              tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
179 <            common.logger.debug(5,"Additional input files: "+str(tmpAddFiles))
180 <            for tmpFile in tmpAddFiles:
181 <                tmpFile = string.strip(tmpFile)
182 <                if not os.path.exists(tmpFile):
183 <                    raise CrabException("Additional input file not found: "+tmpFile)
179 >            for tmp in tmpAddFiles:
180 >                tmp = string.strip(tmp)
181 >                dirname = ''
182 >                if not tmp[0]=="/": dirname = "."
183 >                files = []
184 >                if string.find(tmp,"*")>-1:
185 >                    files = glob.glob(os.path.join(dirname, tmp))
186 >                    if len(files)==0:
187 >                        raise CrabException("No additional input file found with this pattern: "+tmp)
188 >                else:
189 >                    files.append(tmp)
190 >                for file in files:
191 >                    if not os.path.exists(file):
192 >                        raise CrabException("Additional input file not found: "+file)
193                      pass
194 <                storedFile = common.work_space.shareDir()+ tmpFile
195 <                shutil.copyfile(tmpFile, storedFile)
196 <                self.additional_inbox_files.append(string.strip(storedFile))
194 >                    # fname = string.split(file, '/')[-1]
195 >                    # storedFile = common.work_space.pathForTgz()+'share/'+fname
196 >                    # shutil.copyfile(file, storedFile)
197 >                    self.additional_inbox_files.append(string.strip(file))
198                  pass
152 –            common.logger.debug(5,"Inbox files so far : "+str(self.additional_inbox_files))
199              pass
200 +            common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
201          except KeyError:
202              pass
203  
# Line 205 | Line 252 | class Cmssw(JobType):
252          except KeyError:
253              self.sourceSeedVtx = None
254              common.logger.debug(5,"No vertex seed given")
255 +
256 +        try:
257 +            self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
258 +        except KeyError:
259 +            self.sourceSeedG4 = None
260 +            common.logger.debug(5,"No g4 sim hits seed given")
261 +
262 +        try:
263 +            self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
264 +        except KeyError:
265 +            self.sourceSeedMix = None
266 +            common.logger.debug(5,"No mix seed given")
267 +
268          try:
269              self.firstRun = int(cfg_params['CMSSW.first_run'])
270          except KeyError:
271              self.firstRun = None
272              common.logger.debug(5,"No first run given")
273          if self.pset != None: #CarlosDaniele
274 <            self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
274 >            ver = string.split(self.version,"_")
275 >            if (int(ver[1])>=1 and int(ver[2])>=5):
276 >                import PsetManipulator150 as pp
277 >            else:
278 >                import PsetManipulator as pp
279 >            PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
280  
281          #DBSDLS-start
282          ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
# Line 233 | Line 298 | class Cmssw(JobType):
298                  self.jobSplittingForScript()
299              else:
300                  self.jobSplittingNoInput()
301 <        else:
301 >        else:
302              self.jobSplittingByBlocks(blockSites)
303  
304          # modify Pset
# Line 241 | Line 306 | class Cmssw(JobType):
306              try:
307                  if (self.datasetPath): # standard job
308                      # allow to processa a fraction of events in a file
309 <                    self.PsetEdit.inputModule("INPUT")
310 <                    self.PsetEdit.maxEvent("INPUTMAXEVENTS")
311 <                    self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
309 >                    PsetEdit.inputModule("INPUT")
310 >                    PsetEdit.maxEvent("INPUTMAXEVENTS")
311 >                    PsetEdit.skipEvent("INPUTSKIPEVENTS")
312                  else:  # pythia like job
313 <                    self.PsetEdit.maxEvent(self.eventsPerJob)
313 >                    PsetEdit.maxEvent(self.eventsPerJob)
314                      if (self.firstRun):
315 <                        self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
315 >                        PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
316                      if (self.sourceSeed) :
317 <                        self.PsetEdit.pythiaSeed("INPUT")
317 >                        PsetEdit.pythiaSeed("INPUT")
318                          if (self.sourceSeedVtx) :
319 <                            self.PsetEdit.pythiaSeedVtx("INPUTVTX")
319 >                            PsetEdit.vtxSeed("INPUTVTX")
320 >                        if (self.sourceSeedG4) :
321 >                            self.PsetEdit.g4Seed("INPUTG4")
322 >                        if (self.sourceSeedMix) :
323 >                            self.PsetEdit.mixSeed("INPUTMIX")
324                  # add FrameworkJobReport to parameter-set
325 <                self.PsetEdit.addCrabFJR(self.fjrFileName)
326 <                self.PsetEdit.psetWriter(self.configFilename())
325 >                PsetEdit.addCrabFJR(self.fjrFileName)
326 >                PsetEdit.psetWriter(self.configFilename())
327              except:
328                  msg='Error while manipuliating ParameterSet: exiting...'
329                  raise CrabException(msg)
330  
331      def DataDiscoveryAndLocation(self, cfg_params):
332  
333 +        import DataDiscovery
334 +        import DataDiscovery_DBS2
335 +        import DataLocation
336          common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
337  
338          datasetPath=self.datasetPath
339  
268 –        ## TODO
269 –        dataTiersList = ""
270 –        dataTiers = dataTiersList.split(',')
271 –
340          ## Contact the DBS
341 <        common.logger.message("Contacting DBS...")
341 >        common.logger.message("Contacting Data Discovery Services ...")
342          try:
343 <            self.pubdata=DataDiscovery.DataDiscovery(datasetPath, dataTiers, cfg_params)
343 >
344 >            if self.use_dbs_1 == 1 :
345 >                self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
346 >            else :
347 >                self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
348              self.pubdata.fetchDBSInfo()
349  
350          except DataDiscovery.NotExistingDatasetError, ex :
351              msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
352              raise CrabException(msg)
281 –
353          except DataDiscovery.NoDataTierinProvenanceError, ex :
354              msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
355              raise CrabException(msg)
356          except DataDiscovery.DataDiscoveryError, ex:
357 <            msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
357 >            msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
358 >            raise CrabException(msg)
359 >        except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
360 >            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
361 >            raise CrabException(msg)
362 >        except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
363 >            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
364 >            raise CrabException(msg)
365 >        except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
366 >            msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
367              raise CrabException(msg)
288 –
289 –        ## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
290 –        ## self.DBSPaths=self.pubdata.getDBSPaths()
291 –        common.logger.message("Required data are :"+self.datasetPath)
368  
369          self.filesbyblock=self.pubdata.getFiles()
370          self.eventsbyblock=self.pubdata.getEventsPerBlock()
371          self.eventsbyfile=self.pubdata.getEventsPerFile()
296 –        # print str(self.filesbyblock)
297 –        # print 'self.eventsbyfile',len(self.eventsbyfile)
298 –        # print str(self.eventsbyfile)
372  
373          ## get max number of events
374          self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
302 –        common.logger.message("The number of available events is %s\n"%self.maxEvents)
375  
304 –        common.logger.message("Contacting DLS...")
376          ## Contact the DLS and build a list of sites hosting the fileblocks
377          try:
378              dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
# Line 319 | Line 390 | class Cmssw(JobType):
390                  allSites.append(oneSite)
391          allSites = self.uniquelist(allSites)
392  
393 <        common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
394 <        common.logger.debug(6, "List of Sites: "+str(allSites))
393 >        # screen output
394 >        common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
395 >
396          return sites
397      
398      def jobSplittingByBlocks(self, blockSites):
# Line 382 | Line 454 | class Cmssw(JobType):
454          jobCount = 0
455          list_of_lists = []
456  
457 +        # list tracking which jobs are in which jobs belong to which block
458 +        jobsOfBlock = {}
459 +
460          # ---- Iterate over the blocks in the dataset until ---- #
461          # ---- we've met the requested total # of events    ---- #
462          while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
463              block = blocks[blockCount]
464              blockCount += 1
465 +            if block not in jobsOfBlock.keys() :
466 +                jobsOfBlock[block] = []
467              
468 <
469 <            numEventsInBlock = self.eventsbyblock[block]
470 <            common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
468 >            if self.eventsbyblock.has_key(block) :
469 >                numEventsInBlock = self.eventsbyblock[block]
470 >                common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
471              
472 <            files = self.filesbyblock[block]
473 <            numFilesInBlock = len(files)
474 <            if (numFilesInBlock <= 0):
475 <                continue
476 <            fileCount = 0
477 <
478 <            # ---- New block => New job ---- #
479 <            parString = "\\{"
480 <            # counter for number of events in files currently worked on
481 <            filesEventCount = 0
482 <            # flag if next while loop should touch new file
483 <            newFile = 1
484 <            # job event counter
485 <            jobSkipEventCount = 0
472 >                files = self.filesbyblock[block]
473 >                numFilesInBlock = len(files)
474 >                if (numFilesInBlock <= 0):
475 >                    continue
476 >                fileCount = 0
477 >
478 >                # ---- New block => New job ---- #
479 >                parString = "\\{"
480 >                # counter for number of events in files currently worked on
481 >                filesEventCount = 0
482 >                # flag if next while loop should touch new file
483 >                newFile = 1
484 >                # job event counter
485 >                jobSkipEventCount = 0
486              
487 <            # ---- Iterate over the files in the block until we've met the requested ---- #
488 <            # ---- total # of events or we've gone over all the files in this block  ---- #
489 <            while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
490 <                file = files[fileCount]
491 <                if newFile :
492 <                    try:
493 <                        numEventsInFile = self.eventsbyfile[file]
494 <                        common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
495 <                        # increase filesEventCount
496 <                        filesEventCount += numEventsInFile
497 <                        # Add file to current job
498 <                        parString += '\\\"' + file + '\\\"\,'
499 <                        newFile = 0
500 <                    except KeyError:
501 <                        common.logger.message("File "+str(file)+" has unknown number of events: skipping")
487 >                # ---- Iterate over the files in the block until we've met the requested ---- #
488 >                # ---- total # of events or we've gone over all the files in this block  ---- #
489 >                while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
490 >                    file = files[fileCount]
491 >                    if newFile :
492 >                        try:
493 >                            numEventsInFile = self.eventsbyfile[file]
494 >                            common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
495 >                            # increase filesEventCount
496 >                            filesEventCount += numEventsInFile
497 >                            # Add file to current job
498 >                            parString += '\\\"' + file + '\\\"\,'
499 >                            newFile = 0
500 >                        except KeyError:
501 >                            common.logger.message("File "+str(file)+" has unknown number of events: skipping")
502                          
503  
504 <                # if less events in file remain than eventsPerJobRequested
505 <                if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
506 <                    # if last file in block
507 <                    if ( fileCount == numFilesInBlock-1 ) :
508 <                        # end job using last file, use remaining events in block
504 >                    # if less events in file remain than eventsPerJobRequested
505 >                    if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
506 >                        # if last file in block
507 >                        if ( fileCount == numFilesInBlock-1 ) :
508 >                            # end job using last file, use remaining events in block
509 >                            # close job and touch new file
510 >                            fullString = parString[:-2]
511 >                            fullString += '\\}'
512 >                            list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
513 >                            common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
514 >                            self.jobDestination.append(blockSites[block])
515 >                            common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
516 >                            # fill jobs of block dictionary
517 >                            jobsOfBlock[block].append(jobCount+1)
518 >                            # reset counter
519 >                            jobCount = jobCount + 1
520 >                            totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
521 >                            eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
522 >                            jobSkipEventCount = 0
523 >                            # reset file
524 >                            parString = "\\{"
525 >                            filesEventCount = 0
526 >                            newFile = 1
527 >                            fileCount += 1
528 >                        else :
529 >                            # go to next file
530 >                            newFile = 1
531 >                            fileCount += 1
532 >                    # if events in file equal to eventsPerJobRequested
533 >                    elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
534                          # close job and touch new file
535                          fullString = parString[:-2]
536                          fullString += '\\}'
537 <                        list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
538 <                        common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
537 >                        list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
538 >                        common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
539                          self.jobDestination.append(blockSites[block])
540                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
541 +                        jobsOfBlock[block].append(jobCount+1)
542                          # reset counter
543                          jobCount = jobCount + 1
544 <                        totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
545 <                        eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
544 >                        totalEventCount = totalEventCount + eventsPerJobRequested
545 >                        eventsRemaining = eventsRemaining - eventsPerJobRequested
546                          jobSkipEventCount = 0
547                          # reset file
548                          parString = "\\{"
549                          filesEventCount = 0
550                          newFile = 1
551                          fileCount += 1
552 +                        
553 +                    # if more events in file remain than eventsPerJobRequested
554                      else :
555 <                        # go to next file
556 <                        newFile = 1
557 <                        fileCount += 1
558 <                # if events in file equal to eventsPerJobRequested
559 <                elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
560 <                    # close job and touch new file
561 <                    fullString = parString[:-2]
562 <                    fullString += '\\}'
563 <                    list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
564 <                    common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
565 <                    self.jobDestination.append(blockSites[block])
566 <                    common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
567 <                    # reset counter
568 <                    jobCount = jobCount + 1
569 <                    totalEventCount = totalEventCount + eventsPerJobRequested
570 <                    eventsRemaining = eventsRemaining - eventsPerJobRequested
571 <                    jobSkipEventCount = 0
572 <                    # reset file
573 <                    parString = "\\{"
574 <                    filesEventCount = 0
575 <                    newFile = 1
471 <                    fileCount += 1
472 <                    
473 <                # if more events in file remain than eventsPerJobRequested
474 <                else :
475 <                    # close job but don't touch new file
476 <                    fullString = parString[:-2]
477 <                    fullString += '\\}'
478 <                    list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
479 <                    common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
480 <                    self.jobDestination.append(blockSites[block])
481 <                    common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
482 <                    # increase counter
483 <                    jobCount = jobCount + 1
484 <                    totalEventCount = totalEventCount + eventsPerJobRequested
485 <                    eventsRemaining = eventsRemaining - eventsPerJobRequested
486 <                    # calculate skip events for last file
487 <                    # use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
488 <                    jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
489 <                    # remove all but the last file
490 <                    filesEventCount = self.eventsbyfile[file]
491 <                    parString = "\\{"
492 <                    parString += '\\\"' + file + '\\\"\,'
493 <                pass # END if
494 <            pass # END while (iterate over files in the block)
555 >                        # close job but don't touch new file
556 >                        fullString = parString[:-2]
557 >                        fullString += '\\}'
558 >                        list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
559 >                        common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
560 >                        self.jobDestination.append(blockSites[block])
561 >                        common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
562 >                        jobsOfBlock[block].append(jobCount+1)
563 >                        # increase counter
564 >                        jobCount = jobCount + 1
565 >                        totalEventCount = totalEventCount + eventsPerJobRequested
566 >                        eventsRemaining = eventsRemaining - eventsPerJobRequested
567 >                        # calculate skip events for last file
568 >                        # use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
569 >                        jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
570 >                        # remove all but the last file
571 >                        filesEventCount = self.eventsbyfile[file]
572 >                        parString = "\\{"
573 >                        parString += '\\\"' + file + '\\\"\,'
574 >                    pass # END if
575 >                pass # END while (iterate over files in the block)
576          pass # END while (iterate over blocks in the dataset)
577          self.ncjobs = self.total_number_of_jobs = jobCount
578          if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
579              common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
580 <        common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
580 >        common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
581          
582 +        # screen output
583 +        screenOutput = "List of jobs and available destination sites:\n\n"
584 +
585 +        blockCounter = 0
586 +        for block in blocks:
587 +            if block in jobsOfBlock.keys() :
588 +                blockCounter += 1
589 +                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
590 +
591 +        common.logger.message(screenOutput)
592 +
593          self.list_of_args = list_of_lists
594          return
595  
# Line 515 | Line 607 | class Cmssw(JobType):
607              raise CrabException(msg)
608  
609          if (self.selectEventsPerJob):
610 <            self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
610 >            if (self.selectTotalNumberEvents):
611 >                self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
612 >            elif(self.selectNumberOfJobs) :  
613 >                self.total_number_of_jobs =self.theNumberOfJobs
614 >                self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
615 >
616          elif (self.selectNumberOfJobs) :
617              self.total_number_of_jobs = self.theNumberOfJobs
618              self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
619 <
619 >
620          common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
621  
622          # is there any remainder?
# Line 537 | Line 634 | class Cmssw(JobType):
634              ## Since there is no input, any site is good
635             # self.jobDestination.append(["Any"])
636              self.jobDestination.append([""]) #must be empty to write correctly the xml
637 <            args=''
637 >            args=[]
638              if (self.firstRun):
639                      ## pythia first run
640                  #self.list_of_args.append([(str(self.firstRun)+str(i))])
641 <                args=args+(str(self.firstRun)+str(i))
641 >                args.append(str(self.firstRun)+str(i))
642              else:
643                  ## no first run
644                  #self.list_of_args.append([str(i)])
645 <                args=args+str(i)
645 >                args.append(str(i))
646              if (self.sourceSeed):
647 +                args.append(str(self.sourceSeed)+str(i))
648                  if (self.sourceSeedVtx):
649 <                    ## pythia + vtx random seed
650 <                    #self.list_of_args.append([
651 <                    #                          str(self.sourceSeed)+str(i),
652 <                    #                          str(self.sourceSeedVtx)+str(i)
653 <                    #                          ])
654 <                    args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
655 <                else:
656 <                    ## only pythia random seed
657 <                    #self.list_of_args.append([(str(self.sourceSeed)+str(i))])
658 <                    args=args +str(',')+str(self.sourceSeed)+str(i)
659 <            else:
660 <                ## no random seed
563 <                if str(args)=='': args=args+(str(self.firstRun)+str(i))
564 <            arguments=args.split(',')
565 <            if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
566 <            elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
567 <            else :self.list_of_args.append([str(arguments[0])])
649 >                    ## + vtx random seed
650 >                    args.append(str(self.sourceSeedVtx)+str(i))
651 >                if (self.sourceSeedG4):
652 >                    ## + G4 random seed
653 >                    args.append(str(self.sourceSeedG4)+str(i))
654 >                if (self.sourceSeedMix):    
655 >                    ## + Mix random seed
656 >                    args.append(str(self.sourceSeedMix)+str(i))
657 >                pass
658 >            pass
659 >            self.list_of_args.append(args)
660 >        pass
661              
662 <     #   print self.list_of_args
662 >        # print self.list_of_args
663  
664          return
665  
# Line 671 | Line 764 | class Cmssw(JobType):
764                      # the exe is private, so we must ship
765                      common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
766                      path = swArea+'/'
767 <                    exe = string.replace(exeWithPath, path,'')
768 <                    tar.add(path+exe,exe)
767 >                    # distinguish case when script is in user project area or given by full path somewhere else
768 >                    if exeWithPath.find(path) >= 0 :
769 >                        exe = string.replace(exeWithPath, path,'')
770 >                        tar.add(path+exe,os.path.basename(executable))
771 >                    else :
772 >                        tar.add(exeWithPath,os.path.basename(executable))
773                      pass
774                  else:
775                      # the exe is from release, we'll find it on WN
# Line 703 | Line 800 | class Cmssw(JobType):
800              pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
801              if os.path.isdir(pa):
802                  tar.add(pa,paDir)
803 +
804 +            ### FEDE FOR DBS PUBLICATION
805 +            ## Add PRODCOMMON dir to tar
806 +            prodcommonDir = 'ProdCommon'
807 +            prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
808 +            if os.path.isdir(prodcommonPath):
809 +                tar.add(prodcommonPath,prodcommonDir)
810 +            #############################    
811          
812              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
813              tar.close()
814          except :
815              raise CrabException('Could not create tar-ball')
816 <        
816 >
817 >        ## check for tarball size
818 >        tarballinfo = os.stat(self.tgzNameWithPath)
819 >        if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
820 >            raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
821 >
822          ## create tar-ball with ML stuff
823          self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
824          try:
# Line 723 | Line 833 | class Cmssw(JobType):
833          
834          return
835          
836 +    def additionalInputFileTgz(self):
837 +        """
838 +        Put all additional files into a tar ball and return its name
839 +        """
840 +        import tarfile
841 +        tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
842 +        tar = tarfile.open(tarName, "w:gz")
843 +        for file in self.additional_inbox_files:
844 +            tar.add(file,string.split(file,'/')[-1])
845 +        common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
846 +        tar.close()
847 +        return tarName
848 +
849      def wsSetupEnvironment(self, nj):
850          """
851          Returns part of a job script which prepares
# Line 734 | Line 857 | class Cmssw(JobType):
857          ## OLI_Daniele at this level  middleware already known
858  
859          txt += 'if [ $middleware == LCG ]; then \n'
860 +        txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
861 +        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
862 +        txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
863          txt += self.wsSetupCMSLCGEnvironment_()
864          txt += 'elif [ $middleware == OSG ]; then\n'
865          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
866          txt += '    echo "Created working directory: $WORKING_DIR"\n'
867          txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
868          txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
869 <        txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
870 <        txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
871 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
869 >        txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
870 >        txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
871 >        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
872          txt += '        rm -f $RUNTIME_AREA/$repo \n'
873          txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
874          txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 752 | Line 878 | class Cmssw(JobType):
878          txt += '    echo "Change to working directory: $WORKING_DIR"\n'
879          txt += '    cd $WORKING_DIR\n'
880          txt += self.wsSetupCMSOSGEnvironment_()
881 +        txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
882 +        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
883          txt += 'fi\n'
884  
885          # Prepare JobType-specific part
# Line 774 | Line 902 | class Cmssw(JobType):
902          txt += '        cd $RUNTIME_AREA\n'
903          txt += '        /bin/rm -rf $WORKING_DIR\n'
904          txt += '        if [ -d $WORKING_DIR ] ;then\n'
905 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
906 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
907 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
908 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
905 >        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
906 >        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
907 >        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
908 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
909          txt += '            rm -f $RUNTIME_AREA/$repo \n'
910          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
911          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 787 | Line 915 | class Cmssw(JobType):
915          txt += 'fi \n'
916          txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
917          txt += 'cd '+self.version+'\n'
918 +        ########## FEDE FOR DBS2 ######################
919 +        txt += 'SOFTWARE_DIR=`pwd`\n'
920 +        txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
921 +        ###############################################
922          ### needed grep for bug in scramv1 ###
923          txt += scram+' runtime -sh\n'
924          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
# Line 812 | Line 944 | class Cmssw(JobType):
944          txt += '        cd $RUNTIME_AREA\n'
945          txt += '        /bin/rm -rf $WORKING_DIR\n'
946          txt += '        if [ -d $WORKING_DIR ] ;then\n'
947 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
948 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
949 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
950 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
947 >        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
948 >        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
949 >        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
950 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
951          txt += '            rm -f $RUNTIME_AREA/$repo \n'
952          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
953          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 827 | Line 959 | class Cmssw(JobType):
959  
960          # Prepare job-specific part
961          job = common.job_list[nj]
962 +        ### FEDE FOR DBS OUTPUT PUBLICATION
963 +        if (self.datasetPath):
964 +            txt += '\n'
965 +            txt += 'DatasetPath='+self.datasetPath+'\n'
966 +
967 +            datasetpath_split = self.datasetPath.split("/")
968 +            
969 +            txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
970 +            txt += 'DataTier='+datasetpath_split[2]+'\n'
971 +            #txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
972 +            txt += 'ApplicationFamily=cmsRun\n'
973 +
974 +        else:
975 +            txt += 'DatasetPath=MCDataTier\n'
976 +            txt += 'PrimaryDataset=null\n'
977 +            txt += 'DataTier=null\n'
978 +            #txt += 'ProcessedDataset=null\n'
979 +            txt += 'ApplicationFamily=MCDataTier\n'
980          if self.pset != None: #CarlosDaniele
981              pset = os.path.basename(job.configFilename())
982              txt += '\n'
983 +            txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
984              if (self.datasetPath): # standard job
985                  #txt += 'InputFiles=$2\n'
986                  txt += 'InputFiles=${args[1]}\n'
987                  txt += 'MaxEvents=${args[2]}\n'
988                  txt += 'SkipEvents=${args[3]}\n'
989                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
990 <                txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
990 >                txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
991                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
992 <                txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
992 >                txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
993                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
994 <                txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
994 >                txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
995              else:  # pythia like job
996 <                if (self.sourceSeed):
997 <                    txt += 'FirstRun=${args[1]}\n'
996 >                seedIndex=1
997 >                if (self.firstRun):
998 >                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
999                      txt += 'echo "FirstRun: <$FirstRun>"\n'
1000 <                    txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
1001 <                else:
1002 <                    txt += '# Copy untouched pset\n'
851 <                    txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
1000 >                    txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1001 >                    seedIndex=seedIndex+1
1002 >
1003                  if (self.sourceSeed):
1004 < #                    txt += 'Seed=$2\n'
1005 <                    txt += 'Seed=${args[2]}\n'
1006 <                    txt += 'echo "Seed: <$Seed>"\n'
1007 <                    txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
1004 >                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
1005 >                    txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1006 >                    seedIndex=seedIndex+1
1007 >                    ## the following seeds are not always present
1008                      if (self.sourceSeedVtx):
1009 < #                        txt += 'VtxSeed=$3\n'
859 <                        txt += 'VtxSeed=${args[3]}\n'
1009 >                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1010                          txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1011 <                        txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
1012 <                    else:
1013 <                        txt += 'mv tmp_2.cfg pset.cfg\n'
1014 <                else:
1015 <                    txt += 'mv tmp_1.cfg pset.cfg\n'
1016 <                   # txt += '# Copy untouched pset\n'
1017 <                   # txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
1018 <
1011 >                        txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1012 >                        seedIndex += 1
1013 >                    if (self.sourceSeedG4):
1014 >                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1015 >                        txt += 'echo "G4Seed: <$G4Seed>"\n'
1016 >                        txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1017 >                        seedIndex += 1
1018 >                    if (self.sourceSeedMix):
1019 >                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1020 >                        txt += 'echo "MixSeed: <$mixSeed>"\n'
1021 >                        txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1022 >                        seedIndex += 1
1023 >                    pass
1024 >                pass
1025 >            txt += 'mv -f '+pset+' pset.cfg\n'
1026  
1027          if len(self.additional_inbox_files) > 0:
1028 <            for file in self.additional_inbox_files:
1029 <                relFile = file.split("/")[-1]
1030 <                txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
874 <                txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
875 <                txt += '   chmod +x '+relFile+'\n'
876 <                txt += 'fi\n'
1028 >            txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1029 >            txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1030 >            txt += 'fi\n'
1031              pass
1032  
1033          if self.pset != None: #CarlosDaniele
# Line 884 | Line 1038 | class Cmssw(JobType):
1038              txt += 'cat pset.cfg\n'
1039              txt += 'echo "****** end pset.cfg ********"\n'
1040              txt += '\n'
1041 +            ### FEDE FOR DBS OUTPUT PUBLICATION
1042 +            txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1043 +            txt += 'echo "PSETHASH = $PSETHASH" \n'
1044 +            ##############
1045 +            txt += '\n'
1046              # txt += 'echo "***** cat pset1.cfg *********"\n'
1047              # txt += 'cat pset1.cfg\n'
1048              # txt += 'echo "****** end pset1.cfg ********"\n'
# Line 925 | Line 1084 | class Cmssw(JobType):
1084              txt += '   echo "Successful untar" \n'
1085              txt += 'fi \n'
1086              txt += '\n'
1087 <            txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1087 >            txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1088              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1089 <            txt += '   export PYTHONPATH=ProdAgentApi\n'
1089 >            #### FEDE FOR DBS OUTPUT PUBLICATION
1090 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1091 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1092 >            #txt += '   export PYTHONPATH=ProdAgentApi\n'
1093              txt += 'else\n'
1094 <            txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1094 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1095 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1096 >            #txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1097 >            txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1098 >            ###################  
1099              txt += 'fi\n'
1100              txt += '\n'
1101  
# Line 944 | Line 1110 | class Cmssw(JobType):
1110          """
1111          
1112      def executableName(self):
1113 <        if self.pset == None: #CarlosDaniele
1113 >        if self.scriptExe: #CarlosDaniele
1114              return "sh "
1115          else:
1116              return self.executable
1117  
1118      def executableArgs(self):
1119 <        if self.pset == None:#CarlosDaniele
1119 >        if self.scriptExe:#CarlosDaniele
1120              return   self.scriptExe + " $NJob"
1121 <        else:
1122 <            return " -p pset.cfg"
1121 >        else:
1122 >            # if >= CMSSW_1_5_X, add -e
1123 >            version_array = self.scram.getSWVersion().split('_')
1124 >            major = 0
1125 >            minor = 0
1126 >            try:
1127 >                major = int(version_array[1])
1128 >                minor = int(version_array[2])
1129 >            except:
1130 >                msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"  
1131 >                raise CrabException(msg)
1132 >            if major >= 1 and minor >= 5 :
1133 >                return " -e -p pset.cfg"
1134 >            else:
1135 >                return " -p pset.cfg"
1136  
1137      def inputSandbox(self, nj):
1138          """
# Line 968 | Line 1147 | class Cmssw(JobType):
1147          if os.path.isfile(self.MLtgzfile):
1148              inp_box.append(self.MLtgzfile)
1149          ## config
1150 <        if not self.pset is None: #CarlosDaniele
1150 >        if not self.pset is None:
1151              inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1152          ## additional input files
1153 <        #for file in self.additional_inbox_files:
1154 <        #    inp_box.append(common.work_space.cwdDir()+file)
1153 >        tgz = self.additionalInputFileTgz()
1154 >        inp_box.append(tgz)
1155          return inp_box
1156  
1157      def outputSandbox(self, nj):
# Line 1006 | Line 1185 | class Cmssw(JobType):
1185              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1186              txt += '\n'
1187              txt += '# check output file\n'
1188 <            txt += 'ls '+fileWithSuffix+'\n'
1189 <            txt += 'ls_result=$?\n'
1190 <            txt += 'if [ $ls_result -ne 0 ] ; then\n'
1191 <            txt += '   echo "ERROR: Problem with output file"\n'
1188 >            # txt += 'ls '+fileWithSuffix+'\n'
1189 >            # txt += 'ls_result=$?\n'
1190 >            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1191 >            ###### FEDE FOR OUTPUT DATA PUBLICATION ########
1192 >            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1193 >            txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1194 >            ################################################
1195 >            txt += 'else\n'
1196 >            txt += '    exit_status=60302\n'
1197 >            txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1198 >            ############# FEDE ADDED CHECK FOR OUTPUT #############
1199 >            if fileWithSuffix in self.output_file:
1200 >                txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1201 >                txt += '    exit $exit_status\n'
1202 >            #######################################################    
1203              if common.scheduler.boss_scheduler_name == 'condor_g':
1204                  txt += '    if [ $middleware == OSG ]; then \n'
1205                  txt += '        echo "prepare dummy output file"\n'
1206                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1207                  txt += '    fi \n'
1018 –            txt += 'else\n'
1019 –            txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1208              txt += 'fi\n'
1209 <      
1209 >        file_list = []
1210 >        for fileWithSuffix in (self.output_file):
1211 >             file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1212 >            
1213 >        txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1214          txt += 'cd $RUNTIME_AREA\n'
1023 –        txt += 'cd $RUNTIME_AREA\n'
1024 –        ### OLI_DANIELE
1025 –        txt += 'if [ $middleware == OSG ]; then\n'  
1026 –        txt += '    cd $RUNTIME_AREA\n'
1027 –        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1028 –        txt += '    /bin/rm -rf $WORKING_DIR\n'
1029 –        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1030 –        txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1031 –        txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1032 –        txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1033 –        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1034 –        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1035 –        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1036 –        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1037 –        txt += '    fi\n'
1038 –        txt += 'fi\n'
1039 –        txt += '\n'
1040 –
1041 –        file_list = ''
1042 –        ## Add to filelist only files to be possibly copied to SE
1043 –        for fileWithSuffix in self.output_file:
1044 –            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1045 –            file_list=file_list+output_file_num+' '
1046 –        file_list=file_list[:-1]
1047 –        txt += 'file_list="'+file_list+'"\n'
1048 –
1215          return txt
1216  
1217      def numberFile_(self, file, txt):
# Line 1056 | Line 1222 | class Cmssw(JobType):
1222          # take away last extension
1223          name = p[0]
1224          for x in p[1:-1]:
1225 <           name=name+"."+x
1225 >            name=name+"."+x
1226          # add "_txt"
1227          if len(p)>1:
1228 <          ext = p[len(p)-1]
1229 <          result = name + '_' + txt + "." + ext
1228 >            ext = p[len(p)-1]
1229 >            result = name + '_' + txt + "." + ext
1230          else:
1231 <          result = name + '_' + txt
1231 >            result = name + '_' + txt
1232          
1233          return result
1234  
# Line 1075 | Line 1241 | class Cmssw(JobType):
1241              req='Member("VO-cms-' + \
1242                   self.version + \
1243                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1244 +        ## SL add requirement for OS version only if SL4
1245 +        #reSL4 = re.compile( r'slc4' )
1246 +        if self.executable_arch: # and reSL4.search(self.executable_arch):
1247 +            req+=' && Member("VO-cms-' + \
1248 +                 self.executable_arch + \
1249 +                 '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1250  
1251          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1252  
# Line 1094 | Line 1266 | class Cmssw(JobType):
1266          txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1267          txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1268          txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1269 +        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1270          txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1271          txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1272          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1273 +        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1274          txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1275          txt += '   else\n'
1276          txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
# Line 1112 | Line 1286 | class Cmssw(JobType):
1286          txt += '       cd $RUNTIME_AREA\n'
1287          txt += '       /bin/rm -rf $WORKING_DIR\n'
1288          txt += '       if [ -d $WORKING_DIR ] ;then\n'
1289 <        txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1290 <        txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1291 <        txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1292 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1293 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
1294 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1295 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1289 >        txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1290 >        txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1291 >        txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1292 >        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1293 >        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1294 >        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1295 >        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1296          txt += '       fi\n'
1297          txt += '\n'
1298          txt += '       exit 1\n'
# Line 1173 | Line 1347 | class Cmssw(JobType):
1347          txt += '       fi\n'
1348          txt += '   fi\n'
1349          txt += '   \n'
1176 –        txt += '   string=`cat /etc/redhat-release`\n'
1177 –        txt += '   echo $string\n'
1178 –        txt += '   if [[ $string = *alhalla* ]]; then\n'
1179 –        txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1180 –        txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1181 –        txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1182 –        txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1183 –        txt += '   else\n'
1184 –        txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1185 –        txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1186 –        txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1187 –        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1188 –        txt += '       rm -f $RUNTIME_AREA/$repo \n'
1189 –        txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1190 –        txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1191 –        txt += '       exit 1\n'
1192 –        txt += '   fi\n'
1350          txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1351          txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1352          return txt
1353  
1354 +    ### FEDE FOR DBS OUTPUT PUBLICATION
1355 +    def modifyReport(self, nj):
1356 +        """
1357 +        insert the part of the script that modifies the FrameworkJob Report
1358 +        """
1359 +
1360 +        txt = ''
1361 +        txt += 'echo "Modify Job Report" \n'
1362 +        #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1363 +        ################ FEDE FOR DBS2 #############################################
1364 +        txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1365 +        #############################################################################
1366 +        try:
1367 +            publish_data = int(self.cfg_params['USER.publish_data'])          
1368 +        except KeyError:
1369 +            publish_data = 0
1370 +
1371 +        txt += 'if [ -z "$SE" ]; then\n'
1372 +        txt += '    SE="" \n'
1373 +        txt += 'fi \n'
1374 +        txt += 'if [ -z "$SE_PATH" ]; then\n'
1375 +        txt += '    SE_PATH="" \n'
1376 +        txt += 'fi \n'
1377 +        txt += 'echo "SE = $SE"\n'
1378 +        txt += 'echo "SE_PATH = $SE_PATH"\n'
1379 +
1380 +        if (publish_data == 1):  
1381 +            #processedDataset = self.cfg_params['USER.processed_datasetname']
1382 +            processedDataset = self.cfg_params['USER.publish_data_name']
1383 +            txt += 'ProcessedDataset='+processedDataset+'\n'
1384 +            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1385 +            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1386 +            #### FEDE: added slash in LFN ##############
1387 +            txt += '    FOR_LFN=/copy_problems/ \n'
1388 +            txt += 'else \n'
1389 +            txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1390 +            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1391 +            txt += '    FOR_LFN=/store$tmp \n'
1392 +            txt += 'fi \n'
1393 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1394 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1395 +            txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1396 +            #txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1397 +            txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1398 +            txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1399 +            #txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1400 +      
1401 +            txt += 'modifyReport_result=$?\n'
1402 +            txt += 'echo modifyReport_result = $modifyReport_result\n'
1403 +            txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1404 +            txt += '    exit_status=1\n'
1405 +            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1406 +            txt += 'else\n'
1407 +            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1408 +            txt += 'fi\n'
1409 +        else:
1410 +            txt += 'ProcessedDataset=no_data_to_publish \n'
1411 +            #### FEDE: added slash in LFN ##############
1412 +            txt += 'FOR_LFN=/local/ \n'
1413 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1414 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1415 +        return txt
1416 +
1417 +    def cleanEnv(self):
1418 +        ### OLI_DANIELE
1419 +        txt = ''
1420 +        txt += 'if [ $middleware == OSG ]; then\n'  
1421 +        txt += '    cd $RUNTIME_AREA\n'
1422 +        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1423 +        txt += '    /bin/rm -rf $WORKING_DIR\n'
1424 +        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1425 +        txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1426 +        txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1427 +        txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1428 +        txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1429 +        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1430 +        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1431 +        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1432 +        txt += '    fi\n'
1433 +        txt += 'fi\n'
1434 +        txt += '\n'
1435 +        return txt
1436 +
1437      def setParam_(self, param, value):
1438          self._params[param] = value
1439  
# Line 1206 | Line 1446 | class Cmssw(JobType):
1446      def getTaskid(self):
1447          return self._taskId
1448  
1209 – #######################################################################
1449      def uniquelist(self, old):
1450          """
1451          remove duplicates from a list

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines