ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.155 by slacapra, Tue Feb 12 15:20:47 2008 UTC vs.
Revision 1.221 by fanzago, Wed Jun 18 14:02:42 2008 UTC

# Line 5 | Line 5 | from crab_util import *
5   from BlackWhiteListParser import BlackWhiteListParser
6   import common
7   import Scram
8 + from LFNBaseName import *
9  
10   import os, string, glob
11  
12   class Cmssw(JobType):
13 <    def __init__(self, cfg_params, ncjobs):
13 >    def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
14          JobType.__init__(self, 'CMSSW')
15          common.logger.debug(3,'CMSSW::__init__')
16 <
16 >        self.skip_blocks = skip_blocks
17 >
18          self.argsList = []
19  
20          self._params = {}
# Line 33 | Line 35 | class Cmssw(JobType):
35          self.executable = ''
36          self.executable_arch = self.scram.getArch()
37          self.tgz_name = 'default.tgz'
36 –        self.additional_tgz_name = 'additional.tgz'
38          self.scriptName = 'CMSSW.sh'
39 <        self.pset = ''      #scrip use case Da
40 <        self.datasetPath = '' #scrip use case Da
39 >        self.pset = ''
40 >        self.datasetPath = ''
41  
42          # set FJR file name
43          self.fjrFileName = 'crab_fjr.xml'
44  
45          self.version = self.scram.getSWVersion()
46 <
47 <        #
48 <        # Try to block creation in case of arch/version mismatch
49 <        #
50 <
51 <        a = string.split(self.version, "_")
52 <
53 <        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
54 <            msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
55 <            common.logger.message(msg)
55 <        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56 <            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
46 >        version_array = self.version.split('_')
47 >        self.CMSSW_major = 0
48 >        self.CMSSW_minor = 0
49 >        self.CMSSW_patch = 0
50 >        try:
51 >            self.CMSSW_major = int(version_array[1])
52 >            self.CMSSW_minor = int(version_array[2])
53 >            self.CMSSW_patch = int(version_array[3])
54 >        except:
55 >            msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
56              raise CrabException(msg)
57  
59 –        common.taskDB.setDict('codeVersion',self.version)
60 –        self.setParam_('application', self.version)
61 –
58          ### collect Data cards
59  
60          if not cfg_params.has_key('CMSSW.datasetpath'):
61              msg = "Error: datasetpath not defined "
62              raise CrabException(msg)
63 +        
64 +        ### Temporary: added to remove input file control in the case of PU
65 +        if not cfg_params.has_key('USER.dataset_pu'):
66 +            self.dataset_pu = 'NONE'
67 +        else:
68 +            self.dataset_pu = cfg_params['USER.dataset_pu']
69 +        ####    
70 +        
71          tmp =  cfg_params['CMSSW.datasetpath']
72          log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
73          if string.lower(tmp)=='none':
# Line 73 | Line 77 | class Cmssw(JobType):
77              self.datasetPath = tmp
78              self.selectNoInput = 0
79  
76 –        # ML monitoring
77 –        # split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
78 –        if not self.datasetPath:
79 –            self.setParam_('dataset', 'None')
80 –            self.setParam_('owner', 'None')
81 –        else:
82 –            ## SL what is supposed to fail here?
83 –            try:
84 –                datasetpath_split = self.datasetPath.split("/")
85 –                # standard style
86 –                self.setParam_('datasetFull', self.datasetPath)
87 –                self.setParam_('dataset', datasetpath_split[1])
88 –                self.setParam_('owner', datasetpath_split[2])
89 –            except:
90 –                self.setParam_('dataset', self.datasetPath)
91 –                self.setParam_('owner', self.datasetPath)
92 –
93 –        self.setParam_('taskId', common.taskDB.dict('taskId'))
94 –
80          self.dataTiers = []
81 <
81 >        self.debugWrap = ''
82 >        self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
83 >        if self.debug_wrapper: self.debugWrap='--debug'
84          ## now the application
85          self.executable = cfg_params.get('CMSSW.executable','cmsRun')
99 –        self.setParam_('exe', self.executable)
86          log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
87  
88          if not cfg_params.has_key('CMSSW.pset'):
# Line 117 | Line 103 | class Cmssw(JobType):
103          self.output_file_sandbox.append(self.fjrFileName)
104  
105          # other output files to be returned via sandbox or copied to SE
106 +        outfileflag = False
107          self.output_file = []
108          tmp = cfg_params.get('CMSSW.output_file',None)
109          if tmp :
110 <            tmpOutFiles = string.split(tmp,',')
111 <            log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
112 <            for tmp in tmpOutFiles:
113 <                tmp=string.strip(tmp)
127 <                self.output_file.append(tmp)
128 <                pass
129 <        else:
130 <            log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
131 <        pass
110 >            self.output_file = [x.strip() for x in tmp.split(',')]
111 >            outfileflag = True #output found
112 >        #else:
113 >        #    log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
114  
115          # script_exe file as additional file in inputSandbox
116          self.scriptExe = cfg_params.get('USER.script_exe',None)
117          if self.scriptExe :
118 <           if not os.path.isfile(self.scriptExe):
119 <              msg ="ERROR. file "+self.scriptExe+" not found"
120 <              raise CrabException(msg)
121 <           self.additional_inbox_files.append(string.strip(self.scriptExe))
118 >            if not os.path.isfile(self.scriptExe):
119 >                msg ="ERROR. file "+self.scriptExe+" not found"
120 >                raise CrabException(msg)
121 >            self.additional_inbox_files.append(string.strip(self.scriptExe))
122  
141 –        #CarlosDaniele
123          if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
124 <           msg ="Error. script_exe  not defined"
125 <           raise CrabException(msg)
124 >            msg ="Error. script_exe  not defined"
125 >            raise CrabException(msg)
126 >
127 >        # use parent files...
128 >        self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
129  
130          ## additional input files
131          if cfg_params.has_key('USER.additional_input_files'):
# Line 161 | Line 145 | class Cmssw(JobType):
145                      if not os.path.exists(file):
146                          raise CrabException("Additional input file not found: "+file)
147                      pass
164 –                    # fname = string.split(file, '/')[-1]
165 –                    # storedFile = common.work_space.pathForTgz()+'share/'+fname
166 –                    # shutil.copyfile(file, storedFile)
148                      self.additional_inbox_files.append(string.strip(file))
149                  pass
150              pass
# Line 189 | Line 170 | class Cmssw(JobType):
170          if cfg_params.has_key('CMSSW.total_number_of_events'):
171              self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
172              self.selectTotalNumberEvents = 1
173 +            if self.selectNumberOfJobs  == 1:
174 +                if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
175 +                    msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
176 +                    raise CrabException(msg)
177          else:
178              self.total_number_of_events = 0
179              self.selectTotalNumberEvents = 0
180  
181 <        if self.pset != None: #CarlosDaniele
181 >        if self.pset != None:
182               if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
183                   msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
184                   raise CrabException(msg)
# Line 202 | Line 187 | class Cmssw(JobType):
187                   msg = 'Must specify  number_of_jobs.'
188                   raise CrabException(msg)
189  
190 <        ## source seed for pythia
190 >        ## New method of dealing with seeds
191 >        self.incrementSeeds = []
192 >        self.preserveSeeds = []
193 >        if cfg_params.has_key('CMSSW.preserve_seeds'):
194 >            tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
195 >            for tmp in tmpList:
196 >                tmp.strip()
197 >                self.preserveSeeds.append(tmp)
198 >        if cfg_params.has_key('CMSSW.increment_seeds'):
199 >            tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
200 >            for tmp in tmpList:
201 >                tmp.strip()
202 >                self.incrementSeeds.append(tmp)
203 >
204 >        ## Old method of dealing with seeds
205 >        ## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
206 >        ## remove
207          self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
208 +        if self.sourceSeed:
209 +            print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
210 +            self.incrementSeeds.append('sourceSeed')
211 +            self.incrementSeeds.append('theSource')
212  
213          self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
214 +        if self.sourceSeedVtx:
215 +            print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
216 +            self.incrementSeeds.append('VtxSmeared')
217  
218          self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
219 +        if self.sourceSeedG4:
220 +            print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
221 +            self.incrementSeeds.append('g4SimHits')
222  
223          self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
224 +        if self.sourceSeedMix:
225 +            print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
226 +            self.incrementSeeds.append('mix')
227  
228          self.firstRun = cfg_params.get('CMSSW.first_run',None)
229  
216 –        if self.pset != None: #CarlosDaniele
217 –            import PsetManipulator as pp
218 –            PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
230  
231          # Copy/return
221 –
232          self.copy_data = int(cfg_params.get('USER.copy_data',0))
233          self.return_data = int(cfg_params.get('USER.return_data',0))
234  
# Line 234 | Line 244 | class Cmssw(JobType):
244              blockSites = self.DataDiscoveryAndLocation(cfg_params)
245          #DBSDLS-end
246  
237 –        self.tgzNameWithPath = self.getTarBall(self.executable)
238 –
247          ## Select Splitting
248          if self.selectNoInput:
249 <            if self.pset == None: #CarlosDaniele
249 >            if self.pset == None:
250                  self.jobSplittingForScript()
251              else:
252                  self.jobSplittingNoInput()
253          else:
254              self.jobSplittingByBlocks(blockSites)
255  
256 <        # modify Pset
257 <        if self.pset != None: #CarlosDaniele
258 <            try:
259 <                if (self.datasetPath): # standard job
260 <                    # allow to processa a fraction of events in a file
261 <                    PsetEdit.inputModule("INPUTFILE")
262 <                    PsetEdit.maxEvent(0)
263 <                    PsetEdit.skipEvent(0)
264 <                else:  # pythia like job
256 >        # modify Pset only the first time
257 >        if isNew:
258 >            if self.pset != None:
259 >                import PsetManipulator as pp
260 >                PsetEdit = pp.PsetManipulator(self.pset)
261 >                try:
262 >                    # Add FrameworkJobReport to parameter-set, set max events.
263 >                    # Reset later for data jobs by writeCFG which does all modifications
264 >                    PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
265                      PsetEdit.maxEvent(self.eventsPerJob)
266 <                    if (self.firstRun):
267 <                        PsetEdit.pythiaFirstRun(0)  #First Run
268 <                    if (self.sourceSeed) :
269 <                        PsetEdit.pythiaSeed(0)
270 <                        if (self.sourceSeedVtx) :
271 <                            PsetEdit.vtxSeed(0)
272 <                        if (self.sourceSeedG4) :
273 <                            PsetEdit.g4Seed(0)
274 <                        if (self.sourceSeedMix) :
275 <                            PsetEdit.mixSeed(0)
276 <                # add FrameworkJobReport to parameter-set
277 <                PsetEdit.addCrabFJR(self.fjrFileName)
278 <                PsetEdit.psetWriter(self.configFilename())
279 <            except:
280 <                msg='Error while manipuliating ParameterSet: exiting...'
281 <                raise CrabException(msg)
266 >                    PsetEdit.psetWriter(self.configFilename())
267 >                    ## If present, add TFileService to output files
268 >                    if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
269 >                        tfsOutput = PsetEdit.getTFileService()
270 >                        if tfsOutput:
271 >                            if tfsOutput in self.output_file:
272 >                                common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
273 >                            else:
274 >                                outfileflag = True #output found
275 >                                self.output_file.append(tfsOutput)
276 >                                common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
277 >                            pass
278 >                        pass
279 >                    ## If present and requested, add PoolOutputModule to output files
280 >                    if int(cfg_params.get('CMSSW.get_edm_output',0)):
281 >                        edmOutput = PsetEdit.getPoolOutputModule()
282 >                        if edmOutput:
283 >                            if edmOutput in self.output_file:
284 >                                common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
285 >                            else:
286 >                                self.output_file.append(edmOutput)
287 >                                common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
288 >                            pass
289 >                        pass
290 >                except CrabException:
291 >                    msg='Error while manipulating ParameterSet: exiting...'
292 >                    raise CrabException(msg)
293 >            ## Prepare inputSandbox TarBall (only the first time)  
294 >            self.tgzNameWithPath = self.getTarBall(self.executable)
295  
296      def DataDiscoveryAndLocation(self, cfg_params):
297  
# Line 283 | Line 304 | class Cmssw(JobType):
304          ## Contact the DBS
305          common.logger.message("Contacting Data Discovery Services ...")
306          try:
307 <            self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
307 >            self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
308              self.pubdata.fetchDBSInfo()
309  
310          except DataDiscovery.NotExistingDatasetError, ex :
# Line 299 | Line 320 | class Cmssw(JobType):
320          self.filesbyblock=self.pubdata.getFiles()
321          self.eventsbyblock=self.pubdata.getEventsPerBlock()
322          self.eventsbyfile=self.pubdata.getEventsPerFile()
323 +        self.parentFiles=self.pubdata.getParent()
324  
325          ## get max number of events
326 <        self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
326 >        self.maxEvents=self.pubdata.getMaxEvents()
327  
328          ## Contact the DLS and build a list of sites hosting the fileblocks
329          try:
# Line 325 | Line 347 | class Cmssw(JobType):
347  
348          return sites
349  
328 –    def setArgsList(self, argsList):
329 –        self.argsList = argsList
330 –
350      def jobSplittingByBlocks(self, blockSites):
351          """
352          Perform job splitting. Jobs run over an integer number of files
# Line 378 | Line 397 | class Cmssw(JobType):
397          else :
398              totalNumberOfJobs = self.ncjobs
399  
381 –
400          blocks = blockSites.keys()
401          blockCount = 0
402          # Backup variable in case self.maxEvents counted events in a non-included block
# Line 419 | Line 437 | class Cmssw(JobType):
437  
438                  # ---- Iterate over the files in the block until we've met the requested ---- #
439                  # ---- total # of events or we've gone over all the files in this block  ---- #
440 +                pString=''
441                  while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
442                      file = files[fileCount]
443 +                    if self.useParent:
444 +                        parent = self.parentFiles[file]
445 +                        for f in parent :
446 +                            pString += '\\\"' + f + '\\\"\,'
447 +                        common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
448 +                        common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
449                      if newFile :
450                          try:
451                              numEventsInFile = self.eventsbyfile[file]
# Line 433 | Line 458 | class Cmssw(JobType):
458                          except KeyError:
459                              common.logger.message("File "+str(file)+" has unknown number of events: skipping")
460  
461 <
461 >                    eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
462                      # if less events in file remain than eventsPerJobRequested
463 <                    if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
463 >                    if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
464                          # if last file in block
465                          if ( fileCount == numFilesInBlock-1 ) :
466                              # end job using last file, use remaining events in block
467                              # close job and touch new file
468                              fullString = parString[:-2]
469 <                            list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
469 >                            if self.useParent:
470 >                                fullParentString = pString[:-2]
471 >                                list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
472 >                            else:
473 >                                list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
474                              common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
475                              self.jobDestination.append(blockSites[block])
476                              common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
# Line 453 | Line 482 | class Cmssw(JobType):
482                              eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
483                              jobSkipEventCount = 0
484                              # reset file
485 +                            pString = ""
486                              parString = ""
487                              filesEventCount = 0
488                              newFile = 1
# Line 465 | Line 495 | class Cmssw(JobType):
495                      elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
496                          # close job and touch new file
497                          fullString = parString[:-2]
498 <                        list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
498 >                        if self.useParent:
499 >                            fullParentString = pString[:-2]
500 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
501 >                        else:
502 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
503                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
504                          self.jobDestination.append(blockSites[block])
505                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
# Line 476 | Line 510 | class Cmssw(JobType):
510                          eventsRemaining = eventsRemaining - eventsPerJobRequested
511                          jobSkipEventCount = 0
512                          # reset file
513 +                        pString = ""
514                          parString = ""
515                          filesEventCount = 0
516                          newFile = 1
# Line 485 | Line 520 | class Cmssw(JobType):
520                      else :
521                          # close job but don't touch new file
522                          fullString = parString[:-2]
523 <                        list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
523 >                        if self.useParent:
524 >                            fullParentString = pString[:-2]
525 >                            list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
526 >                        else:
527 >                            list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
528                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
529                          self.jobDestination.append(blockSites[block])
530                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
# Line 499 | Line 538 | class Cmssw(JobType):
538                          jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
539                          # remove all but the last file
540                          filesEventCount = self.eventsbyfile[file]
541 <                        parString = ""
542 <                        parString += '\\\"' + file + '\\\"\,'
541 >                        if self.useParent:
542 >                            for f in parent : pString += '\\\"' + f + '\\\"\,'
543 >                        parString = '\\\"' + file + '\\\"\,'
544                      pass # END if
545                  pass # END while (iterate over files in the block)
546          pass # END while (iterate over blocks in the dataset)
# Line 520 | Line 560 | class Cmssw(JobType):
560          for block in blocks:
561              if block in jobsOfBlock.keys() :
562                  blockCounter += 1
563 <                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
563 >                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
564 >                    ','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
565                  if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
566                      noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
567                      bloskNoSite.append( blockCounter )
# Line 602 | Line 643 | class Cmssw(JobType):
643              if (self.firstRun):
644                  ## pythia first run
645                  args.append(str(self.firstRun)+str(i))
605 –            if (self.sourceSeed):
606 –                args.append(str(self.sourceSeed)+str(i))
607 –                if (self.sourceSeedVtx):
608 –                    ## + vtx random seed
609 –                    args.append(str(self.sourceSeedVtx)+str(i))
610 –                if (self.sourceSeedG4):
611 –                    ## + G4 random seed
612 –                    args.append(str(self.sourceSeedG4)+str(i))
613 –                if (self.sourceSeedMix):
614 –                    ## + Mix random seed
615 –                    args.append(str(self.sourceSeedMix)+str(i))
616 –                pass
617 –            pass
646              self.list_of_args.append(args)
619 –        pass
647  
648          return
649  
650  
651 <    def jobSplittingForScript(self):#CarlosDaniele
651 >    def jobSplittingForScript(self):
652          """
653          Perform job splitting based on number of job
654          """
# Line 637 | Line 664 | class Cmssw(JobType):
664          # argument is seed number.$i
665          self.list_of_args = []
666          for i in range(self.total_number_of_jobs):
640 –            ## Since there is no input, any site is good
641 –           # self.jobDestination.append(["Any"])
667              self.jobDestination.append([""])
643 –            ## no random seed
668              self.list_of_args.append([str(i)])
669          return
670  
671 <    def split(self, jobParams):
671 >    def split(self, jobParams,firstJobID):
672  
649 –        common.jobDB.load()
650 –        #### Fabio
673          njobs = self.total_number_of_jobs
674          arglist = self.list_of_args
675          # create the empty structure
676          for i in range(njobs):
677              jobParams.append("")
678  
679 <        for job in range(njobs):
680 <            jobParams[job] = arglist[job]
681 <            # print str(arglist[job])
682 <            # print jobParams[job]
683 <            common.jobDB.setArguments(job, jobParams[job])
684 <            common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
685 <            common.jobDB.setDestination(job, self.jobDestination[job])
679 >        listID=[]
680 >        listField=[]
681 >        for id in range(njobs):
682 >            job = id + int(firstJobID)
683 >            jobParams[id] = arglist[id]
684 >            listID.append(job+1)
685 >            job_ToSave ={}
686 >            concString = ' '
687 >            argu=''
688 >            if len(jobParams[id]):
689 >                argu +=   concString.join(jobParams[id] )
690 >            job_ToSave['arguments']= str(job+1)+' '+argu
691 >            job_ToSave['dlsDestination']= self.jobDestination[id]
692 >            listField.append(job_ToSave)
693 >            msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
694 >            +"                     Destination: "+str(self.jobDestination[id])
695 >            common.logger.debug(5,msg)
696 >        common._db.updateJob_(listID,listField)
697 >        self.argsList = (len(jobParams[0])+1)
698  
665 –        common.jobDB.save()
699          return
700  
668 –    def getJobTypeArguments(self, nj, sched):
669 –        result = ''
670 –        for i in common.jobDB.arguments(nj):
671 –            result=result+str(i)+" "
672 –        return result
673 –
701      def numberOfJobs(self):
675 –        # Fabio
702          return self.total_number_of_jobs
703  
704      def getTarBall(self, exe):
705          """
706          Return the TarBall with lib and exe
707          """
682 –
683 –        # if it exist, just return it
684 –        #
685 –        # Marco. Let's start to use relative path for Boss XML files
686 –        #
708          self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
709          if os.path.exists(self.tgzNameWithPath):
710              return self.tgzNameWithPath
# Line 697 | Line 718 | class Cmssw(JobType):
718  
719          # First of all declare the user Scram area
720          swArea = self.scram.getSWArea_()
700 –        #print "swArea = ", swArea
701 –        # swVersion = self.scram.getSWVersion()
702 –        # print "swVersion = ", swVersion
721          swReleaseTop = self.scram.getReleaseTop_()
704 –        #print "swReleaseTop = ", swReleaseTop
722  
723          ## check if working area is release top
724          if swReleaseTop == '' or swArea == swReleaseTop:
725 +            common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
726              return
727  
728          import tarfile
# Line 746 | Line 764 | class Cmssw(JobType):
764                  tar.add(module,moduleDir)
765  
766              ## Now check if any data dir(s) is present
767 <            swAreaLen=len(swArea)
768 <            for root, dirs, files in os.walk(swArea):
769 <                if "data" in dirs:
770 <                    common.logger.debug(5,"data "+root+"/data"+" to be tarred")
771 <                    tar.add(root+"/data",root[swAreaLen:]+"/data")
772 <
773 <            ### Removed ProdAgent Api dependencies ###
774 <            ### Add ProdAgent dir to tar
775 <            #paDir = 'ProdAgentApi'
776 <            #pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
777 <            #if os.path.isdir(pa):
778 <            #    tar.add(pa,paDir)
767 >            self.dataExist = False
768 >            todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
769 >            while len(todo_list):
770 >                entry, name = todo_list.pop()
771 >                if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
772 >                    continue
773 >                if os.path.isdir(swArea+"/src/"+entry):
774 >                    entryPath = entry + '/'
775 >                    todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
776 >                    if name == 'data':
777 >                        self.dataExist=True
778 >                        common.logger.debug(5,"data "+entry+" to be tarred")
779 >                        tar.add(swArea+"/src/"+entry,"src/"+entry)
780 >                    pass
781 >                pass
782 >
783 >            ### CMSSW ParameterSet
784 >            if not self.pset is None:
785 >                cfg_file = common.work_space.jobDir()+self.configFilename()
786 >                tar.add(cfg_file,self.configFilename())
787 >                common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
788 >
789  
790              ## Add ProdCommon dir to tar
791 <            prodcommonDir = 'ProdCommon'
792 <            prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
793 <            if os.path.isdir(prodcommonPath):
794 <                tar.add(prodcommonPath,prodcommonDir)
791 >            prodcommonDir = './'
792 >            prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
793 >            neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools','ProdCommon/Core','ProdCommon/MCPayloads', 'IMProv']
794 >            for file in neededStuff:
795 >                tar.add(prodcommonPath+file,prodcommonDir+file)
796 >            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
797  
798 +            ##### ML stuff
799 +            ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
800 +            path=os.environ['CRABDIR'] + '/python/'
801 +            for file in ML_file_list:
802 +                tar.add(path+file,file)
803              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
804 +
805 +            ##### Utils
806 +            Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py']
807 +            for file in Utils_file_list:
808 +                tar.add(path+file,file)
809 +            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
810 +
811 +            ##### AdditionalFiles
812 +            for file in self.additional_inbox_files:
813 +                tar.add(file,string.split(file,'/')[-1])
814 +            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
815 +
816              tar.close()
817 <        except :
818 <            raise CrabException('Could not create tar-ball')
817 >        except IOError:
818 >            raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
819 >        except tarfile.TarError:
820 >            raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
821  
822          ## check for tarball size
823          tarballinfo = os.stat(self.tgzNameWithPath)
# Line 776 | Line 825 | class Cmssw(JobType):
825              raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
826  
827          ## create tar-ball with ML stuff
779 –        self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
780 –        try:
781 –            tar = tarfile.open(self.MLtgzfile, "w:gz")
782 –            path=os.environ['CRABDIR'] + '/python/'
783 –            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
784 –                tar.add(path+file,file)
785 –            common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
786 –            tar.close()
787 –        except :
788 –            raise CrabException('Could not create ML files tar-ball')
828  
829 <        return
791 <
792 <    def additionalInputFileTgz(self):
793 <        """
794 <        Put all additional files into a tar ball and return its name
795 <        """
796 <        import tarfile
797 <        tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
798 <        tar = tarfile.open(tarName, "w:gz")
799 <        for file in self.additional_inbox_files:
800 <            tar.add(file,string.split(file,'/')[-1])
801 <        common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
802 <        tar.close()
803 <        return tarName
804 <
805 <    def wsSetupEnvironment(self, nj):
829 >    def wsSetupEnvironment(self, nj=0):
830          """
831          Returns part of a job script which prepares
832          the execution environment for the job 'nj'.
833          """
834 +        if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
835 +            psetName = 'pset.py'
836 +        else:
837 +            psetName = 'pset.cfg'
838          # Prepare JobType-independent part
839 <        txt = ''
839 >        txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
840          txt += 'echo ">>> setup environment"\n'
841          txt += 'if [ $middleware == LCG ]; then \n'
842          txt += self.wsSetupCMSLCGEnvironment_()
843          txt += 'elif [ $middleware == OSG ]; then\n'
844          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
845          txt += '    if [ ! $? == 0 ] ;then\n'
846 <        txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
847 <        txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
848 <        txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
821 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
822 <        txt += '        exit 1\n'
846 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
847 >        txt += '        job_exit_code=10016\n'
848 >        txt += '        func_exit\n'
849          txt += '    fi\n'
850          txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
851          txt += '\n'
# Line 827 | Line 853 | class Cmssw(JobType):
853          txt += '    cd $WORKING_DIR\n'
854          txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
855          txt += self.wsSetupCMSOSGEnvironment_()
830 –        #txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
831 –        #txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
856          txt += 'fi\n'
857  
858          # Prepare JobType-specific part
# Line 839 | Line 863 | class Cmssw(JobType):
863          txt += scram+' project CMSSW '+self.version+'\n'
864          txt += 'status=$?\n'
865          txt += 'if [ $status != 0 ] ; then\n'
866 <        txt += '    echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
867 <        txt += '    echo "JOB_EXIT_STATUS = 10034"\n'
868 <        txt += '    echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
845 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
846 <        txt += '    if [ $middleware == OSG ]; then \n'
847 <        txt += '        cd $RUNTIME_AREA\n'
848 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
849 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
850 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
851 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
852 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
853 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
854 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
855 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
856 <        txt += '        fi\n'
857 <        txt += '    fi \n'
858 <        txt += '    exit 1 \n'
866 >        txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
867 >        txt += '    job_exit_code=10034\n'
868 >        txt += '    func_exit\n'
869          txt += 'fi \n'
870          txt += 'cd '+self.version+'\n'
861 –        ########## FEDE FOR DBS2 ######################
871          txt += 'SOFTWARE_DIR=`pwd`\n'
872          txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
864 –        ###############################################
865 –        ### needed grep for bug in scramv1 ###
873          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
874 +        txt += 'if [ $? != 0 ] ; then\n'
875 +        txt += '    echo "ERROR ==> Problem with the command: "\n'
876 +        txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
877 +        txt += '    job_exit_code=10034\n'
878 +        txt += '    func_exit\n'
879 +        txt += 'fi \n'
880          # Handle the arguments:
881          txt += "\n"
882          txt += "## number of arguments (first argument always jobnumber)\n"
883          txt += "\n"
884 <        txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
884 >        txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
885          txt += "then\n"
886 <        txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
887 <        txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
888 <        txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
876 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
877 <        txt += '    if [ $middleware == OSG ]; then \n'
878 <        txt += '        cd $RUNTIME_AREA\n'
879 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
880 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
881 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
882 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
883 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
884 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
885 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
886 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
887 <        txt += '        fi\n'
888 <        txt += '    fi \n'
889 <        txt += "    exit 1\n"
886 >        txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
887 >        txt += '    job_exit_code=50113\n'
888 >        txt += "    func_exit\n"
889          txt += "fi\n"
890          txt += "\n"
891  
892          # Prepare job-specific part
893          job = common.job_list[nj]
895 –        ### FEDE FOR DBS OUTPUT PUBLICATION
894          if (self.datasetPath):
895              txt += '\n'
896              txt += 'DatasetPath='+self.datasetPath+'\n'
# Line 908 | Line 906 | class Cmssw(JobType):
906              txt += 'PrimaryDataset=null\n'
907              txt += 'DataTier=null\n'
908              txt += 'ApplicationFamily=MCDataTier\n'
909 <        if self.pset != None: #CarlosDaniele
909 >        if self.pset != None:
910              pset = os.path.basename(job.configFilename())
911              txt += '\n'
912              txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
913              if (self.datasetPath): # standard job
914 <                txt += 'InputFiles=${args[1]}\n'
915 <                txt += 'MaxEvents=${args[2]}\n'
916 <                txt += 'SkipEvents=${args[3]}\n'
914 >                txt += 'InputFiles=${args[1]}; export InputFiles\n'
915 >                if (self.useParent):  
916 >                    txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
917 >                    txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
918 >                    txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
919 >                else:
920 >                    txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
921 >                    txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
922                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
923 <                txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
923 >                if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
924                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
922 –                txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
925                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
924 –                txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
926              else:  # pythia like job
927 <                seedIndex=1
927 >                txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
928 >                txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
929 >                txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
930 >                txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
931                  if (self.firstRun):
932 <                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
932 >                    txt += 'FirstRun=${args[1]}; export FirstRun\n'
933                      txt += 'echo "FirstRun: <$FirstRun>"\n'
930 –                    txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
931 –                    seedIndex=seedIndex+1
934  
935 <                if (self.sourceSeed):
934 <                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
935 <                    txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
936 <                    seedIndex=seedIndex+1
937 <                    ## the following seeds are not always present
938 <                    if (self.sourceSeedVtx):
939 <                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
940 <                        txt += 'echo "VtxSeed: <$VtxSeed>"\n'
941 <                        txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
942 <                        seedIndex += 1
943 <                    if (self.sourceSeedG4):
944 <                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
945 <                        txt += 'echo "G4Seed: <$G4Seed>"\n'
946 <                        txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
947 <                        seedIndex += 1
948 <                    if (self.sourceSeedMix):
949 <                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
950 <                        txt += 'echo "MixSeed: <$mixSeed>"\n'
951 <                        txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
952 <                        seedIndex += 1
953 <                    pass
954 <                pass
955 <            txt += 'mv -f '+pset+' pset.cfg\n'
935 >            txt += 'mv -f ' + pset + ' ' + psetName + '\n'
936  
957 –        if len(self.additional_inbox_files) > 0:
958 –            txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
959 –            txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
960 –            txt += 'fi\n'
961 –            pass
937  
938 <        if self.pset != None: #CarlosDaniele
938 >        if self.pset != None:
939 >            # FUTURE: Can simply for 2_1_x and higher
940              txt += '\n'
941 <            txt += 'echo "***** cat pset.cfg *********"\n'
942 <            txt += 'cat pset.cfg\n'
943 <            txt += 'echo "****** end pset.cfg ********"\n'
944 <            txt += '\n'
945 <            ### FEDE FOR DBS OUTPUT PUBLICATION
946 <            txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
941 >            if self.debug_wrapper==True:
942 >                txt += 'echo "***** cat ' + psetName + ' *********"\n'
943 >                txt += 'cat ' + psetName + '\n'
944 >                txt += 'echo "****** end ' + psetName + ' ********"\n'
945 >                txt += '\n'
946 >            txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
947              txt += 'echo "PSETHASH = $PSETHASH" \n'
972 –            ##############
948              txt += '\n'
949          return txt
950  
951 <    def wsBuildExe(self, nj=0):
951 >    def wsUntarSoftware(self, nj=0):
952          """
953          Put in the script the commands to build an executable
954          or a library.
955          """
956  
957 <        txt = ""
957 >        txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
958  
959          if os.path.isfile(self.tgzNameWithPath):
960              txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
961              txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
962 +            if  self.debug_wrapper:
963 +                txt += 'ls -Al \n'
964              txt += 'untar_status=$? \n'
965              txt += 'if [ $untar_status -ne 0 ]; then \n'
966 <            txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
967 <            txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
968 <            txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
992 <            txt += '   if [ $middleware == OSG ]; then \n'
993 <            txt += '       cd $RUNTIME_AREA\n'
994 <            txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
995 <            txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
996 <            txt += '       /bin/rm -rf $WORKING_DIR\n'
997 <            txt += '       if [ -d $WORKING_DIR ] ;then\n'
998 <            txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
999 <            txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1000 <            txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1001 <            txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1002 <            txt += '       fi\n'
1003 <            txt += '   fi \n'
1004 <            txt += '   \n'
1005 <            txt += '   exit 1 \n'
966 >            txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
967 >            txt += '   job_exit_code=$untar_status\n'
968 >            txt += '   func_exit\n'
969              txt += 'else \n'
970              txt += '   echo "Successful untar" \n'
971              txt += 'fi \n'
972              txt += '\n'
973 <            #### Removed ProdAgent API dependencies
1011 <            txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
973 >            txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
974              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
975 <            #### FEDE FOR DBS OUTPUT PUBLICATION
1014 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
975 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
976              txt += 'else\n'
977 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
977 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
978              txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1018 –            ###################
979              txt += 'fi\n'
980              txt += '\n'
981  
# Line 1023 | Line 983 | class Cmssw(JobType):
983  
984          return txt
985  
986 <    def modifySteeringCards(self, nj):
986 >    def wsBuildExe(self, nj=0):
987          """
988 <        modify the card provided by the user,
989 <        writing a new card into share dir
988 >        Put in the script the commands to build an executable
989 >        or a library.
990          """
991  
992 +        txt = '\n#Written by cms_cmssw::wsBuildExe\n'
993 +        txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
994 +
995 +        txt += 'rm -r lib/ module/ \n'
996 +        txt += 'mv $RUNTIME_AREA/lib/ . \n'
997 +        txt += 'mv $RUNTIME_AREA/module/ . \n'
998 +        if self.dataExist == True:
999 +            txt += 'rm -r src/ \n'
1000 +            txt += 'mv $RUNTIME_AREA/src/ . \n'
1001 +        if len(self.additional_inbox_files)>0:
1002 +            for file in self.additional_inbox_files:
1003 +                txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1004 +        # txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1005 +        # txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1006 +
1007 +        txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1008 +        txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1009 +        txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1010 +        txt += 'else\n'
1011 +        txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1012 +        txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1013 +        txt += 'fi\n'
1014 +        txt += '\n'
1015 +
1016 +        return txt
1017 +
1018 +
1019      def executableName(self):
1020 <        if self.scriptExe: #CarlosDaniele
1020 >        if self.scriptExe:
1021              return "sh "
1022          else:
1023              return self.executable
1024  
1025      def executableArgs(self):
1026 +        # FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1027          if self.scriptExe:#CarlosDaniele
1028              return   self.scriptExe + " $NJob"
1029          else:
1030 <            # if >= CMSSW_1_5_X, add -j crab_fjr.xml
1031 <            version_array = self.scram.getSWVersion().split('_')
1032 <            major = 0
1033 <            minor = 0
1034 <            try:
1035 <                major = int(version_array[1])
1036 <                minor = int(version_array[2])
1037 <            except:
1050 <                msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1051 <                raise CrabException(msg)
1052 <            if major >= 1 and minor >= 5 :
1053 <                return " -j " + self.fjrFileName + " -p pset.cfg"
1030 >            ex_args = ""
1031 >            # FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1032 >            # Framework job report
1033 >            if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1034 >                ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1035 >            # Type of config file
1036 >            if self.CMSSW_major >= 2 :
1037 >                ex_args += " -p pset.py"
1038              else:
1039 <                return " -p pset.cfg"
1039 >                ex_args += " -p pset.cfg"
1040 >            return ex_args
1041  
1042      def inputSandbox(self, nj):
1043          """
1044          Returns a list of filenames to be put in JDL input sandbox.
1045          """
1046          inp_box = []
1062 –        # # dict added to delete duplicate from input sandbox file list
1063 –        # seen = {}
1064 –        ## code
1047          if os.path.isfile(self.tgzNameWithPath):
1048              inp_box.append(self.tgzNameWithPath)
1049 <        if os.path.isfile(self.MLtgzfile):
1050 <            inp_box.append(self.MLtgzfile)
1069 <        ## config
1070 <        if not self.pset is None:
1071 <            inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1072 <        ## additional input files
1073 <        tgz = self.additionalInputFileTgz()
1074 <        inp_box.append(tgz)
1049 >        wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1050 >        inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1051          return inp_box
1052  
1053      def outputSandbox(self, nj):
# Line 1083 | Line 1059 | class Cmssw(JobType):
1059          ## User Declared output files
1060          for out in (self.output_file+self.output_file_sandbox):
1061              n_out = nj + 1
1062 <            out_box.append(self.numberFile_(out,str(n_out)))
1062 >            out_box.append(numberFile(out,str(n_out)))
1063          return out_box
1064  
1089 –    def prepareSteeringCards(self):
1090 –        """
1091 –        Make initial modifications of the user's steering card file.
1092 –        """
1093 –        return
1065  
1066      def wsRenameOutput(self, nj):
1067          """
1068          Returns part of a job script which renames the produced files.
1069          """
1070  
1071 <        txt = '\n'
1071 >        txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1072          txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1073          txt += 'echo ">>> current directory content:"\n'
1074 <        txt += 'ls \n'
1074 >        if self.debug_wrapper:
1075 >            txt += 'ls -Al\n'
1076          txt += '\n'
1077  
1106 –        txt += 'output_exit_status=0\n'
1107 –
1108 –        for fileWithSuffix in (self.output_file_sandbox):
1109 –            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1110 –            txt += '\n'
1111 –            txt += '# check output file\n'
1112 –            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1113 –            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1114 –            txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1115 –            txt += 'else\n'
1116 –            txt += '    exit_status=60302\n'
1117 –            txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1118 –            if common.scheduler.name() == 'CONDOR_G':
1119 –                txt += '    if [ $middleware == OSG ]; then \n'
1120 –                txt += '        echo "prepare dummy output file"\n'
1121 –                txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1122 –                txt += '    fi \n'
1123 –            txt += 'fi\n'
1124 –
1078          for fileWithSuffix in (self.output_file):
1079 <            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1079 >            output_file_num = numberFile(fileWithSuffix, '$NJob')
1080              txt += '\n'
1081              txt += '# check output file\n'
1082              txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
# Line 1134 | Line 1087 | class Cmssw(JobType):
1087                  txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1088                  txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1089              txt += 'else\n'
1090 <            txt += '    exit_status=60302\n'
1091 <            txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1092 <            txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1140 <            txt += '    output_exit_status=$exit_status\n'
1141 <            if common.scheduler.name() == 'CONDOR_G':
1090 >            txt += '    job_exit_code=60302\n'
1091 >            txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1092 >            if common.scheduler.name().upper() == 'CONDOR_G':
1093                  txt += '    if [ $middleware == OSG ]; then \n'
1094                  txt += '        echo "prepare dummy output file"\n'
1095                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
# Line 1146 | Line 1097 | class Cmssw(JobType):
1097              txt += 'fi\n'
1098          file_list = []
1099          for fileWithSuffix in (self.output_file):
1100 <             file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1100 >             file_list.append(numberFile(fileWithSuffix, '$NJob'))
1101  
1102          txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1103          txt += '\n'
1104          txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1105          txt += 'echo ">>> current directory content:"\n'
1106 <        txt += 'ls \n'
1106 >        if self.debug_wrapper:
1107 >            txt += 'ls -Al\n'
1108          txt += '\n'
1109          txt += 'cd $RUNTIME_AREA\n'
1110          txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1111          return txt
1112  
1161 –    def numberFile_(self, file, txt):
1162 –        """
1163 –        append _'txt' before last extension of a file
1164 –        """
1165 –        p = string.split(file,".")
1166 –        # take away last extension
1167 –        name = p[0]
1168 –        for x in p[1:-1]:
1169 –            name=name+"."+x
1170 –        # add "_txt"
1171 –        if len(p)>1:
1172 –            ext = p[len(p)-1]
1173 –            result = name + '_' + txt + "." + ext
1174 –        else:
1175 –            result = name + '_' + txt
1176 –
1177 –        return result
1178 –
1113      def getRequirements(self, nj=[]):
1114          """
1115          return job requirements to add to jdl files
# Line 1185 | Line 1119 | class Cmssw(JobType):
1119              req='Member("VO-cms-' + \
1120                   self.version + \
1121                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1122 <        ## SL add requirement for OS version only if SL4
1189 <        #reSL4 = re.compile( r'slc4' )
1190 <        if self.executable_arch: # and reSL4.search(self.executable_arch):
1122 >        if self.executable_arch:
1123              req+=' && Member("VO-cms-' + \
1124                   self.executable_arch + \
1125                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1126  
1127          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1128 +        if common.scheduler.name() == "glitecoll":
1129 +            req += ' && other.GlueCEStateStatus == "Production" '
1130  
1131          return req
1132  
1133      def configFilename(self):
1134          """ return the config filename """
1135 <        return self.name()+'.cfg'
1135 >        # FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1136 >        if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1137 >          return self.name()+'.py'
1138 >        else:
1139 >          return self.name()+'.cfg'
1140  
1141      def wsSetupCMSOSGEnvironment_(self):
1142          """
1143          Returns part of a job script which is prepares
1144          the execution environment and which is common for all CMS jobs.
1145          """
1146 <        txt = '    echo ">>> setup CMS OSG environment:"\n'
1146 >        txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1147 >        txt += '    echo ">>> setup CMS OSG environment:"\n'
1148          txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1149          txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1150          txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
# Line 1213 | Line 1152 | class Cmssw(JobType):
1152          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1153          txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1154          txt += '    else\n'
1155 <        txt += '        echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1156 <        txt += '        echo "JOB_EXIT_STATUS = 10020"\n'
1157 <        txt += '        echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1219 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1220 <        txt += '\n'
1221 <        txt += '        cd $RUNTIME_AREA\n'
1222 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1223 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
1224 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
1225 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
1226 <        txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1227 <        txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1228 <        txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1229 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1230 <        txt += '        fi\n'
1231 <        txt += '\n'
1232 <        txt += '        exit 1\n'
1155 >        txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1156 >        txt += '        job_exit_code=10020\n'
1157 >        txt += '        func_exit\n'
1158          txt += '    fi\n'
1159          txt += '\n'
1160 <        txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1160 >        txt += '    echo "==> setup cms environment ok"\n'
1161          txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1162  
1163          return txt
1164  
1240 –    ### OLI_DANIELE
1165      def wsSetupCMSLCGEnvironment_(self):
1166          """
1167          Returns part of a job script which is prepares
1168          the execution environment and which is common for all CMS jobs.
1169          """
1170 <        txt = '    echo ">>> setup CMS LCG environment:"\n'
1170 >        txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1171 >        txt += '    echo ">>> setup CMS LCG environment:"\n'
1172          txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1173          txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1174          txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1175          txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1176 <        txt += '        echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1177 <        txt += '        echo "JOB_EXIT_STATUS = 10031" \n'
1178 <        txt += '        echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1254 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1255 <        txt += '        exit 1\n'
1176 >        txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1177 >        txt += '        job_exit_code=10031\n'
1178 >        txt += '        func_exit\n'
1179          txt += '    else\n'
1180          txt += '        echo "Sourcing environment... "\n'
1181          txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1182 <        txt += '            echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1183 <        txt += '            echo "JOB_EXIT_STATUS = 10020"\n'
1184 <        txt += '            echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1262 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1263 <        txt += '            exit 1\n'
1182 >        txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1183 >        txt += '            job_exit_code=10020\n'
1184 >        txt += '            func_exit\n'
1185          txt += '        fi\n'
1186          txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1187          txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1188          txt += '        result=$?\n'
1189          txt += '        if [ $result -ne 0 ]; then\n'
1190 <        txt += '            echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1191 <        txt += '            echo "JOB_EXIT_STATUS = 10032"\n'
1192 <        txt += '            echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1272 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1273 <        txt += '            exit 1\n'
1190 >        txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1191 >        txt += '            job_exit_code=10032\n'
1192 >        txt += '            func_exit\n'
1193          txt += '        fi\n'
1194          txt += '    fi\n'
1195          txt += '    \n'
1196 <        txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1196 >        txt += '    echo "==> setup cms environment ok"\n'
1197          return txt
1198  
1280 –    ### FEDE FOR DBS OUTPUT PUBLICATION
1199      def modifyReport(self, nj):
1200          """
1201          insert the part of the script that modifies the FrameworkJob Report
1202          """
1203 <
1204 <        txt = ''
1287 <        try:
1288 <            publish_data = int(self.cfg_params['USER.publish_data'])
1289 <        except KeyError:
1290 <            publish_data = 0
1203 >        txt = '\n#Written by cms_cmssw::modifyReport\n'
1204 >        publish_data = int(self.cfg_params.get('USER.publish_data',0))
1205          if (publish_data == 1):
1206 <            txt += 'echo ">>> Modify Job Report:" \n'
1207 <            ################ FEDE FOR DBS2 #############################################
1294 <            #txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1295 <            txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1296 <            #############################################################################
1206 >            processedDataset = self.cfg_params['USER.publish_data_name']
1207 >            LFNBaseName = LFNBase(processedDataset)
1208  
1209 <            txt += 'if [ -z "$SE" ]; then\n'
1210 <            txt += '    SE="" \n'
1211 <            txt += 'fi \n'
1212 <            txt += 'if [ -z "$SE_PATH" ]; then\n'
1213 <            txt += '    SE_PATH="" \n'
1214 <            txt += 'fi \n'
1215 <            txt += 'echo "SE = $SE"\n'
1305 <            txt += 'echo "SE_PATH = $SE_PATH"\n'
1209 >            txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1210 >            txt += '    FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1211 >            txt += 'else\n'
1212 >            txt += '    FOR_LFN=/copy_problems/ \n'
1213 >            txt += '    SE=""\n'
1214 >            txt += '    SE_PATH=""\n'
1215 >            txt += 'fi\n'
1216  
1217 <            processedDataset = self.cfg_params['USER.publish_data_name']
1217 >            txt += 'echo ">>> Modify Job Report:" \n'
1218 >            txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1219              txt += 'ProcessedDataset='+processedDataset+'\n'
1309 –            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1310 –            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1311 –            #### FEDE: added slash in LFN ##############
1312 –            txt += '    FOR_LFN=/copy_problems/ \n'
1313 –            txt += 'else \n'
1314 –            txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1315 –            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1316 –            txt += '    FOR_LFN=/store$tmp \n'
1317 –            txt += 'fi \n'
1220              txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1221 +            txt += 'echo "SE = $SE"\n'
1222 +            txt += 'echo "SE_PATH = $SE_PATH"\n'
1223              txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1224              txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1225 <            txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1226 <            txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1323 <
1225 >            txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1226 >            txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1227              txt += 'modifyReport_result=$?\n'
1325 –            txt += 'echo modifyReport_result = $modifyReport_result\n'
1228              txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1229 <            txt += '    exit_status=1\n'
1230 <            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1229 >            txt += '    modifyReport_result=70500\n'
1230 >            txt += '    job_exit_code=$modifyReport_result\n'
1231 >            txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1232 >            txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1233              txt += 'else\n'
1234 <            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1234 >            txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1235              txt += 'fi\n'
1332 –        else:
1333 –            txt += 'echo "no data publication required"\n'
1236          return txt
1237  
1238 <    def cleanEnv(self):
1239 <        txt = ''
1240 <        txt += 'if [ $middleware == OSG ]; then\n'
1241 <        txt += '    cd $RUNTIME_AREA\n'
1242 <        txt += '    echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1243 <        txt += '    echo ">>> Remove working directory: $WORKING_DIR"\n'
1244 <        txt += '    /bin/rm -rf $WORKING_DIR\n'
1245 <        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1246 <        txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1247 <        txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1248 <        txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1249 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1238 >    def wsParseFJR(self):
1239 >        """
1240 >        Parse the FrameworkJobReport to obtain useful infos
1241 >        """
1242 >        txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1243 >        txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1244 >        txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1245 >        txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1246 >        txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1247 >        if self.debug_wrapper :
1248 >            txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1249 >        txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1250 >        txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1251 >        txt += '            echo ">>> crab_fjr.xml contents: "\n'
1252 >        txt += '            cat $RUNTIME_AREA/crab_fjr_NJob.xml\n'
1253 >        txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1254 >        txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1255 >        txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1256 >        txt += '        else\n'
1257 >        txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1258 >        txt += '        fi\n'
1259 >        txt += '    else\n'
1260 >        txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1261          txt += '    fi\n'
1262 +          #### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1263 +
1264 +        if (self.datasetPath and self.dataset_pu == 'NONE'):
1265 +          # VERIFY PROCESSED DATA
1266 +            txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1267 +            txt += '      echo ">>> Verify list of processed files:"\n'
1268 +            txt += '      echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n'
1269 +            txt += '      python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1270 +            txt += '      cat input-files.txt  | sort | uniq > tmp.txt\n'
1271 +            txt += '      mv tmp.txt input-files.txt\n'
1272 +            txt += '      echo "cat input-files.txt"\n'
1273 +            txt += '      echo "----------------------"\n'
1274 +            txt += '      cat input-files.txt\n'
1275 +            txt += '      cat processed-files.txt | sort | uniq > tmp.txt\n'
1276 +            txt += '      mv tmp.txt processed-files.txt\n'
1277 +            txt += '      echo "----------------------"\n'
1278 +            txt += '      echo "cat processed-files.txt"\n'
1279 +            txt += '      echo "----------------------"\n'
1280 +            txt += '      cat processed-files.txt\n'
1281 +            txt += '      echo "----------------------"\n'
1282 +            txt += '      diff -q input-files.txt processed-files.txt\n'
1283 +            txt += '      fileverify_status=$?\n'
1284 +            txt += '      if [ $fileverify_status -ne 0 ]; then\n'
1285 +            txt += '         executable_exit_status=30001\n'
1286 +            txt += '         echo "ERROR ==> not all input files processed"\n'
1287 +            txt += '         echo "      ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1288 +            txt += '         echo "      ==> diff input-files.txt processed-files.txt"\n'
1289 +            txt += '      fi\n'
1290 +            txt += '    fi\n'
1291 +            txt += '\n'
1292 +        txt += 'else\n'
1293 +        txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1294          txt += 'fi\n'
1295          txt += '\n'
1296 +        txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1297 +        txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1298 +        txt += 'job_exit_code=$executable_exit_status\n'
1299 +
1300          return txt
1301  
1302      def setParam_(self, param, value):
# Line 1365 | Line 1314 | class Cmssw(JobType):
1314              nd[e]=0
1315          return nd.keys()
1316  
1317 <
1369 <    def checkOut(self, limit):
1317 >    def outList(self):
1318          """
1319          check the dimension of the output files
1320          """
1321 <        txt = 'echo ">>> Starting output sandbox limit check :"\n'
1322 <        allOutFiles = ""
1321 >        txt = ''
1322 >        txt += 'echo ">>> list of expected files on output sandbox"\n'
1323          listOutFiles = []
1324 <        txt += 'stdoutFile=`ls *stdout` \n'
1325 <        txt += 'stderrFile=`ls *stderr` \n'
1324 >        stdout = 'CMSSW_$NJob.stdout'
1325 >        stderr = 'CMSSW_$NJob.stderr'
1326          if (self.return_data == 1):
1327 <            for fileOut in (self.output_file+self.output_file_sandbox):
1328 <                allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile"
1329 <        else:            
1330 <            for fileOut in (self.output_file_sandbox):
1331 <                txt += 'echo " '+fileOut+'";\n'
1332 <                allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile"
1333 <        txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1334 <        txt += 'ls -gGhrta;\n'
1335 <        txt += 'sum=0;\n'
1336 <        txt += 'for file in '+str(allOutFiles)+' ; do\n'
1337 <        txt += '    if [ -e $file ]; then\n'
1338 <        txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1391 <        txt += '        sum=`expr $sum + $tt`\n'
1392 <        txt += '    else\n'
1393 <        txt += '        echo "WARNING: output file $file not found!"\n'
1394 <        txt += '    fi\n'
1395 <        txt += 'done\n'
1396 <        txt += 'echo "Total Output dimension: $sum";\n'
1397 <        txt += 'limit='+str(limit)+';\n'
1398 <        txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1399 <        txt += 'if [ $limit -lt $sum ]; then\n'
1400 <        txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1401 <        txt += '    echo "         checking the output file sizes..."\n'
1402 <        txt += '    tot=0;\n'
1403 <        txt += '    for filefile in '+str(allOutFiles)+' ; do\n'
1404 <        txt += '        dimFile=`ls -gGrta $filefile | awk \'{ print $3 }\';`\n'
1405 <        txt += '        tot=`expr $tot + $tt`;\n'
1406 <        txt += '        if [ $limit -lt $dimFile ]; then\n'
1407 <        txt += '            echo "deleting file: $filefile";\n'
1408 <        txt += '            rm -f $filefile\n'
1409 <        txt += '        elif [ $limit -lt $tot ]; then\n'
1410 <        txt += '            echo "deleting file: $filefile";\n'
1411 <        txt += '            rm -f $filefile\n'
1412 <        txt += '        else\n'
1413 <        txt += '            echo "saving file: $filefile"\n'
1414 <        txt += '        fi\n'
1415 <        txt += '    done\n'
1416 <
1417 <        txt += '    ls -agGhrt;\n'
1418 <        txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1419 <        txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1420 <        txt += '    exit_status=70000;\n'
1421 <        txt += 'else'
1422 <        txt += '    echo "Total Output dimension $sum is fine.";\n'
1423 <        txt += 'fi\n'
1424 <        txt += 'echo "Ending output sandbox limit check"\n'
1327 >            for file in (self.output_file+self.output_file_sandbox):
1328 >                listOutFiles.append(numberFile(file, '$NJob'))
1329 >            listOutFiles.append(stdout)
1330 >            listOutFiles.append(stderr)
1331 >        else:
1332 >            for file in (self.output_file_sandbox):
1333 >                listOutFiles.append(numberFile(file, '$NJob'))
1334 >            listOutFiles.append(stdout)
1335 >            listOutFiles.append(stderr)
1336 >        txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1337 >        txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1338 >        txt += 'export filesToCheck\n'
1339          return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines