ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.112 by corvo, Thu Aug 9 16:53:21 2007 UTC vs.
Revision 1.183 by spiga, Wed Apr 30 18:21:07 2008 UTC

# Line 2 | Line 2 | from JobType import JobType
2   from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5 + from BlackWhiteListParser import BlackWhiteListParser
6   import common
7   import Scram
8 + from LFNBaseName import *
9  
10   import os, string, glob
11  
# Line 12 | Line 14 | class Cmssw(JobType):
14          JobType.__init__(self, 'CMSSW')
15          common.logger.debug(3,'CMSSW::__init__')
16  
17 +        self.argsList = []
18 +
19          self._params = {}
20          self.cfg_params = cfg_params
21 +        # init BlackWhiteListParser
22 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23  
24 <        try:
19 <            self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
20 <        except KeyError:
21 <            self.MaxTarBallSize = 9.5
24 >        self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
25  
26          # number of jobs requested to be created, limit obj splitting
27          self.ncjobs = ncjobs
28  
29          log = common.logger
30 <        
30 >
31          self.scram = Scram.Scram(cfg_params)
32          self.additional_inbox_files = []
33          self.scriptExe = ''
34          self.executable = ''
35          self.executable_arch = self.scram.getArch()
36          self.tgz_name = 'default.tgz'
34 –        self.additional_tgz_name = 'additional.tgz'
37          self.scriptName = 'CMSSW.sh'
38 <        self.pset = ''      #scrip use case Da  
38 >        self.pset = ''      #scrip use case Da
39          self.datasetPath = '' #scrip use case Da
40  
41          # set FJR file name
42          self.fjrFileName = 'crab_fjr.xml'
43  
44          self.version = self.scram.getSWVersion()
45 <        
45 >        version_array = self.version.split('_')
46 >        self.major_version = 0
47 >        self.minor_version = 0
48 >        try:
49 >            self.major_version = int(version_array[1])
50 >            self.minor_version = int(version_array[2])
51 >        except:
52 >            msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
53 >            raise CrabException(msg)
54 >
55 >
56          #
57          # Try to block creation in case of arch/version mismatch
58          #
59  
60 <        a = string.split(self.version, "_")
60 > #        a = string.split(self.version, "_")
61 > #
62 > #        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
63 > #            msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
64 > #            common.logger.message(msg)
65 > #        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
66 > #            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
67 > #            raise CrabException(msg)
68 > #
69  
50 –        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
51 –            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
52 –            raise CrabException(msg)
53 –        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
54 –            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
55 –            raise CrabException(msg)
56 –        
57 –        common.taskDB.setDict('codeVersion',self.version)
58 –        self.setParam_('application', self.version)
70  
71          ### collect Data cards
72  
73 <        ## get DBS mode
74 <        try:
64 <            self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
65 <        except KeyError:
66 <            self.use_dbs_1 = 0
67 <            
68 <        try:
69 <            tmp =  cfg_params['CMSSW.datasetpath']
70 <            log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
71 <            if string.lower(tmp)=='none':
72 <                self.datasetPath = None
73 <                self.selectNoInput = 1
74 <            else:
75 <                self.datasetPath = tmp
76 <                self.selectNoInput = 0
77 <        except KeyError:
78 <            msg = "Error: datasetpath not defined "  
73 >        if not cfg_params.has_key('CMSSW.datasetpath'):
74 >            msg = "Error: datasetpath not defined "
75              raise CrabException(msg)
76 <
77 <        # ML monitoring
78 <        # split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
79 <        if not self.datasetPath:
80 <            self.setParam_('dataset', 'None')
85 <            self.setParam_('owner', 'None')
76 >        tmp =  cfg_params['CMSSW.datasetpath']
77 >        log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
78 >        if string.lower(tmp)=='none':
79 >            self.datasetPath = None
80 >            self.selectNoInput = 1
81          else:
82 <            try:
83 <                datasetpath_split = self.datasetPath.split("/")
89 <                # standard style
90 <                if self.use_dbs_1 == 1 :
91 <                    self.setParam_('dataset', datasetpath_split[1])
92 <                    self.setParam_('owner', datasetpath_split[-1])
93 <                else:
94 <                    self.setParam_('dataset', datasetpath_split[1])
95 <                    self.setParam_('owner', datasetpath_split[2])
96 <            except:
97 <                self.setParam_('dataset', self.datasetPath)
98 <                self.setParam_('owner', self.datasetPath)
99 <                
100 <        self.setTaskid_()
101 <        self.setParam_('taskId', self.cfg_params['taskId'])
82 >            self.datasetPath = tmp
83 >            self.selectNoInput = 0
84  
85          self.dataTiers = []
86  
87          ## now the application
88 <        try:
89 <            self.executable = cfg_params['CMSSW.executable']
108 <            self.setParam_('exe', self.executable)
109 <            log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
110 <            msg = "Default executable cmsRun overridden. Switch to " + self.executable
111 <            log.debug(3,msg)
112 <        except KeyError:
113 <            self.executable = 'cmsRun'
114 <            self.setParam_('exe', self.executable)
115 <            msg = "User executable not defined. Use cmsRun"
116 <            log.debug(3,msg)
117 <            pass
88 >        self.executable = cfg_params.get('CMSSW.executable','cmsRun')
89 >        log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
90  
91 <        try:
120 <            self.pset = cfg_params['CMSSW.pset']
121 <            log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
122 <            if self.pset.lower() != 'none' :
123 <                if (not os.path.exists(self.pset)):
124 <                    raise CrabException("User defined PSet file "+self.pset+" does not exist")
125 <            else:
126 <                self.pset = None
127 <        except KeyError:
91 >        if not cfg_params.has_key('CMSSW.pset'):
92              raise CrabException("PSet file missing. Cannot run cmsRun ")
93 +        self.pset = cfg_params['CMSSW.pset']
94 +        log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
95 +        if self.pset.lower() != 'none' :
96 +            if (not os.path.exists(self.pset)):
97 +                raise CrabException("User defined PSet file "+self.pset+" does not exist")
98 +        else:
99 +            self.pset = None
100  
101          # output files
102          ## stuff which must be returned always via sandbox
# Line 135 | Line 106 | class Cmssw(JobType):
106          self.output_file_sandbox.append(self.fjrFileName)
107  
108          # other output files to be returned via sandbox or copied to SE
109 <        try:
110 <            self.output_file = []
111 <            tmp = cfg_params['CMSSW.output_file']
112 <            if tmp != '':
113 <                tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
114 <                log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
115 <                for tmp in tmpOutFiles:
116 <                    tmp=string.strip(tmp)
146 <                    self.output_file.append(tmp)
147 <                    pass
148 <            else:
149 <                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
109 >        self.output_file = []
110 >        tmp = cfg_params.get('CMSSW.output_file',None)
111 >        if tmp :
112 >            tmpOutFiles = string.split(tmp,',')
113 >            log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
114 >            for tmp in tmpOutFiles:
115 >                tmp=string.strip(tmp)
116 >                self.output_file.append(tmp)
117                  pass
118 <            pass
152 <        except KeyError:
118 >        else:
119              log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
120 <            pass
120 >        pass
121  
122          # script_exe file as additional file in inputSandbox
123 <        try:
124 <            self.scriptExe = cfg_params['USER.script_exe']
125 <            if self.scriptExe != '':
126 <               if not os.path.isfile(self.scriptExe):
127 <                  msg ="ERROR. file "+self.scriptExe+" not found"
128 <                  raise CrabException(msg)
163 <               self.additional_inbox_files.append(string.strip(self.scriptExe))
164 <        except KeyError:
165 <            self.scriptExe = ''
123 >        self.scriptExe = cfg_params.get('USER.script_exe',None)
124 >        if self.scriptExe :
125 >            if not os.path.isfile(self.scriptExe):
126 >                msg ="ERROR. file "+self.scriptExe+" not found"
127 >                raise CrabException(msg)
128 >            self.additional_inbox_files.append(string.strip(self.scriptExe))
129  
130          #CarlosDaniele
131          if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
132 <           msg ="Error. script_exe  not defined"
133 <           raise CrabException(msg)
132 >            msg ="Error. script_exe  not defined"
133 >            raise CrabException(msg)
134  
135          ## additional input files
136 <        try:
136 >        if cfg_params.has_key('USER.additional_input_files'):
137              tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
138              for tmp in tmpAddFiles:
139                  tmp = string.strip(tmp)
# Line 194 | Line 157 | class Cmssw(JobType):
157                  pass
158              pass
159              common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
160 <        except KeyError:
198 <            pass
199 <
200 <        # files per job
201 <        try:
202 <            if (cfg_params['CMSSW.files_per_jobs']):
203 <                raise CrabException("files_per_jobs no longer supported.  Quitting.")
204 <        except KeyError:
205 <            pass
160 >        pass
161  
162          ## Events per job
163 <        try:
163 >        if cfg_params.has_key('CMSSW.events_per_job'):
164              self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
165              self.selectEventsPerJob = 1
166 <        except KeyError:
166 >        else:
167              self.eventsPerJob = -1
168              self.selectEventsPerJob = 0
169 <    
169 >
170          ## number of jobs
171 <        try:
171 >        if cfg_params.has_key('CMSSW.number_of_jobs'):
172              self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
173              self.selectNumberOfJobs = 1
174 <        except KeyError:
174 >        else:
175              self.theNumberOfJobs = 0
176              self.selectNumberOfJobs = 0
177  
178 <        try:
178 >        if cfg_params.has_key('CMSSW.total_number_of_events'):
179              self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
180              self.selectTotalNumberEvents = 1
181 <        except KeyError:
181 >        else:
182              self.total_number_of_events = 0
183              self.selectTotalNumberEvents = 0
184  
185 <        if self.pset != None: #CarlosDaniele
185 >        if self.pset != None: #CarlosDaniele
186               if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
187                   msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
188                   raise CrabException(msg)
# Line 236 | Line 191 | class Cmssw(JobType):
191                   msg = 'Must specify  number_of_jobs.'
192                   raise CrabException(msg)
193  
194 <        ## source seed for pythia
195 <        try:
196 <            self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
197 <        except KeyError:
198 <            self.sourceSeed = None
199 <            common.logger.debug(5,"No seed given")
200 <
201 <        try:
202 <            self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
203 <        except KeyError:
204 <            self.sourceSeedVtx = None
205 <            common.logger.debug(5,"No vertex seed given")
206 <
207 <        try:
208 <            self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
209 <        except KeyError:
210 <            self.sourceSeedG4 = None
211 <            common.logger.debug(5,"No g4 sim hits seed given")
194 >        ## New method of dealing with seeds
195 >        self.incrementSeeds = []
196 >        self.preserveSeeds = []
197 >        if cfg_params.has_key('CMSSW.preserve_seeds'):
198 >            tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
199 >            for tmp in tmpList:
200 >                tmp.strip()
201 >                self.preserveSeeds.append(tmp)
202 >        if cfg_params.has_key('CMSSW.increment_seeds'):
203 >            tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
204 >            for tmp in tmpList:
205 >                tmp.strip()
206 >                self.incrementSeeds.append(tmp)
207 >
208 >        ## Old method of dealing with seeds
209 >        ## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
210 >        ## remove
211 >        self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
212 >        if self.sourceSeed:
213 >            print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
214 >            self.incrementSeeds.append('sourceSeed')
215 >
216 >        self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
217 >        if self.sourceSeedVtx:
218 >            print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
219 >            self.incrementSeeds.append('VtxSmeared')
220 >
221 >        self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
222 >        if self.sourceSeedG4:
223 >            print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
224 >            self.incrementSeeds.append('g4SimHits')
225 >
226 >        self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
227 >        if self.sourceSeedMix:
228 >            print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
229 >            self.incrementSeeds.append('mix')
230  
231 <        try:
259 <            self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
260 <        except KeyError:
261 <            self.sourceSeedMix = None
262 <            common.logger.debug(5,"No mix seed given")
231 >        self.firstRun = cfg_params.get('CMSSW.first_run',None)
232  
264 –        try:
265 –            self.firstRun = int(cfg_params['CMSSW.first_run'])
266 –        except KeyError:
267 –            self.firstRun = None
268 –            common.logger.debug(5,"No first run given")
233          if self.pset != None: #CarlosDaniele
234 <            ver = string.split(self.version,"_")
271 <            if (int(ver[1])>=1 and int(ver[2])>=5):
272 <                import PsetManipulator150 as pp
273 <            else:
274 <                import PsetManipulator as pp
234 >            import PsetManipulator as pp
235              PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
236  
237 +        # Copy/return
238 +
239 +        self.copy_data = int(cfg_params.get('USER.copy_data',0))
240 +        self.return_data = int(cfg_params.get('USER.return_data',0))
241 +
242          #DBSDLS-start
243 <        ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
243 >        ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
244          self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
245          self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
246          self.jobDestination=[]  # Site destination(s) for each job (list of lists)
# Line 284 | Line 249 | class Cmssw(JobType):
249          blockSites = {}
250          if self.datasetPath:
251              blockSites = self.DataDiscoveryAndLocation(cfg_params)
252 <        #DBSDLS-end          
252 >        #DBSDLS-end
253 >
254  
289 –        self.tgzNameWithPath = self.getTarBall(self.executable)
290 –    
255          ## Select Splitting
256 <        if self.selectNoInput:
256 >        if self.selectNoInput:
257              if self.pset == None: #CarlosDaniele
258                  self.jobSplittingForScript()
259              else:
# Line 300 | Line 264 | class Cmssw(JobType):
264          # modify Pset
265          if self.pset != None: #CarlosDaniele
266              try:
267 <                if (self.datasetPath): # standard job
268 <                    # allow to processa a fraction of events in a file
269 <                    PsetEdit.inputModule("INPUT")
270 <                    PsetEdit.maxEvent("INPUTMAXEVENTS")
307 <                    PsetEdit.skipEvent("INPUTSKIPEVENTS")
308 <                else:  # pythia like job
309 <                    PsetEdit.maxEvent(self.eventsPerJob)
310 <                    if (self.firstRun):
311 <                        PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
312 <                    if (self.sourceSeed) :
313 <                        PsetEdit.pythiaSeed("INPUT")
314 <                        if (self.sourceSeedVtx) :
315 <                            PsetEdit.vtxSeed("INPUTVTX")
316 <                        if (self.sourceSeedG4) :
317 <                            self.PsetEdit.g4Seed("INPUTG4")
318 <                        if (self.sourceSeedMix) :
319 <                            self.PsetEdit.mixSeed("INPUTMIX")
320 <                # add FrameworkJobReport to parameter-set
321 <                PsetEdit.addCrabFJR(self.fjrFileName)
267 >                # Add FrameworkJobReport to parameter-set, set max events.
268 >                # Reset later for data jobs by writeCFG which does all modifications
269 >                PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
270 >                PsetEdit.maxEvent(self.eventsPerJob)
271                  PsetEdit.psetWriter(self.configFilename())
272              except:
273                  msg='Error while manipuliating ParameterSet: exiting...'
274                  raise CrabException(msg)
275 +        self.tgzNameWithPath = self.getTarBall(self.executable)
276  
277      def DataDiscoveryAndLocation(self, cfg_params):
278  
279          import DataDiscovery
330 –        import DataDiscovery_DBS2
280          import DataLocation
281          common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
282  
# Line 336 | Line 285 | class Cmssw(JobType):
285          ## Contact the DBS
286          common.logger.message("Contacting Data Discovery Services ...")
287          try:
288 <
340 <            if self.use_dbs_1 == 1 :
341 <                self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
342 <            else :
343 <                self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
288 >            self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
289              self.pubdata.fetchDBSInfo()
290  
291          except DataDiscovery.NotExistingDatasetError, ex :
# Line 352 | Line 297 | class Cmssw(JobType):
297          except DataDiscovery.DataDiscoveryError, ex:
298              msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
299              raise CrabException(msg)
355 –        except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
356 –            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
357 –            raise CrabException(msg)
358 –        except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
359 –            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
360 –            raise CrabException(msg)
361 –        except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
362 –            msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
363 –            raise CrabException(msg)
300  
301          self.filesbyblock=self.pubdata.getFiles()
302          self.eventsbyblock=self.pubdata.getEventsPerBlock()
303          self.eventsbyfile=self.pubdata.getEventsPerFile()
304  
305          ## get max number of events
306 <        self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
306 >        self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
307  
308          ## Contact the DLS and build a list of sites hosting the fileblocks
309          try:
# Line 376 | Line 312 | class Cmssw(JobType):
312          except DataLocation.DataLocationError , ex:
313              msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
314              raise CrabException(msg)
315 <        
315 >
316  
317          sites = dataloc.getSites()
318          allSites = []
# Line 390 | Line 326 | class Cmssw(JobType):
326          common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
327  
328          return sites
329 <    
329 >
330 >  # to Be Removed  DS -- BL
331 >  #  def setArgsList(self, argsList):
332 >  #      self.argsList = argsList
333 >
334      def jobSplittingByBlocks(self, blockSites):
335          """
336          Perform job splitting. Jobs run over an integer number of files
# Line 440 | Line 380 | class Cmssw(JobType):
380              totalNumberOfJobs = 999999999
381          else :
382              totalNumberOfJobs = self.ncjobs
443 –            
383  
384          blocks = blockSites.keys()
385          blockCount = 0
# Line 460 | Line 399 | class Cmssw(JobType):
399              blockCount += 1
400              if block not in jobsOfBlock.keys() :
401                  jobsOfBlock[block] = []
402 <            
402 >
403              if self.eventsbyblock.has_key(block) :
404                  numEventsInBlock = self.eventsbyblock[block]
405                  common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
406 <            
406 >
407                  files = self.filesbyblock[block]
408                  numFilesInBlock = len(files)
409                  if (numFilesInBlock <= 0):
# Line 472 | Line 411 | class Cmssw(JobType):
411                  fileCount = 0
412  
413                  # ---- New block => New job ---- #
414 <                parString = "\\{"
414 >                parString = ""
415                  # counter for number of events in files currently worked on
416                  filesEventCount = 0
417                  # flag if next while loop should touch new file
418                  newFile = 1
419                  # job event counter
420                  jobSkipEventCount = 0
421 <            
421 >
422                  # ---- Iterate over the files in the block until we've met the requested ---- #
423                  # ---- total # of events or we've gone over all the files in this block  ---- #
424                  while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
# Line 495 | Line 434 | class Cmssw(JobType):
434                              newFile = 0
435                          except KeyError:
436                              common.logger.message("File "+str(file)+" has unknown number of events: skipping")
498 –                        
437  
438 +                    eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
439                      # if less events in file remain than eventsPerJobRequested
440 <                    if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
440 >                    if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
441                          # if last file in block
442                          if ( fileCount == numFilesInBlock-1 ) :
443                              # end job using last file, use remaining events in block
444                              # close job and touch new file
445                              fullString = parString[:-2]
507 –                            fullString += '\\}'
446                              list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
447                              common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
448                              self.jobDestination.append(blockSites[block])
# Line 517 | Line 455 | class Cmssw(JobType):
455                              eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
456                              jobSkipEventCount = 0
457                              # reset file
458 <                            parString = "\\{"
458 >                            parString = ""
459                              filesEventCount = 0
460                              newFile = 1
461                              fileCount += 1
# Line 529 | Line 467 | class Cmssw(JobType):
467                      elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
468                          # close job and touch new file
469                          fullString = parString[:-2]
532 –                        fullString += '\\}'
470                          list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
471                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
472                          self.jobDestination.append(blockSites[block])
# Line 541 | Line 478 | class Cmssw(JobType):
478                          eventsRemaining = eventsRemaining - eventsPerJobRequested
479                          jobSkipEventCount = 0
480                          # reset file
481 <                        parString = "\\{"
481 >                        parString = ""
482                          filesEventCount = 0
483                          newFile = 1
484                          fileCount += 1
485 <                        
485 >
486                      # if more events in file remain than eventsPerJobRequested
487                      else :
488                          # close job but don't touch new file
489                          fullString = parString[:-2]
553 –                        fullString += '\\}'
490                          list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
491                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
492                          self.jobDestination.append(blockSites[block])
# Line 565 | Line 501 | class Cmssw(JobType):
501                          jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
502                          # remove all but the last file
503                          filesEventCount = self.eventsbyfile[file]
504 <                        parString = "\\{"
569 <                        parString += '\\\"' + file + '\\\"\,'
504 >                        parString = '\\\"' + file + '\\\"\,'
505                      pass # END if
506                  pass # END while (iterate over files in the block)
507          pass # END while (iterate over blocks in the dataset)
# Line 574 | Line 509 | class Cmssw(JobType):
509          if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
510              common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
511          common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
512 <        
512 >
513          # screen output
514          screenOutput = "List of jobs and available destination sites:\n\n"
515  
516 +        # keep trace of block with no sites to print a warning at the end
517 +        noSiteBlock = []
518 +        bloskNoSite = []
519 +
520          blockCounter = 0
521          for block in blocks:
522              if block in jobsOfBlock.keys() :
523                  blockCounter += 1
524 <                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
524 >                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
525 >                    ','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
526 >                if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
527 >                    noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
528 >                    bloskNoSite.append( blockCounter )
529 >
530 >        common.logger.message(screenOutput)
531 >        if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
532 >            msg = 'WARNING: No sites are hosting any part of data for block:\n                '
533 >            virgola = ""
534 >            if len(bloskNoSite) > 1:
535 >                virgola = ","
536 >            for block in bloskNoSite:
537 >                msg += ' ' + str(block) + virgola
538 >            msg += '\n               Related jobs:\n                 '
539 >            virgola = ""
540 >            if len(noSiteBlock) > 1:
541 >                virgola = ","
542 >            for range_jobs in noSiteBlock:
543 >                msg += str(range_jobs) + virgola
544 >            msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
545 >            if self.cfg_params.has_key('EDG.se_white_list'):
546 >                msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
547 >                msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
548 >                msg += 'Please check if the dataset is available at this site!)\n'
549 >            if self.cfg_params.has_key('EDG.ce_white_list'):
550 >                msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
551 >                msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
552 >                msg += 'Please check if the dataset is available at this site!)\n'
553  
554 <       # common.logger.message(screenOutput)
554 >            common.logger.message(msg)
555  
556          self.list_of_args = list_of_lists
557          return
# Line 594 | Line 561 | class Cmssw(JobType):
561          Perform job splitting based on number of event per job
562          """
563          common.logger.debug(5,'Splitting per events')
564 <        common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
565 <        common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
566 <        common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
564 >
565 >        if (self.selectEventsPerJob):
566 >            common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
567 >        if (self.selectNumberOfJobs):
568 >            common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
569 >        if (self.selectTotalNumberEvents):
570 >            common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
571  
572          if (self.total_number_of_events < 0):
573              msg='Cannot split jobs per Events with "-1" as total number of events'
# Line 605 | Line 576 | class Cmssw(JobType):
576          if (self.selectEventsPerJob):
577              if (self.selectTotalNumberEvents):
578                  self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
579 <            elif(self.selectNumberOfJobs) :  
579 >            elif(self.selectNumberOfJobs) :
580                  self.total_number_of_jobs =self.theNumberOfJobs
581 <                self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
581 >                self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
582  
583          elif (self.selectNumberOfJobs) :
584              self.total_number_of_jobs = self.theNumberOfJobs
585              self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
586 <
586 >
587          common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
588  
589          # is there any remainder?
# Line 628 | Line 599 | class Cmssw(JobType):
599          self.list_of_args = []
600          for i in range(self.total_number_of_jobs):
601              ## Since there is no input, any site is good
602 <           # self.jobDestination.append(["Any"])
632 <            self.jobDestination.append([""]) #must be empty to write correctly the xml
602 >            self.jobDestination.append([""]) #must be empty to write correctly the xml
603              args=[]
604              if (self.firstRun):
605 <                    ## pythia first run
636 <                #self.list_of_args.append([(str(self.firstRun)+str(i))])
605 >                ## pythia first run
606                  args.append(str(self.firstRun)+str(i))
638 –            else:
639 –                ## no first run
640 –                #self.list_of_args.append([str(i)])
641 –                args.append(str(i))
642 –            if (self.sourceSeed):
643 –                args.append(str(self.sourceSeed)+str(i))
644 –                if (self.sourceSeedVtx):
645 –                    ## + vtx random seed
646 –                    args.append(str(self.sourceSeedVtx)+str(i))
647 –                if (self.sourceSeedG4):
648 –                    ## + G4 random seed
649 –                    args.append(str(self.sourceSeedG4)+str(i))
650 –                if (self.sourceSeedMix):    
651 –                    ## + Mix random seed
652 –                    args.append(str(self.sourceSeedMix)+str(i))
653 –                pass
654 –            pass
607              self.list_of_args.append(args)
656 –        pass
657 –            
658 –        # print self.list_of_args
608  
609          return
610  
# Line 684 | Line 633 | class Cmssw(JobType):
633          return
634  
635      def split(self, jobParams):
636 <
688 <        common.jobDB.load()
636 >
637          #### Fabio
638          njobs = self.total_number_of_jobs
639          arglist = self.list_of_args
640          # create the empty structure
641          for i in range(njobs):
642              jobParams.append("")
643 <        
643 >
644 >        listID=[]
645 >        listField=[]
646          for job in range(njobs):
647              jobParams[job] = arglist[job]
648 <            # print str(arglist[job])
649 <            # print jobParams[job]
650 <            common.jobDB.setArguments(job, jobParams[job])
651 <            common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
652 <            common.jobDB.setDestination(job, self.jobDestination[job])
648 >            listID.append(job+1)
649 >            job_ToSave ={}
650 >            concString = ' '
651 >            argu=''
652 >            if len(jobParams[job]):
653 >                argu +=   concString.join(jobParams[job] )
654 >            job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS
655 >            job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS
656 >            #common._db.updateJob_(job,job_ToSave)## new BL--DS
657 >            listField.append(job_ToSave)
658 >            msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
659 >            +"                     Destination: "+str(self.jobDestination[job])
660 >            common.logger.debug(5,msg)
661 >            #common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
662 >        common._db.updateJob_(listID,listField)## new BL--DS
663 >        ## Pay Attention Here....DS--BL
664 >        self.argsList = (len(jobParams[0])+1)
665  
704 –        common.jobDB.save()
666          return
667 <    
707 <    def getJobTypeArguments(self, nj, sched):
708 <        result = ''
709 <        for i in common.jobDB.arguments(nj):
710 <            result=result+str(i)+" "
711 <        return result
712 <  
667 >
668      def numberOfJobs(self):
669          # Fabio
670          return self.total_number_of_jobs
# Line 718 | Line 673 | class Cmssw(JobType):
673          """
674          Return the TarBall with lib and exe
675          """
676 <        
676 >
677          # if it exist, just return it
678          #
679          # Marco. Let's start to use relative path for Boss XML files
# Line 741 | Line 696 | class Cmssw(JobType):
696          # print "swVersion = ", swVersion
697          swReleaseTop = self.scram.getReleaseTop_()
698          #print "swReleaseTop = ", swReleaseTop
699 <        
699 >
700          ## check if working area is release top
701          if swReleaseTop == '' or swArea == swReleaseTop:
702 +            common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
703              return
704  
705          import tarfile
# Line 754 | Line 710 | class Cmssw(JobType):
710                  exeWithPath = self.scram.findFile_(executable)
711                  if ( not exeWithPath ):
712                      raise CrabException('User executable '+executable+' not found')
713 <    
713 >
714                  ## then check if it's private or not
715                  if exeWithPath.find(swReleaseTop) == -1:
716                      # the exe is private, so we must ship
# Line 763 | Line 719 | class Cmssw(JobType):
719                      # distinguish case when script is in user project area or given by full path somewhere else
720                      if exeWithPath.find(path) >= 0 :
721                          exe = string.replace(exeWithPath, path,'')
722 <                        tar.add(path+exe,os.path.basename(executable))
722 >                        tar.add(path+exe,exe)
723                      else :
724                          tar.add(exeWithPath,os.path.basename(executable))
725                      pass
726                  else:
727                      # the exe is from release, we'll find it on WN
728                      pass
729 <    
729 >
730              ## Now get the libraries: only those in local working area
731              libDir = 'lib'
732              lib = swArea+'/' +libDir
733              common.logger.debug(5,"lib "+lib+" to be tarred")
734              if os.path.exists(lib):
735                  tar.add(lib,libDir)
736 <    
736 >
737              ## Now check if module dir is present
738              moduleDir = 'module'
739              module = swArea + '/' + moduleDir
# Line 786 | Line 742 | class Cmssw(JobType):
742  
743              ## Now check if any data dir(s) is present
744              swAreaLen=len(swArea)
745 +            self.dataExist = False
746              for root, dirs, files in os.walk(swArea):
747                  if "data" in dirs:
748 +                    self.dataExist=True
749                      common.logger.debug(5,"data "+root+"/data"+" to be tarred")
750                      tar.add(root+"/data",root[swAreaLen:]+"/data")
751  
752 <            ## Add ProdAgent dir to tar
753 <            paDir = 'ProdAgentApi'
754 <            pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
755 <            if os.path.isdir(pa):
756 <                tar.add(pa,paDir)
752 >            ### CMSSW ParameterSet
753 >            if not self.pset is None:
754 >                cfg_file = common.work_space.jobDir()+self.configFilename()
755 >                tar.add(cfg_file,self.configFilename())
756 >                common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
757 >
758  
759 <            ### FEDE FOR DBS PUBLICATION
801 <            ## Add PRODCOMMON dir to tar
759 >            ## Add ProdCommon dir to tar
760              prodcommonDir = 'ProdCommon'
761              prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
762              if os.path.isdir(prodcommonPath):
763                  tar.add(prodcommonPath,prodcommonDir)
806 –            #############################    
807 –        
764              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
765 +
766 +            ##### ML stuff
767 +            ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
768 +            path=os.environ['CRABDIR'] + '/python/'
769 +            for file in ML_file_list:
770 +                tar.add(path+file,file)
771 +            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
772 +
773 +            ##### Utils
774 +            Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']
775 +            for file in Utils_file_list:
776 +                tar.add(path+file,file)
777 +            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
778 +
779 +            ##### AdditionalFiles
780 +            for file in self.additional_inbox_files:
781 +                tar.add(file,string.split(file,'/')[-1])
782 +            common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
783 +
784              tar.close()
785          except :
786              raise CrabException('Could not create tar-ball')
# Line 816 | Line 791 | class Cmssw(JobType):
791              raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
792  
793          ## create tar-ball with ML stuff
819 –        self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
820 –        try:
821 –            tar = tarfile.open(self.MLtgzfile, "w:gz")
822 –            path=os.environ['CRABDIR'] + '/python/'
823 –            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
824 –                tar.add(path+file,file)
825 –            common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
826 –            tar.close()
827 –        except :
828 –            raise CrabException('Could not create ML files tar-ball')
829 –        
830 –        return
831 –        
832 –    def additionalInputFileTgz(self):
833 –        """
834 –        Put all additional files into a tar ball and return its name
835 –        """
836 –        import tarfile
837 –        tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
838 –        tar = tarfile.open(tarName, "w:gz")
839 –        for file in self.additional_inbox_files:
840 –            tar.add(file,string.split(file,'/')[-1])
841 –        common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
842 –        tar.close()
843 –        return tarName
794  
795 <    def wsSetupEnvironment(self, nj):
795 >    def wsSetupEnvironment(self, nj=0):
796          """
797          Returns part of a job script which prepares
798          the execution environment for the job 'nj'.
799          """
800          # Prepare JobType-independent part
801 <        txt = ''
802 <  
803 <        ## OLI_Daniele at this level  middleware already known
854 <
855 <        txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
856 <        txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
857 <        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
858 <        txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
859 <        txt += 'if [ $middleware == LCG ]; then \n'
801 >        txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
802 >        txt += 'echo ">>> setup environment"\n'
803 >        txt += 'if [ $middleware == LCG ]; then \n'
804          txt += self.wsSetupCMSLCGEnvironment_()
805          txt += 'elif [ $middleware == OSG ]; then\n'
806          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
807 <        txt += '    echo "Created working directory: $WORKING_DIR"\n'
808 <        txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
809 <        txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
810 <        txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
867 <        txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
868 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
869 <        txt += '        rm -f $RUNTIME_AREA/$repo \n'
870 <        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
871 <        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
872 <        txt += '        exit 1\n'
807 >        txt += '    if [ ! $? == 0 ] ;then\n'
808 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
809 >        txt += '        job_exit_code=10016\n'
810 >        txt += '        func_exit\n'
811          txt += '    fi\n'
812 +        txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
813          txt += '\n'
814          txt += '    echo "Change to working directory: $WORKING_DIR"\n'
815          txt += '    cd $WORKING_DIR\n'
816 <        txt += self.wsSetupCMSOSGEnvironment_()
816 >        txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
817 >        txt += self.wsSetupCMSOSGEnvironment_()
818 >        #txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
819 >        #txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
820          txt += 'fi\n'
821  
822          # Prepare JobType-specific part
823          scram = self.scram.commandName()
824          txt += '\n\n'
825 <        txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
825 >        txt += 'echo ">>> specific cmssw setup environment:"\n'
826 >        txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
827          txt += scram+' project CMSSW '+self.version+'\n'
828          txt += 'status=$?\n'
829          txt += 'if [ $status != 0 ] ; then\n'
830 <        txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
831 <        txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
832 <        txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
890 <        txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
891 <        txt += '   rm -f $RUNTIME_AREA/$repo \n'
892 <        txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
893 <        txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
894 <        ## OLI_Daniele
895 <        txt += '    if [ $middleware == OSG ]; then \n'
896 <        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
897 <        txt += '        cd $RUNTIME_AREA\n'
898 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
899 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
900 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
901 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
902 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
903 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
904 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
905 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
906 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
907 <        txt += '        fi\n'
908 <        txt += '    fi \n'
909 <        txt += '   exit 1 \n'
830 >        txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
831 >        txt += '    job_exit_code=10034\n'
832 >        txt += '    func_exit\n'
833          txt += 'fi \n'
911 –        txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
834          txt += 'cd '+self.version+'\n'
913 –        ########## FEDE FOR DBS2 ######################
835          txt += 'SOFTWARE_DIR=`pwd`\n'
836 <        txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
916 <        ###############################################
917 <        ### needed grep for bug in scramv1 ###
918 <        txt += scram+' runtime -sh\n'
836 >        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
837          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
838 <        txt += 'echo $PATH\n'
839 <
838 >        txt += 'if [ $? != 0 ] ; then\n'
839 >        txt += '    echo "ERROR ==> Problem with the command: "\n'
840 >        txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
841 >        txt += '    job_exit_code=10034\n'
842 >        txt += '    func_exit\n'
843 >        txt += 'fi \n'
844          # Handle the arguments:
845          txt += "\n"
846          txt += "## number of arguments (first argument always jobnumber)\n"
847          txt += "\n"
848 < #        txt += "narg=$#\n"
927 <        txt += "if [ $nargs -lt 2 ]\n"
848 >        txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
849          txt += "then\n"
850 <        txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
851 <        txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
852 <        txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
932 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
933 <        txt += '    rm -f $RUNTIME_AREA/$repo \n'
934 <        txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
935 <        txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
936 <        ## OLI_Daniele
937 <        txt += '    if [ $middleware == OSG ]; then \n'
938 <        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
939 <        txt += '        cd $RUNTIME_AREA\n'
940 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
941 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
942 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
943 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
944 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
945 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
946 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
947 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
948 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
949 <        txt += '        fi\n'
950 <        txt += '    fi \n'
951 <        txt += "    exit 1\n"
850 >        txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
851 >        txt += '    job_exit_code=50113\n'
852 >        txt += "    func_exit\n"
853          txt += "fi\n"
854          txt += "\n"
855  
856          # Prepare job-specific part
857          job = common.job_list[nj]
858          ### FEDE FOR DBS OUTPUT PUBLICATION
859 <        if (self.datasetPath):
859 >        if (self.datasetPath):
860              txt += '\n'
861              txt += 'DatasetPath='+self.datasetPath+'\n'
862  
863              datasetpath_split = self.datasetPath.split("/")
864 <            
864 >
865              txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
866              txt += 'DataTier='+datasetpath_split[2]+'\n'
966 –            #txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
867              txt += 'ApplicationFamily=cmsRun\n'
868  
869          else:
870              txt += 'DatasetPath=MCDataTier\n'
871              txt += 'PrimaryDataset=null\n'
872              txt += 'DataTier=null\n'
973 –            #txt += 'ProcessedDataset=null\n'
873              txt += 'ApplicationFamily=MCDataTier\n'
874 <        if self.pset != None: #CarlosDaniele
874 >        if self.pset != None:
875              pset = os.path.basename(job.configFilename())
876              txt += '\n'
877              txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
878              if (self.datasetPath): # standard job
879 <                #txt += 'InputFiles=$2\n'
880 <                txt += 'InputFiles=${args[1]}\n'
881 <                txt += 'MaxEvents=${args[2]}\n'
983 <                txt += 'SkipEvents=${args[3]}\n'
879 >                txt += 'InputFiles=${args[1]}; export InputFiles\n'
880 >                txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
881 >                txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
882                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
985 –                txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
883                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
987 –                txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
884                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
989 –                txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
885              else:  # pythia like job
886 <                seedIndex=1
886 >                txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
887 >                txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
888 >                txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
889 >                txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
890                  if (self.firstRun):
891 <                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
891 >                    txt += 'FirstRun=${args[1]}; export FirstRun\n'
892                      txt += 'echo "FirstRun: <$FirstRun>"\n'
995 –                    txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
996 –                    seedIndex=seedIndex+1
893  
998 –                if (self.sourceSeed):
999 –                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
1000 –                    txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1001 –                    seedIndex=seedIndex+1
1002 –                    ## the following seeds are not always present
1003 –                    if (self.sourceSeedVtx):
1004 –                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1005 –                        txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1006 –                        txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1007 –                        seedIndex += 1
1008 –                    if (self.sourceSeedG4):
1009 –                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1010 –                        txt += 'echo "G4Seed: <$G4Seed>"\n'
1011 –                        txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1012 –                        seedIndex += 1
1013 –                    if (self.sourceSeedMix):
1014 –                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1015 –                        txt += 'echo "MixSeed: <$mixSeed>"\n'
1016 –                        txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1017 –                        seedIndex += 1
1018 –                    pass
1019 –                pass
894              txt += 'mv -f '+pset+' pset.cfg\n'
895  
1022 –        if len(self.additional_inbox_files) > 0:
1023 –            txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1024 –            txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1025 –            txt += 'fi\n'
1026 –            pass
896  
897 <        if self.pset != None: #CarlosDaniele
1029 <            txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1030 <        
897 >        if self.pset != None:
898              txt += '\n'
899              txt += 'echo "***** cat pset.cfg *********"\n'
900              txt += 'cat pset.cfg\n'
901              txt += 'echo "****** end pset.cfg ********"\n'
902              txt += '\n'
1036 –            ### FEDE FOR DBS OUTPUT PUBLICATION
903              txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
904              txt += 'echo "PSETHASH = $PSETHASH" \n'
1039 –            ##############
905              txt += '\n'
1041 –            # txt += 'echo "***** cat pset1.cfg *********"\n'
1042 –            # txt += 'cat pset1.cfg\n'
1043 –            # txt += 'echo "****** end pset1.cfg ********"\n'
906          return txt
907  
908 <    def wsBuildExe(self, nj=0):
908 >    def wsUntarSoftware(self, nj=0):
909          """
910          Put in the script the commands to build an executable
911          or a library.
912          """
913  
914 <        txt = ""
914 >        txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
915  
916          if os.path.isfile(self.tgzNameWithPath):
917 <            txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
917 >            txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
918              txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
919 +            txt += 'ls -Al \n'
920              txt += 'untar_status=$? \n'
921              txt += 'if [ $untar_status -ne 0 ]; then \n'
922 <            txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
923 <            txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
924 <            txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
1062 <            txt += '   if [ $middleware == OSG ]; then \n'
1063 <            txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1064 <            txt += '       cd $RUNTIME_AREA\n'
1065 <            txt += '       /bin/rm -rf $WORKING_DIR\n'
1066 <            txt += '       if [ -d $WORKING_DIR ] ;then\n'
1067 <            txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1068 <            txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1069 <            txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1070 <            txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1071 <            txt += '           rm -f $RUNTIME_AREA/$repo \n'
1072 <            txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1073 <            txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1074 <            txt += '       fi\n'
1075 <            txt += '   fi \n'
1076 <            txt += '   \n'
1077 <            txt += '   exit 1 \n'
922 >            txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
923 >            txt += '   job_exit_code=$untar_status\n'
924 >            txt += '   func_exit\n'
925              txt += 'else \n'
926              txt += '   echo "Successful untar" \n'
927              txt += 'fi \n'
928              txt += '\n'
929 <            txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
929 >            txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
930              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
931 <            #### FEDE FOR DBS OUTPUT PUBLICATION
1085 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1086 <            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1087 <            #txt += '   export PYTHONPATH=ProdAgentApi\n'
931 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
932              txt += 'else\n'
933 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1090 <            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1091 <            #txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
933 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
934              txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1093 –            ###################  
935              txt += 'fi\n'
936              txt += '\n'
937  
938              pass
939 <        
939 >
940 >        return txt
941 >
942 >    def wsBuildExe(self, nj=0):
943 >        """
944 >        Put in the script the commands to build an executable
945 >        or a library.
946 >        """
947 >
948 >        txt = '\n#Written by cms_cmssw::wsBuildExe\n'
949 >        txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
950 >
951 >        txt += 'rm -r lib/ module/ \n'
952 >        txt += 'mv $RUNTIME_AREA/lib/ . \n'
953 >        txt += 'mv $RUNTIME_AREA/module/ . \n'
954 >        if self.dataExist == True: txt += 'mv $RUNTIME_AREA/src/ . \n'
955 >        if len(self.additional_inbox_files)>0:
956 >            for file in self.additional_inbox_files:
957 >                txt += 'mv $RUNTIME_AREA/'+file+' . \n'
958 >        txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
959 >
960 >        txt += 'if [ -z "$PYTHONPATH" ]; then\n'
961 >        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
962 >        txt += 'else\n'
963 >        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
964 >        txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
965 >        txt += 'fi\n'
966 >        txt += '\n'
967 >
968          return txt
969  
970      def modifySteeringCards(self, nj):
971          """
972 <        modify the card provided by the user,
972 >        modify the card provided by the user,
973          writing a new card into share dir
974          """
975 <        
975 >
976      def executableName(self):
977          if self.scriptExe: #CarlosDaniele
978              return "sh "
# Line 1111 | Line 980 | class Cmssw(JobType):
980              return self.executable
981  
982      def executableArgs(self):
983 +        # FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
984          if self.scriptExe:#CarlosDaniele
985              return   self.scriptExe + " $NJob"
986 <        else:
987 <            return " -p pset.cfg"
986 >        else:
987 >            ex_args = ""
988 >            # FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
989 >            # Framework job report
990 >            if (self.major_version >= 1 and self.minor_version >= 5) or (self.major_version >= 2):
991 >                ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
992 >            # Type of cfg file
993 >            if self.major_version >= 2 :
994 >                ex_args += " -p pset.py"
995 >            else:
996 >                ex_args += " -p pset.cfg"
997 >            return ex_args
998  
999      def inputSandbox(self, nj):
1000          """
# Line 1126 | Line 1006 | class Cmssw(JobType):
1006          ## code
1007          if os.path.isfile(self.tgzNameWithPath):
1008              inp_box.append(self.tgzNameWithPath)
1009 <        if os.path.isfile(self.MLtgzfile):
1010 <            inp_box.append(self.MLtgzfile)
1131 <        ## config
1132 <        if not self.pset is None:
1133 <            inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1134 <        ## additional input files
1135 <        tgz = self.additionalInputFileTgz()
1136 <        inp_box.append(tgz)
1009 >        wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1010 >        inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1011          return inp_box
1012  
1013      def outputSandbox(self, nj):
# Line 1144 | Line 1018 | class Cmssw(JobType):
1018  
1019          ## User Declared output files
1020          for out in (self.output_file+self.output_file_sandbox):
1021 <            n_out = nj + 1
1021 >            n_out = nj + 1
1022              out_box.append(self.numberFile_(out,str(n_out)))
1023          return out_box
1024  
# Line 1159 | Line 1033 | class Cmssw(JobType):
1033          Returns part of a job script which renames the produced files.
1034          """
1035  
1036 <        txt = '\n'
1037 <        txt += '# directory content\n'
1036 >        txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1037 >        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1038 >        txt += 'echo ">>> current directory content:"\n'
1039          txt += 'ls \n'
1040 +        txt += '\n'
1041  
1042 <        for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1042 >        for fileWithSuffix in (self.output_file):
1043              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1044              txt += '\n'
1045              txt += '# check output file\n'
1170 –            # txt += 'ls '+fileWithSuffix+'\n'
1171 –            # txt += 'ls_result=$?\n'
1046              txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1047 <            txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1047 >            if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1048 >                txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1049 >                txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1050 >            else:
1051 >                txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1052 >                txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1053              txt += 'else\n'
1054 <            txt += '   exit_status=60302\n'
1055 <            txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1056 <            if common.scheduler.boss_scheduler_name == 'condor_g':
1054 >            txt += '    job_exit_code=60302\n'
1055 >            txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1056 >            if common.scheduler.name().upper() == 'CONDOR_G':
1057                  txt += '    if [ $middleware == OSG ]; then \n'
1058                  txt += '        echo "prepare dummy output file"\n'
1059                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
# Line 1183 | Line 1062 | class Cmssw(JobType):
1062          file_list = []
1063          for fileWithSuffix in (self.output_file):
1064               file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1065 +
1066          txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1067 <      
1067 >        txt += '\n'
1068 >        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1069 >        txt += 'echo ">>> current directory content:"\n'
1070 >        txt += 'ls \n'
1071 >        txt += '\n'
1072          txt += 'cd $RUNTIME_AREA\n'
1073 <        #### FEDE this is the cleanEnv function
1190 <        ### OLI_DANIELE
1191 <        #txt += 'if [ $middleware == OSG ]; then\n'  
1192 <        #txt += '    cd $RUNTIME_AREA\n'
1193 <        #txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1194 <        #txt += '    /bin/rm -rf $WORKING_DIR\n'
1195 <        #txt += '    if [ -d $WORKING_DIR ] ;then\n'
1196 <        #txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1197 <        #txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1198 <        #txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1199 <        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1200 <        #txt += '        rm -f $RUNTIME_AREA/$repo \n'
1201 <        #txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1202 <        #txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1203 <        #txt += '    fi\n'
1204 <        #txt += 'fi\n'
1205 <        #txt += '\n'
1206 <
1207 <
1073 >        txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1074          return txt
1075  
1076      def numberFile_(self, file, txt):
# Line 1222 | Line 1088 | class Cmssw(JobType):
1088              result = name + '_' + txt + "." + ext
1089          else:
1090              result = name + '_' + txt
1091 <        
1091 >
1092          return result
1093  
1094      def getRequirements(self, nj=[]):
1095          """
1096 <        return job requirements to add to jdl files
1096 >        return job requirements to add to jdl files
1097          """
1098          req = ''
1099          if self.version:
# Line 1242 | Line 1108 | class Cmssw(JobType):
1108                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1109  
1110          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1111 +        if common.scheduler.name() == "glitecoll":
1112 +            req += ' && other.GlueCEStateStatus == "Production" '
1113  
1114          return req
1115  
1116      def configFilename(self):
1117          """ return the config filename """
1118 <        return self.name()+'.cfg'
1118 >        # FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1119 >        if (self.major_version >= 2 and self.minor_version >= 1) or (self.major_version >= 3):
1120 >          return self.name()+'.py'
1121 >        else:
1122 >          return self.name()+'.cfg'
1123  
1252 –    ### OLI_DANIELE
1124      def wsSetupCMSOSGEnvironment_(self):
1125          """
1126          Returns part of a job script which is prepares
1127          the execution environment and which is common for all CMS jobs.
1128          """
1129 <        txt = '\n'
1130 <        txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1131 <        txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1132 <        txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1133 <        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1134 <        txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1264 <        txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1129 >        txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1130 >        txt += '    echo ">>> setup CMS OSG environment:"\n'
1131 >        txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1132 >        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1133 >        txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1134 >        txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1135          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1136 <        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1137 <        txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1138 <        txt += '   else\n'
1139 <        txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1140 <        txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1141 <        txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1272 <        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1273 <        txt += '       rm -f $RUNTIME_AREA/$repo \n'
1274 <        txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1275 <        txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1276 <        txt += '       exit 1\n'
1277 <        txt += '\n'
1278 <        txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1279 <        txt += '       cd $RUNTIME_AREA\n'
1280 <        txt += '       /bin/rm -rf $WORKING_DIR\n'
1281 <        txt += '       if [ -d $WORKING_DIR ] ;then\n'
1282 <        txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1283 <        txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1284 <        txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1285 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1286 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1287 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1288 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1289 <        txt += '       fi\n'
1290 <        txt += '\n'
1291 <        txt += '       exit 1\n'
1292 <        txt += '   fi\n'
1136 >        txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1137 >        txt += '    else\n'
1138 >        txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1139 >        txt += '        job_exit_code=10020\n'
1140 >        txt += '        func_exit\n'
1141 >        txt += '    fi\n'
1142          txt += '\n'
1143 <        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1144 <        txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1143 >        txt += '    echo "==> setup cms environment ok"\n'
1144 >        txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1145  
1146          return txt
1147 <
1147 >
1148      ### OLI_DANIELE
1149      def wsSetupCMSLCGEnvironment_(self):
1150          """
1151          Returns part of a job script which is prepares
1152          the execution environment and which is common for all CMS jobs.
1153          """
1154 <        txt  = '   \n'
1155 <        txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1156 <        txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1157 <        txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1158 <        txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1159 <        txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1160 <        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1161 <        txt += '       rm -f $RUNTIME_AREA/$repo \n'
1162 <        txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1163 <        txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1164 <        txt += '       exit 1\n'
1165 <        txt += '   else\n'
1166 <        txt += '       echo "Sourcing environment... "\n'
1167 <        txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1168 <        txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1169 <        txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1170 <        txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1171 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1172 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1173 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1174 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1175 <        txt += '           exit 1\n'
1176 <        txt += '       fi\n'
1177 <        txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1178 <        txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1179 <        txt += '       result=$?\n'
1180 <        txt += '       if [ $result -ne 0 ]; then\n'
1332 <        txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1333 <        txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1334 <        txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1335 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1336 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1337 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1338 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1339 <        txt += '           exit 1\n'
1340 <        txt += '       fi\n'
1341 <        txt += '   fi\n'
1342 <        txt += '   \n'
1343 <        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1344 <        txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1154 >        txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1155 >        txt += '    echo ">>> setup CMS LCG environment:"\n'
1156 >        txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1157 >        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1158 >        txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1159 >        txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1160 >        txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1161 >        txt += '        job_exit_code=10031\n'
1162 >        txt += '        func_exit\n'
1163 >        txt += '    else\n'
1164 >        txt += '        echo "Sourcing environment... "\n'
1165 >        txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1166 >        txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1167 >        txt += '            job_exit_code=10020\n'
1168 >        txt += '            func_exit\n'
1169 >        txt += '        fi\n'
1170 >        txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1171 >        txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1172 >        txt += '        result=$?\n'
1173 >        txt += '        if [ $result -ne 0 ]; then\n'
1174 >        txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1175 >        txt += '            job_exit_code=10032\n'
1176 >        txt += '            func_exit\n'
1177 >        txt += '        fi\n'
1178 >        txt += '    fi\n'
1179 >        txt += '    \n'
1180 >        txt += '    echo "==> setup cms environment ok"\n'
1181          return txt
1182  
1183 <    ### FEDE FOR DBS OUTPUT PUBLICATION
1183 >    ### FEDE FOR DBS OUTPUT PUBLICATION
1184      def modifyReport(self, nj):
1185          """
1186 <        insert the part of the script that modifies the FrameworkJob Report
1186 >        insert the part of the script that modifies the FrameworkJob Report
1187          """
1188  
1189 <        txt = ''
1190 <        txt += 'echo "Modify Job Report" \n'
1191 <        #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1356 <        ################ FEDE FOR DBS2 #############################################
1357 <        txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1358 <        #############################################################################
1359 <        try:
1360 <            publish_data = int(self.cfg_params['USER.publish_data'])          
1361 <        except KeyError:
1362 <            publish_data = 0
1363 <
1364 <        txt += 'if [ -z "$SE" ]; then\n'
1365 <        txt += '    SE="" \n'
1366 <        txt += 'fi \n'
1367 <        txt += 'if [ -z "$SE_PATH" ]; then\n'
1368 <        txt += '    SE_PATH="" \n'
1369 <        txt += 'fi \n'
1370 <        txt += 'echo "SE = $SE"\n'
1371 <        txt += 'echo "SE_PATH = $SE_PATH"\n'
1372 <
1373 <        if (publish_data == 1):  
1374 <            #processedDataset = self.cfg_params['USER.processed_datasetname']
1189 >        txt = '\n#Written by cms_cmssw::modifyReport\n'
1190 >        publish_data = int(self.cfg_params.get('USER.publish_data',0))
1191 >        if (publish_data == 1):
1192              processedDataset = self.cfg_params['USER.publish_data_name']
1193 <            txt += 'ProcessedDataset='+processedDataset+'\n'
1194 <            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1195 <            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1196 <            #### FEDE: added slash in LFN ##############
1193 >            LFNBaseName = LFNBase(processedDataset)
1194 >
1195 >            txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1196 >            txt += '    FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1197 >            txt += 'else\n'
1198              txt += '    FOR_LFN=/copy_problems/ \n'
1199 <            txt += 'else \n'
1200 <            txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1201 <            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1202 <            txt += '    FOR_LFN=/store$tmp \n'
1203 <            txt += 'fi \n'
1199 >            txt += '    SE=""\n'
1200 >            txt += '    SE_PATH=""\n'
1201 >            txt += 'fi\n'
1202 >
1203 >            txt += 'echo ">>> Modify Job Report:" \n'
1204 >            txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1205 >            txt += 'ProcessedDataset='+processedDataset+'\n'
1206              txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1207 +            txt += 'echo "SE = $SE"\n'
1208 +            txt += 'echo "SE_PATH = $SE_PATH"\n'
1209              txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1210              txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1211 <            #txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1212 <            txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1391 <            txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1392 <            #txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1393 <      
1211 >            txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1212 >            txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1213              txt += 'modifyReport_result=$?\n'
1395 –            txt += 'echo modifyReport_result = $modifyReport_result\n'
1214              txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1215 <            txt += '    exit_status=1\n'
1216 <            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1215 >            txt += '    modifyReport_result=70500\n'
1216 >            txt += '    job_exit_code=$modifyReport_result\n'
1217 >            txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1218 >            txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1219              txt += 'else\n'
1220 <            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1220 >            txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1221              txt += 'fi\n'
1402 –        else:
1403 –            txt += 'ProcessedDataset=no_data_to_publish \n'
1404 –            #### FEDE: added slash in LFN ##############
1405 –            txt += 'FOR_LFN=/local/ \n'
1406 –            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1407 –            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1408 –        return txt
1409 –
1410 –    def cleanEnv(self):
1411 –        ### OLI_DANIELE
1412 –        txt = ''
1413 –        txt += 'if [ $middleware == OSG ]; then\n'  
1414 –        txt += '    cd $RUNTIME_AREA\n'
1415 –        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1416 –        txt += '    /bin/rm -rf $WORKING_DIR\n'
1417 –        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1418 –        txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1419 –        txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1420 –        txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1421 –        txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1422 –        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1423 –        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1424 –        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1425 –        txt += '    fi\n'
1426 –        txt += 'fi\n'
1427 –        txt += '\n'
1222          return txt
1223  
1224      def setParam_(self, param, value):
# Line 1433 | Line 1227 | class Cmssw(JobType):
1227      def getParams(self):
1228          return self._params
1229  
1436 –    def setTaskid_(self):
1437 –        self._taskId = self.cfg_params['taskId']
1438 –        
1439 –    def getTaskid(self):
1440 –        return self._taskId
1441 –
1230      def uniquelist(self, old):
1231          """
1232          remove duplicates from a list
# Line 1447 | Line 1235 | class Cmssw(JobType):
1235          for e in old:
1236              nd[e]=0
1237          return nd.keys()
1238 +
1239 +    def outList(self):
1240 +        """
1241 +        check the dimension of the output files
1242 +        """
1243 +        txt = ''
1244 +        txt += 'echo ">>> list of expected files on output sandbox"\n'
1245 +        listOutFiles = []
1246 +        stdout = 'CMSSW_$NJob.stdout'
1247 +        stderr = 'CMSSW_$NJob.stderr'
1248 +        if (self.return_data == 1):
1249 +            for file in (self.output_file+self.output_file_sandbox):
1250 +                listOutFiles.append(self.numberFile_(file, '$NJob'))
1251 +            listOutFiles.append(stdout)
1252 +            listOutFiles.append(stderr)
1253 +        else:
1254 +            for file in (self.output_file_sandbox):
1255 +                listOutFiles.append(self.numberFile_(file, '$NJob'))
1256 +            listOutFiles.append(stdout)
1257 +            listOutFiles.append(stderr)
1258 +        txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1259 +        txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1260 +        txt += 'export filesToCheck\n'
1261 +        return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines