ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.132 by ewv, Mon Oct 29 21:03:11 2007 UTC vs.
Revision 1.169 by spiga, Tue Apr 1 14:53:36 2008 UTC

# Line 13 | Line 13 | class Cmssw(JobType):
13          JobType.__init__(self, 'CMSSW')
14          common.logger.debug(3,'CMSSW::__init__')
15  
16 +        self.argsList = []
17 +
18          self._params = {}
19          self.cfg_params = cfg_params
18 –
20          # init BlackWhiteListParser
21          self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
22  
23 <        try:
23 <            self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24 <        except KeyError:
25 <            self.MaxTarBallSize = 9.5
23 >        self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
24  
25          # number of jobs requested to be created, limit obj splitting
26          self.ncjobs = ncjobs
# Line 49 | Line 47 | class Cmssw(JobType):
47          # Try to block creation in case of arch/version mismatch
48          #
49  
50 <        a = string.split(self.version, "_")
51 <
52 <        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53 <            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
54 <            raise CrabException(msg)
55 <        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56 <            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57 <            raise CrabException(msg)
58 <
59 <        common.taskDB.setDict('codeVersion',self.version)
50 > #        a = string.split(self.version, "_")
51 > #
52 > #        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53 > #            msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
54 > #            common.logger.message(msg)
55 > #        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56 > #            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57 > #            raise CrabException(msg)
58 > #
59 >
60          self.setParam_('application', self.version)
61  
62          ### collect Data cards
63  
64 <        ## get DBS mode
67 <        try:
68 <            self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69 <        except KeyError:
70 <            self.use_dbs_1 = 0
71 <
72 <        try:
73 <            tmp =  cfg_params['CMSSW.datasetpath']
74 <            log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75 <            if string.lower(tmp)=='none':
76 <                self.datasetPath = None
77 <                self.selectNoInput = 1
78 <            else:
79 <                self.datasetPath = tmp
80 <                self.selectNoInput = 0
81 <        except KeyError:
64 >        if not cfg_params.has_key('CMSSW.datasetpath'):
65              msg = "Error: datasetpath not defined "
66              raise CrabException(msg)
67 +        tmp =  cfg_params['CMSSW.datasetpath']
68 +        log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
69 +        if string.lower(tmp)=='none':
70 +            self.datasetPath = None
71 +            self.selectNoInput = 1
72 +        else:
73 +            self.datasetPath = tmp
74 +            self.selectNoInput = 0
75  
76          # ML monitoring
77          # split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
# Line 88 | Line 79 | class Cmssw(JobType):
79              self.setParam_('dataset', 'None')
80              self.setParam_('owner', 'None')
81          else:
82 +            ## SL what is supposed to fail here?
83              try:
84                  datasetpath_split = self.datasetPath.split("/")
85                  # standard style
86                  self.setParam_('datasetFull', self.datasetPath)
87 <                if self.use_dbs_1 == 1 :
88 <                    self.setParam_('dataset', datasetpath_split[1])
97 <                    self.setParam_('owner', datasetpath_split[-1])
98 <                else:
99 <                    self.setParam_('dataset', datasetpath_split[1])
100 <                    self.setParam_('owner', datasetpath_split[2])
87 >                self.setParam_('dataset', datasetpath_split[1])
88 >                self.setParam_('owner', datasetpath_split[2])
89              except:
90                  self.setParam_('dataset', self.datasetPath)
91                  self.setParam_('owner', self.datasetPath)
92  
93 <        self.setTaskid_()
106 <        self.setParam_('taskId', self.cfg_params['taskId'])
93 >        self.setParam_('taskId', common._db.queryTask('name')) ## new BL--DS
94  
95          self.dataTiers = []
96  
97          ## now the application
98 <        try:
99 <            self.executable = cfg_params['CMSSW.executable']
100 <            self.setParam_('exe', self.executable)
114 <            log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
115 <            msg = "Default executable cmsRun overridden. Switch to " + self.executable
116 <            log.debug(3,msg)
117 <        except KeyError:
118 <            self.executable = 'cmsRun'
119 <            self.setParam_('exe', self.executable)
120 <            msg = "User executable not defined. Use cmsRun"
121 <            log.debug(3,msg)
122 <            pass
98 >        self.executable = cfg_params.get('CMSSW.executable','cmsRun')
99 >        self.setParam_('exe', self.executable)
100 >        log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
101  
102 <        try:
125 <            self.pset = cfg_params['CMSSW.pset']
126 <            log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
127 <            if self.pset.lower() != 'none' :
128 <                if (not os.path.exists(self.pset)):
129 <                    raise CrabException("User defined PSet file "+self.pset+" does not exist")
130 <            else:
131 <                self.pset = None
132 <        except KeyError:
102 >        if not cfg_params.has_key('CMSSW.pset'):
103              raise CrabException("PSet file missing. Cannot run cmsRun ")
104 +        self.pset = cfg_params['CMSSW.pset']
105 +        log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
106 +        if self.pset.lower() != 'none' :
107 +            if (not os.path.exists(self.pset)):
108 +                raise CrabException("User defined PSet file "+self.pset+" does not exist")
109 +        else:
110 +            self.pset = None
111  
112          # output files
113          ## stuff which must be returned always via sandbox
# Line 140 | Line 117 | class Cmssw(JobType):
117          self.output_file_sandbox.append(self.fjrFileName)
118  
119          # other output files to be returned via sandbox or copied to SE
120 <        try:
121 <            self.output_file = []
122 <            tmp = cfg_params['CMSSW.output_file']
123 <            if tmp != '':
124 <                tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
125 <                log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
126 <                for tmp in tmpOutFiles:
127 <                    tmp=string.strip(tmp)
151 <                    self.output_file.append(tmp)
152 <                    pass
153 <            else:
154 <                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
120 >        self.output_file = []
121 >        tmp = cfg_params.get('CMSSW.output_file',None)
122 >        if tmp :
123 >            tmpOutFiles = string.split(tmp,',')
124 >            log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
125 >            for tmp in tmpOutFiles:
126 >                tmp=string.strip(tmp)
127 >                self.output_file.append(tmp)
128                  pass
129 <            pass
157 <        except KeyError:
129 >        else:
130              log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
131 <            pass
131 >        pass
132  
133          # script_exe file as additional file in inputSandbox
134 <        try:
135 <            self.scriptExe = cfg_params['USER.script_exe']
136 <            if self.scriptExe != '':
137 <               if not os.path.isfile(self.scriptExe):
138 <                  msg ="ERROR. file "+self.scriptExe+" not found"
139 <                  raise CrabException(msg)
168 <               self.additional_inbox_files.append(string.strip(self.scriptExe))
169 <        except KeyError:
170 <            self.scriptExe = ''
134 >        self.scriptExe = cfg_params.get('USER.script_exe',None)
135 >        if self.scriptExe :
136 >           if not os.path.isfile(self.scriptExe):
137 >              msg ="ERROR. file "+self.scriptExe+" not found"
138 >              raise CrabException(msg)
139 >           self.additional_inbox_files.append(string.strip(self.scriptExe))
140  
141          #CarlosDaniele
142          if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
# Line 175 | Line 144 | class Cmssw(JobType):
144             raise CrabException(msg)
145  
146          ## additional input files
147 <        try:
147 >        if cfg_params.has_key('USER.additional_input_files'):
148              tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
149              for tmp in tmpAddFiles:
150                  tmp = string.strip(tmp)
# Line 199 | Line 168 | class Cmssw(JobType):
168                  pass
169              pass
170              common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
171 <        except KeyError:
203 <            pass
204 <
205 <        # files per job
206 <        try:
207 <            if (cfg_params['CMSSW.files_per_jobs']):
208 <                raise CrabException("files_per_jobs no longer supported.  Quitting.")
209 <        except KeyError:
210 <            pass
171 >        pass
172  
173          ## Events per job
174 <        try:
174 >        if cfg_params.has_key('CMSSW.events_per_job'):
175              self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
176              self.selectEventsPerJob = 1
177 <        except KeyError:
177 >        else:
178              self.eventsPerJob = -1
179              self.selectEventsPerJob = 0
180  
181          ## number of jobs
182 <        try:
182 >        if cfg_params.has_key('CMSSW.number_of_jobs'):
183              self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
184              self.selectNumberOfJobs = 1
185 <        except KeyError:
185 >        else:
186              self.theNumberOfJobs = 0
187              self.selectNumberOfJobs = 0
188  
189 <        try:
189 >        if cfg_params.has_key('CMSSW.total_number_of_events'):
190              self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
191              self.selectTotalNumberEvents = 1
192 <        except KeyError:
192 >        else:
193              self.total_number_of_events = 0
194              self.selectTotalNumberEvents = 0
195  
# Line 241 | Line 202 | class Cmssw(JobType):
202                   msg = 'Must specify  number_of_jobs.'
203                   raise CrabException(msg)
204  
205 <        ## source seed for pythia
206 <        try:
207 <            self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
208 <        except KeyError:
209 <            self.sourceSeed = None
210 <            common.logger.debug(5,"No seed given")
211 <
212 <        try:
213 <            self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
214 <        except KeyError:
215 <            self.sourceSeedVtx = None
216 <            common.logger.debug(5,"No vertex seed given")
217 <
218 <        try:
219 <            self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
220 <        except KeyError:
221 <            self.sourceSeedG4 = None
222 <            common.logger.debug(5,"No g4 sim hits seed given")
205 >        ## New method of dealing with seeds
206 >        self.incrementSeeds = []
207 >        self.preserveSeeds = []
208 >        if cfg_params.has_key('CMSSW.preserve_seeds'):
209 >            tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
210 >            for tmp in tmpList:
211 >                tmp.strip()
212 >                self.preserveSeeds.append(tmp)
213 >        if cfg_params.has_key('CMSSW.increment_seeds'):
214 >            tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
215 >            for tmp in tmpList:
216 >                tmp.strip()
217 >                self.incrementSeeds.append(tmp)
218 >
219 >        ## Old method of dealing with seeds
220 >        ## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
221 >        ## remove
222 >        self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
223 >        if self.sourceSeed:
224 >          print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
225 >          self.incrementSeeds.append('sourceSeed')
226 >
227 >        self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
228 >        if self.sourceSeedVtx:
229 >          print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
230 >          self.incrementSeeds.append('VtxSmeared')
231 >
232 >        self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
233 >        if self.sourceSeedG4:
234 >          print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
235 >          self.incrementSeeds.append('g4SimHits')
236 >
237 >        self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
238 >        if self.sourceSeedMix:
239 >          print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
240 >          self.incrementSeeds.append('mix')
241  
242 <        try:
264 <            self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
265 <        except KeyError:
266 <            self.sourceSeedMix = None
267 <            common.logger.debug(5,"No mix seed given")
242 >        self.firstRun = cfg_params.get('CMSSW.first_run',None)
243  
269 –        try:
270 –            self.firstRun = int(cfg_params['CMSSW.first_run'])
271 –        except KeyError:
272 –            self.firstRun = None
273 –            common.logger.debug(5,"No first run given")
244          if self.pset != None: #CarlosDaniele
245              import PsetManipulator as pp
246              PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
247  
248 +        # Copy/return
249 +
250 +        self.copy_data = int(cfg_params.get('USER.copy_data',0))
251 +        self.return_data = int(cfg_params.get('USER.return_data',0))
252 +
253          #DBSDLS-start
254          ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
255          self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
# Line 301 | Line 276 | class Cmssw(JobType):
276          # modify Pset
277          if self.pset != None: #CarlosDaniele
278              try:
279 <                if (self.datasetPath): # standard job
280 <                    # allow to processa a fraction of events in a file
306 <                    PsetEdit.inputModule("INPUTFILE")
307 <                    PsetEdit.maxEvent(0)
308 <                    PsetEdit.skipEvent(0)
309 <                else:  # pythia like job
310 <                    PsetEdit.maxEvent(self.eventsPerJob)
311 <                    if (self.firstRun):
312 <                        PsetEdit.pythiaFirstRun(0)  #First Run
313 <                    if (self.sourceSeed) :
314 <                        PsetEdit.pythiaSeed(0)
315 <                        if (self.sourceSeedVtx) :
316 <                            PsetEdit.vtxSeed(0)
317 <                        if (self.sourceSeedG4) :
318 <                            PsetEdit.g4Seed(0)
319 <                        if (self.sourceSeedMix) :
320 <                            PsetEdit.mixSeed(0)
321 <                # add FrameworkJobReport to parameter-set
279 >                # Add FrameworkJobReport to parameter-set, set max events.
280 >                # Reset later for data jobs by writeCFG which does all modifications
281                  PsetEdit.addCrabFJR(self.fjrFileName)
282 +                PsetEdit.maxEvent(self.eventsPerJob)
283                  PsetEdit.psetWriter(self.configFilename())
284              except:
285                  msg='Error while manipuliating ParameterSet: exiting...'
# Line 328 | Line 288 | class Cmssw(JobType):
288      def DataDiscoveryAndLocation(self, cfg_params):
289  
290          import DataDiscovery
331 –        import DataDiscovery_DBS2
291          import DataLocation
292          common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
293  
# Line 337 | Line 296 | class Cmssw(JobType):
296          ## Contact the DBS
297          common.logger.message("Contacting Data Discovery Services ...")
298          try:
299 <
341 <            if self.use_dbs_1 == 1 :
342 <                self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
343 <            else :
344 <                self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
299 >            self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
300              self.pubdata.fetchDBSInfo()
301  
302          except DataDiscovery.NotExistingDatasetError, ex :
# Line 353 | Line 308 | class Cmssw(JobType):
308          except DataDiscovery.DataDiscoveryError, ex:
309              msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
310              raise CrabException(msg)
356 –        except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
357 –            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
358 –            raise CrabException(msg)
359 –        except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
360 –            msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
361 –            raise CrabException(msg)
362 –        except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
363 –            msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
364 –            raise CrabException(msg)
311  
312          self.filesbyblock=self.pubdata.getFiles()
313          self.eventsbyblock=self.pubdata.getEventsPerBlock()
# Line 392 | Line 338 | class Cmssw(JobType):
338  
339          return sites
340  
341 +  # to Be Removed  DS -- BL
342 +  #  def setArgsList(self, argsList):
343 +  #      self.argsList = argsList
344 +
345      def jobSplittingByBlocks(self, blockSites):
346          """
347          Perform job splitting. Jobs run over an integer number of files
# Line 563 | Line 513 | class Cmssw(JobType):
513                          jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
514                          # remove all but the last file
515                          filesEventCount = self.eventsbyfile[file]
516 <                        parString = ""
567 <                        parString += '\\\"' + file + '\\\"\,'
516 >                        parString = '\\\"' + file + '\\\"\,'
517                      pass # END if
518                  pass # END while (iterate over files in the block)
519          pass # END while (iterate over blocks in the dataset)
# Line 604 | Line 553 | class Cmssw(JobType):
553              for range_jobs in noSiteBlock:
554                  msg += str(range_jobs) + virgola
555              msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
556 +            if self.cfg_params.has_key('EDG.se_white_list'):
557 +                msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
558 +                msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
559 +                msg += 'Please check if the dataset is available at this site!)\n'
560 +            if self.cfg_params.has_key('EDG.ce_white_list'):
561 +                msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
562 +                msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
563 +                msg += 'Please check if the dataset is available at this site!)\n'
564 +
565              common.logger.message(msg)
566  
567          self.list_of_args = list_of_lists
# Line 652 | Line 610 | class Cmssw(JobType):
610          self.list_of_args = []
611          for i in range(self.total_number_of_jobs):
612              ## Since there is no input, any site is good
655 –           # self.jobDestination.append(["Any"])
613              self.jobDestination.append([""]) #must be empty to write correctly the xml
614              args=[]
615              if (self.firstRun):
616 <                    ## pythia first run
660 <                #self.list_of_args.append([(str(self.firstRun)+str(i))])
616 >                ## pythia first run
617                  args.append(str(self.firstRun)+str(i))
662 –            else:
663 –                ## no first run
664 –                #self.list_of_args.append([str(i)])
665 –                args.append(str(i))
666 –            if (self.sourceSeed):
667 –                args.append(str(self.sourceSeed)+str(i))
668 –                if (self.sourceSeedVtx):
669 –                    ## + vtx random seed
670 –                    args.append(str(self.sourceSeedVtx)+str(i))
671 –                if (self.sourceSeedG4):
672 –                    ## + G4 random seed
673 –                    args.append(str(self.sourceSeedG4)+str(i))
674 –                if (self.sourceSeedMix):
675 –                    ## + Mix random seed
676 –                    args.append(str(self.sourceSeedMix)+str(i))
677 –                pass
678 –            pass
618              self.list_of_args.append(args)
680 –        pass
681 –
682 –        # print self.list_of_args
619  
620          return
621  
# Line 709 | Line 645 | class Cmssw(JobType):
645  
646      def split(self, jobParams):
647  
712 –        common.jobDB.load()
648          #### Fabio
649          njobs = self.total_number_of_jobs
650          arglist = self.list_of_args
# Line 717 | Line 652 | class Cmssw(JobType):
652          for i in range(njobs):
653              jobParams.append("")
654  
655 +        listID=[]
656 +        listField=[]
657          for job in range(njobs):
658              jobParams[job] = arglist[job]
659 <            # print str(arglist[job])
660 <            # print jobParams[job]
661 <            common.jobDB.setArguments(job, jobParams[job])
662 <            common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
663 <            common.jobDB.setDestination(job, self.jobDestination[job])
659 >            listID.append(job+1)
660 >            job_ToSave ={}
661 >            concString = ' '
662 >            argu=''
663 >            if len(jobParams[job]):
664 >                argu +=   concString.join(jobParams[job] )
665 >            job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS
666 >            job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS
667 >            #common._db.updateJob_(job,job_ToSave)## new BL--DS
668 >            listField.append(job_ToSave)
669 >            msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
670 >            +"                     Destination: "+str(self.jobDestination[job])
671 >            common.logger.debug(5,msg)
672 >            #common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
673 >        common._db.updateJob_(listID,listField)## new BL--DS
674 >        ## Pay Attention Here....DS--BL
675 >        self.argsList = (len(jobParams[1])+1)
676  
728 –        common.jobDB.save()
677          return
678 <
679 <    def getJobTypeArguments(self, nj, sched):
680 <        result = ''
681 <        for i in common.jobDB.arguments(nj):
682 <            result=result+str(i)+" "
683 <        return result
678 > #
679 > #    def getJobTypeArguments(self, nj, sched):
680 > #        result = ''
681 > #        jobs=[]
682 > #        jobs.append(nj)
683 > #        for i in common._db.queryJob('arguments',jobs):##  BL--DS
684 > #            result=result+str(i)+" "
685 > #        return result
686  
687      def numberOfJobs(self):
688          # Fabio
# Line 815 | Line 765 | class Cmssw(JobType):
765                      common.logger.debug(5,"data "+root+"/data"+" to be tarred")
766                      tar.add(root+"/data",root[swAreaLen:]+"/data")
767  
818 –            ## Add ProdAgent dir to tar
819 –            paDir = 'ProdAgentApi'
820 –            pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
821 –            if os.path.isdir(pa):
822 –                tar.add(pa,paDir)
768  
769 <            ### FEDE FOR DBS PUBLICATION
825 <            ## Add PRODCOMMON dir to tar
769 >            ## Add ProdCommon dir to tar
770              prodcommonDir = 'ProdCommon'
771              prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
772              if os.path.isdir(prodcommonPath):
773                  tar.add(prodcommonPath,prodcommonDir)
830 –            #############################
774  
775              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
776              tar.close()
# Line 844 | Line 787 | class Cmssw(JobType):
787          try:
788              tar = tarfile.open(self.MLtgzfile, "w:gz")
789              path=os.environ['CRABDIR'] + '/python/'
790 <            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
790 >            #for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py']:
791 >            ### FEDE ####
792 >            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']:
793 >            ###############
794                  tar.add(path+file,file)
795              common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
796              tar.close()
# Line 866 | Line 812 | class Cmssw(JobType):
812          tar.close()
813          return tarName
814  
815 <    def wsSetupEnvironment(self, nj):
815 >    def wsSetupEnvironment(self, nj=0):
816          """
817          Returns part of a job script which prepares
818          the execution environment for the job 'nj'.
819          """
820          # Prepare JobType-independent part
821 <        txt = ''
822 <
877 <        ## OLI_Daniele at this level  middleware already known
878 <
821 >        txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
822 >        txt += 'echo ">>> setup environment"\n'
823          txt += 'if [ $middleware == LCG ]; then \n'
880 –        txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
881 –        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
882 –        txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
824          txt += self.wsSetupCMSLCGEnvironment_()
825          txt += 'elif [ $middleware == OSG ]; then\n'
826          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
827          txt += '    if [ ! $? == 0 ] ;then\n'
828 <        txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
829 <        txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
830 <        txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
831 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
832 <        txt += '        rm -f $RUNTIME_AREA/$repo \n'
833 <        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
834 <        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
835 <        txt += '        exit 1\n'
828 >        #txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
829 >        #txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
830 >        #txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
831 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
832 >        #txt += '        exit 1\n'
833 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
834 >        txt += '        job_exit_code=10016\n'
835 >        txt += '        func_exit\n'
836          txt += '    fi\n'
837 <        txt += '    echo "Created working directory: $WORKING_DIR"\n'
837 >        txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
838          txt += '\n'
839          txt += '    echo "Change to working directory: $WORKING_DIR"\n'
840          txt += '    cd $WORKING_DIR\n'
841 +        txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
842          txt += self.wsSetupCMSOSGEnvironment_()
843 <        txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
844 <        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
843 >        #txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
844 >        #txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
845          txt += 'fi\n'
846  
847          # Prepare JobType-specific part
848          scram = self.scram.commandName()
849          txt += '\n\n'
850 <        txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
850 >        txt += 'echo ">>> specific cmssw setup environment:"\n'
851 >        txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
852          txt += scram+' project CMSSW '+self.version+'\n'
853          txt += 'status=$?\n'
854          txt += 'if [ $status != 0 ] ; then\n'
855 <        txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856 <        txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
857 <        txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
858 <        txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
859 <        txt += '   rm -f $RUNTIME_AREA/$repo \n'
860 <        txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
861 <        txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
862 <        ## OLI_Daniele
863 <        txt += '    if [ $middleware == OSG ]; then \n'
864 <        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
865 <        txt += '        cd $RUNTIME_AREA\n'
866 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
867 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
868 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
869 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
870 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
871 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
872 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
873 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
874 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
875 <        txt += '        fi\n'
876 <        txt += '    fi \n'
934 <        txt += '   exit 1 \n'
855 >        #txt += '    echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856 >        #txt += '    echo "JOB_EXIT_STATUS = 10034"\n'
857 >        #txt += '    echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
858 >        #txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
859 >        txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
860 >        txt += '    job_exit_code=10034\n'
861 >        #txt += '    if [ $middleware == OSG ]; then \n'
862 >        #txt += '        cd $RUNTIME_AREA\n'
863 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
864 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
865 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
866 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
867 >        #txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
868 >        #txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
869 >        #txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
870 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
871 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
872 >        #txt += '            job_exit_code=10017\n'
873 >        #txt += '        fi\n'
874 >        #txt += '    fi \n'
875 >        #txt += '    exit 1 \n'
876 >        txt += '    func_exit\n'
877          txt += 'fi \n'
936 –        txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
878          txt += 'cd '+self.version+'\n'
879          ########## FEDE FOR DBS2 ######################
880          txt += 'SOFTWARE_DIR=`pwd`\n'
881 <        txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
881 >        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
882          ###############################################
883          ### needed grep for bug in scramv1 ###
943 –        txt += scram+' runtime -sh\n'
884          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
945 –        txt += 'echo $PATH\n'
946 –
885          # Handle the arguments:
886          txt += "\n"
887          txt += "## number of arguments (first argument always jobnumber)\n"
888          txt += "\n"
889 < #        txt += "narg=$#\n"
890 <        txt += "if [ $nargs -lt 2 ]\n"
889 >       # txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
890 >        txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
891          txt += "then\n"
892 <        txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893 <        txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
894 <        txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
895 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
896 <        txt += '    rm -f $RUNTIME_AREA/$repo \n'
897 <        txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
898 <        txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
899 <        ## OLI_Daniele
900 <        txt += '    if [ $middleware == OSG ]; then \n'
901 <        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
902 <        txt += '        cd $RUNTIME_AREA\n'
903 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
904 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
905 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
906 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
907 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
908 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
909 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
910 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
911 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
912 <        txt += '        fi\n'
913 <        txt += '    fi \n'
976 <        txt += "    exit 1\n"
892 >        #txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893 >        #txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
894 >        #txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
895 >        #txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
896 >        txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
897 >        txt += '    job_exit_code=50113\n'
898 >        #txt += '    if [ $middleware == OSG ]; then \n'
899 >        #txt += '        cd $RUNTIME_AREA\n'
900 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
901 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
902 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
903 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
904 >        #txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
905 >        #txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
906 >        #txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
907 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
908 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
909 >        #txt += '            job_exit_code=10017\n'
910 >        #txt += '        fi\n'
911 >        #txt += '    fi\n'
912 >        #txt += "    exit 1\n"
913 >        txt += "    func_exit\n"
914          txt += "fi\n"
915          txt += "\n"
916  
# Line 988 | Line 925 | class Cmssw(JobType):
925  
926              txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
927              txt += 'DataTier='+datasetpath_split[2]+'\n'
991 –            #txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
928              txt += 'ApplicationFamily=cmsRun\n'
929  
930          else:
931              txt += 'DatasetPath=MCDataTier\n'
932              txt += 'PrimaryDataset=null\n'
933              txt += 'DataTier=null\n'
998 –            #txt += 'ProcessedDataset=null\n'
934              txt += 'ApplicationFamily=MCDataTier\n'
935 <        if self.pset != None: #CarlosDaniele
935 >        if self.pset != None:
936              pset = os.path.basename(job.configFilename())
937              txt += '\n'
938              txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
939              if (self.datasetPath): # standard job
940 <                #txt += 'InputFiles=$2\n'
941 <                txt += 'InputFiles=${args[1]}\n'
942 <                txt += 'MaxEvents=${args[2]}\n'
1008 <                txt += 'SkipEvents=${args[3]}\n'
940 >                txt += 'InputFiles=${args[1]}; export InputFiles\n'
941 >                txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
942 >                txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
943                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
1010 –                txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
944                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1012 –                txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
945                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1014 –                txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
946              else:  # pythia like job
947 <                seedIndex=1
947 >                txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
948 >                txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
949 >                txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
950 >                txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
951                  if (self.firstRun):
952 <                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
952 >                    txt += 'FirstRun=${args[1]}; export FirstRun\n'
953                      txt += 'echo "FirstRun: <$FirstRun>"\n'
1020 –                    txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1021 –                    seedIndex=seedIndex+1
954  
1023 –                if (self.sourceSeed):
1024 –                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
1025 –                    txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1026 –                    seedIndex=seedIndex+1
1027 –                    ## the following seeds are not always present
1028 –                    if (self.sourceSeedVtx):
1029 –                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1030 –                        txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1031 –                        txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1032 –                        seedIndex += 1
1033 –                    if (self.sourceSeedG4):
1034 –                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1035 –                        txt += 'echo "G4Seed: <$G4Seed>"\n'
1036 –                        txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1037 –                        seedIndex += 1
1038 –                    if (self.sourceSeedMix):
1039 –                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1040 –                        txt += 'echo "MixSeed: <$mixSeed>"\n'
1041 –                        txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1042 –                        seedIndex += 1
1043 –                    pass
1044 –                pass
955              txt += 'mv -f '+pset+' pset.cfg\n'
956  
957          if len(self.additional_inbox_files) > 0:
# Line 1050 | Line 960 | class Cmssw(JobType):
960              txt += 'fi\n'
961              pass
962  
963 <        if self.pset != None: #CarlosDaniele
1054 <            txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1055 <
963 >        if self.pset != None:
964              txt += '\n'
965              txt += 'echo "***** cat pset.cfg *********"\n'
966              txt += 'cat pset.cfg\n'
967              txt += 'echo "****** end pset.cfg ********"\n'
968              txt += '\n'
1061 –            ### FEDE FOR DBS OUTPUT PUBLICATION
969              txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
970              txt += 'echo "PSETHASH = $PSETHASH" \n'
1064 –            ##############
971              txt += '\n'
1066 –            # txt += 'echo "***** cat pset1.cfg *********"\n'
1067 –            # txt += 'cat pset1.cfg\n'
1068 –            # txt += 'echo "****** end pset1.cfg ********"\n'
972          return txt
973 <
974 <    def wsBuildExe(self, nj=0):
973 >    #### FEDE #####
974 >    def wsUntarSoftware(self, nj=0):
975          """
976          Put in the script the commands to build an executable
977          or a library.
978          """
979  
980 <        txt = ""
980 >        txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
981  
982          if os.path.isfile(self.tgzNameWithPath):
983 <            txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
983 >            txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
984              txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
985              txt += 'untar_status=$? \n'
986              txt += 'if [ $untar_status -ne 0 ]; then \n'
987 <            txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
988 <            txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
989 <            txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
1087 <            txt += '   if [ $middleware == OSG ]; then \n'
1088 <            txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1089 <            txt += '       cd $RUNTIME_AREA\n'
1090 <            txt += '       /bin/rm -rf $WORKING_DIR\n'
1091 <            txt += '       if [ -d $WORKING_DIR ] ;then\n'
1092 <            txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1093 <            txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1094 <            txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1095 <            txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1096 <            txt += '           rm -f $RUNTIME_AREA/$repo \n'
1097 <            txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1098 <            txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1099 <            txt += '       fi\n'
1100 <            txt += '   fi \n'
1101 <            txt += '   \n'
1102 <            txt += '   exit 1 \n'
987 >            txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
988 >            txt += '   job_exit_code=$untar_status\n'
989 >            txt += '   func_exit\n'
990              txt += 'else \n'
991              txt += '   echo "Successful untar" \n'
992              txt += 'fi \n'
993              txt += '\n'
994 <            txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
994 >            txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
995              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
996 <            #### FEDE FOR DBS OUTPUT PUBLICATION
1110 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1111 <            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1112 <            #txt += '   export PYTHONPATH=ProdAgentApi\n'
996 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
997              txt += 'else\n'
998 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1115 <            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1116 <            #txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
998 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
999              txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1118 –            ###################
1000              txt += 'fi\n'
1001              txt += '\n'
1002  
1003              pass
1004  
1005          return txt
1006 +        
1007 +    def wsBuildExe(self, nj=0):
1008 +        """
1009 +        Put in the script the commands to build an executable
1010 +        or a library.
1011 +        """
1012 +
1013 +        txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1014 +        txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1015 +
1016 +        txt += 'mv $RUNTIME_AREA/lib . \n'
1017 +        txt += 'mv $RUNTIME_AREA/module . \n'
1018 +        txt += 'mv $RUNTIME_AREA/ProdCommon . \n'
1019 +        
1020 +
1021 +        #if os.path.isfile(self.tgzNameWithPath):
1022 +        #    txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1023 +        #    txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1024 +        #    txt += 'untar_status=$? \n'
1025 +        #    txt += 'if [ $untar_status -ne 0 ]; then \n'
1026 +        #    txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
1027 +        #    txt += '   job_exit_code=$untar_status\n'
1028 +        #    txt += '   func_exit\n'
1029 +        #    txt += 'else \n'
1030 +        #    txt += '   echo "Successful untar" \n'
1031 +        #    txt += 'fi \n'
1032 +        #    txt += '\n'
1033 +        #    txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
1034 +        txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1035 +        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
1036 +        txt += 'else\n'
1037 +        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1038 +        txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1039 +        txt += 'fi\n'
1040 +        txt += '\n'
1041 +
1042 +        return txt
1043 +    ############################################################################
1044  
1045      def modifySteeringCards(self, nj):
1046          """
# Line 1136 | Line 1055 | class Cmssw(JobType):
1055              return self.executable
1056  
1057      def executableArgs(self):
1058 +        # FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1059          if self.scriptExe:#CarlosDaniele
1060              return   self.scriptExe + " $NJob"
1061          else:
1142 –            # if >= CMSSW_1_5_X, add -e
1062              version_array = self.scram.getSWVersion().split('_')
1063              major = 0
1064              minor = 0
# Line 1149 | Line 1068 | class Cmssw(JobType):
1068              except:
1069                  msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1070                  raise CrabException(msg)
1071 +
1072 +            ex_args = ""
1073 +
1074 +            # Framework job report
1075              if major >= 1 and minor >= 5 :
1076 <                return " -e -p pset.cfg"
1076 >                #ex_args += " -j " + self.fjrFileName
1077 >            ### FEDE it could be improved!!! ####    
1078 >                ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1079 >            #######################################
1080 >            # Type of cfg file
1081 >            if major >= 2 :
1082 >                ex_args += " -p pset.pycfg"
1083              else:
1084 <                return " -p pset.cfg"
1084 >                ex_args += " -p pset.cfg"
1085 >            return ex_args
1086  
1087      def inputSandbox(self, nj):
1088          """
# Line 1172 | Line 1102 | class Cmssw(JobType):
1102          ## additional input files
1103          tgz = self.additionalInputFileTgz()
1104          inp_box.append(tgz)
1105 +        ## executable
1106 +        wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1107 +        inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1108          return inp_box
1109  
1110      def outputSandbox(self, nj):
# Line 1197 | Line 1130 | class Cmssw(JobType):
1130          Returns part of a job script which renames the produced files.
1131          """
1132  
1133 <        txt = '\n'
1134 <        txt += '# directory content\n'
1133 >        txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1134 >        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1135 >        txt += 'echo ">>> current directory content:"\n'
1136          txt += 'ls \n'
1137 +        txt += '\n'
1138  
1139 <        txt += 'output_exit_status=0\n'
1139 >        #txt += 'output_exit_status=0\n'
1140  
1141 <        for fileWithSuffix in (self.output_file_sandbox):
1142 <            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1143 <            txt += '\n'
1144 <            txt += '# check output file\n'
1145 <            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1146 <            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1147 <            txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1148 <            txt += 'else\n'
1149 <            txt += '    exit_status=60302\n'
1150 <            txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1151 <            if common.scheduler.boss_scheduler_name == 'condor_g':
1152 <                txt += '    if [ $middleware == OSG ]; then \n'
1153 <                txt += '        echo "prepare dummy output file"\n'
1154 <                txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1155 <                txt += '    fi \n'
1156 <            txt += 'fi\n'
1141 >        ### FEDE #######
1142 >        #for fileWithSuffix in (self.output_file_sandbox):
1143 >        #    output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1144 >        #    txt += '\n'
1145 >        #    txt += '# check output file\n'
1146 >        #    txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1147 >        #    txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1148 >        #    txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1149 >        #    txt += 'else\n'
1150 >        #    txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1151 >        #    txt += '    job_exit_code=60302\n'
1152 >        #    if common.scheduler.name().upper() == 'CONDOR_G':
1153 >        #        txt += '    if [ $middleware == OSG ]; then \n'
1154 >        #        txt += '        echo "prepare dummy output file"\n'
1155 >        #        txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1156 >        #        txt += '    fi \n'
1157 >        #    txt += 'fi\n'
1158  
1159          for fileWithSuffix in (self.output_file):
1160              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1161              txt += '\n'
1162              txt += '# check output file\n'
1163              txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1164 <            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1165 <            txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1164 >            if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1165 >                txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1166 >                txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1167 >            else:
1168 >                txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1169 >                txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1170              txt += 'else\n'
1171 <            txt += '    exit_status=60302\n'
1172 <            txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1173 <            txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1174 <            txt += '    output_exit_status=$exit_status\n'
1175 <            if common.scheduler.boss_scheduler_name == 'condor_g':
1171 >            #txt += '    exit_status=60302\n'
1172 >            #txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1173 >            #txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1174 >            #txt += '    output_exit_status=$exit_status\n'
1175 >            txt += '    job_exit_code=60302\n'
1176 >            txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1177 >            if common.scheduler.name().upper() == 'CONDOR_G':
1178                  txt += '    if [ $middleware == OSG ]; then \n'
1179                  txt += '        echo "prepare dummy output file"\n'
1180                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
# Line 1243 | Line 1185 | class Cmssw(JobType):
1185               file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1186  
1187          txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1188 +        txt += '\n'
1189 +        txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1190 +        txt += 'echo ">>> current directory content:"\n'
1191 +        txt += 'ls \n'
1192 +        txt += '\n'
1193          txt += 'cd $RUNTIME_AREA\n'
1194 +        txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1195          return txt
1196  
1197      def numberFile_(self, file, txt):
# Line 1281 | Line 1229 | class Cmssw(JobType):
1229                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1230  
1231          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1232 +        if common.scheduler.name() == "glitecoll":
1233 +            req += ' && other.GlueCEStateStatus == "Production" '
1234  
1235          return req
1236  
# Line 1288 | Line 1238 | class Cmssw(JobType):
1238          """ return the config filename """
1239          return self.name()+'.cfg'
1240  
1291 –    ### OLI_DANIELE
1241      def wsSetupCMSOSGEnvironment_(self):
1242          """
1243          Returns part of a job script which is prepares
1244          the execution environment and which is common for all CMS jobs.
1245          """
1246 <        txt = '\n'
1247 <        txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1248 <        txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1249 <        txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1250 <        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1251 <        txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1303 <        txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1246 >        txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1247 >        txt += '    echo ">>> setup CMS OSG environment:"\n'
1248 >        txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1249 >        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1250 >        txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1251 >        txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1252          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1253 <        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1254 <        txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1255 <        txt += '   else\n'
1256 <        txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1257 <        txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1258 <        txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1259 <        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1260 <        txt += '       rm -f $RUNTIME_AREA/$repo \n'
1261 <        txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1262 <        txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1263 <        txt += '       exit 1\n'
1264 <        txt += '\n'
1265 <        txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1266 <        txt += '       cd $RUNTIME_AREA\n'
1267 <        txt += '       /bin/rm -rf $WORKING_DIR\n'
1268 <        txt += '       if [ -d $WORKING_DIR ] ;then\n'
1269 <        txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1270 <        txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1271 <        txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1272 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1325 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1326 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1327 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1328 <        txt += '       fi\n'
1253 >        txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1254 >        txt += '    else\n'
1255 >        #txt += '        echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1256 >        #txt += '        echo "JOB_EXIT_STATUS = 10020"\n'
1257 >        #txt += '        echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1258 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1259 >        txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1260 >        txt += '        job_exit_code=10020\n'
1261 >        #txt += '        cd $RUNTIME_AREA\n'
1262 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1263 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
1264 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
1265 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
1266 >        #txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1267 >        #txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1268 >        #txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1269 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1270 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1271 >        #txt += '            job_exit_code=10017\n'
1272 >        #txt += '        fi\n'
1273          txt += '\n'
1274 <        txt += '       exit 1\n'
1275 <        txt += '   fi\n'
1274 >        #txt += '        exit 1\n'
1275 >        txt += '        func_exit\n'
1276 >        txt += '    fi\n'
1277          txt += '\n'
1278 <        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1279 <        txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1278 >        txt += '    echo "==> setup cms environment ok"\n'
1279 >        txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1280  
1281          return txt
1282  
# Line 1341 | Line 1286 | class Cmssw(JobType):
1286          Returns part of a job script which is prepares
1287          the execution environment and which is common for all CMS jobs.
1288          """
1289 <        txt  = '   \n'
1290 <        txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1291 <        txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1292 <        txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1293 <        txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1294 <        txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1295 <        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1296 <        txt += '       rm -f $RUNTIME_AREA/$repo \n'
1297 <        txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1298 <        txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1299 <        txt += '       exit 1\n'
1300 <        txt += '   else\n'
1301 <        txt += '       echo "Sourcing environment... "\n'
1302 <        txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1303 <        txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1304 <        txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1305 <        txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1306 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1307 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1308 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1309 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1310 <        txt += '           exit 1\n'
1311 <        txt += '       fi\n'
1312 <        txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1313 <        txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1314 <        txt += '       result=$?\n'
1315 <        txt += '       if [ $result -ne 0 ]; then\n'
1316 <        txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1317 <        txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1318 <        txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1319 <        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1320 <        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1321 <        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1322 <        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1323 <        txt += '           exit 1\n'
1324 <        txt += '       fi\n'
1325 <        txt += '   fi\n'
1326 <        txt += '   \n'
1327 <        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1328 <        txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1289 >        txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1290 >        txt += '    echo ">>> setup CMS LCG environment:"\n'
1291 >        txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1292 >        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1293 >        txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1294 >        txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1295 >        #txt += '        echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1296 >        #txt += '        echo "JOB_EXIT_STATUS = 10031" \n'
1297 >        #txt += '        echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1298 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1299 >        #txt += '        exit 1\n'
1300 >        txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1301 >        txt += '        job_exit_code=10031\n'
1302 >        txt += '        func_exit\n'
1303 >        txt += '    else\n'
1304 >        txt += '        echo "Sourcing environment... "\n'
1305 >        txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1306 >        #txt += '            echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1307 >        #txt += '            echo "JOB_EXIT_STATUS = 10020"\n'
1308 >        #txt += '            echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1309 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1310 >        #txt += '            exit 1\n'
1311 >        txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1312 >        txt += '            job_exit_code=10020\n'
1313 >        txt += '            func_exit\n'
1314 >        txt += '        fi\n'
1315 >        txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1316 >        txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1317 >        txt += '        result=$?\n'
1318 >        txt += '        if [ $result -ne 0 ]; then\n'
1319 >        #txt += '            echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1320 >        #txt += '            echo "JOB_EXIT_STATUS = 10032"\n'
1321 >        #txt += '            echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1322 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1323 >        #txt += '            exit 1\n'
1324 >        txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1325 >        txt += '            job_exit_code=10032\n'
1326 >        txt += '            func_exit\n'
1327 >        txt += '        fi\n'
1328 >        txt += '    fi\n'
1329 >        txt += '    \n'
1330 >        txt += '    echo "==> setup cms environment ok"\n'
1331          return txt
1332  
1333      ### FEDE FOR DBS OUTPUT PUBLICATION
# Line 1389 | Line 1336 | class Cmssw(JobType):
1336          insert the part of the script that modifies the FrameworkJob Report
1337          """
1338  
1339 <        txt = ''
1339 >        txt = '\n#Written by cms_cmssw::modifyReport\n'
1340          try:
1341              publish_data = int(self.cfg_params['USER.publish_data'])
1342          except KeyError:
1343              publish_data = 0
1344          if (publish_data == 1):
1345 <            txt += 'echo "Modify Job Report" \n'
1346 <            #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1347 <            ################ FEDE FOR DBS2 #############################################
1348 <            txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1349 <            #############################################################################
1350 <            #try:
1351 <            #    publish_data = int(self.cfg_params['USER.publish_data'])
1352 <            #except KeyError:
1353 <            #    publish_data = 0
1354 <
1355 <            txt += 'if [ -z "$SE" ]; then\n'
1356 <            txt += '    SE="" \n'
1410 <            txt += 'fi \n'
1411 <            txt += 'if [ -z "$SE_PATH" ]; then\n'
1412 <            txt += '    SE_PATH="" \n'
1413 <            txt += 'fi \n'
1414 <            txt += 'echo "SE = $SE"\n'
1415 <            txt += 'echo "SE_PATH = $SE_PATH"\n'
1345 >            
1346 >            txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1347 >            txt += '    echo ">>> Modify Job Report:" \n'
1348 >            txt += '    chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1349 >            #txt += '    if [ -z "$SE" ]; then\n'
1350 >            #txt += '        SE="" \n'
1351 >            #txt += '    fi \n'
1352 >            #txt += '    if [ -z "$SE_PATH" ]; then\n'
1353 >            #txt += '        SE_PATH="" \n'
1354 >            #txt += '    fi \n'
1355 >            txt += '    echo "SE = $SE"\n'
1356 >            txt += '    echo "SE_PATH = $SE_PATH"\n'
1357  
1417 –        #if (publish_data == 1):
1418 –            #processedDataset = self.cfg_params['USER.processed_datasetname']
1358              processedDataset = self.cfg_params['USER.publish_data_name']
1359 <            txt += 'ProcessedDataset='+processedDataset+'\n'
1360 <            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1361 <            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1362 <            #### FEDE: added slash in LFN ##############
1363 <            txt += '    FOR_LFN=/copy_problems/ \n'
1364 <            txt += 'else \n'
1359 >            txt += '    ProcessedDataset='+processedDataset+'\n'
1360 >            #txt += '    if [ "$SE_PATH" == "" ]; then\n'
1361 >            #txt += '        FOR_LFN=/copy_problems/ \n'
1362 >            #txt += '    else \n'
1363 >            #txt += '        tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1364 >            #txt += '        FOR_LFN=/store$tmp \n'
1365 >            #txt += '    fi \n'
1366              txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1427 –            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1367              txt += '    FOR_LFN=/store$tmp \n'
1368 <            txt += 'fi \n'
1369 <            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1370 <            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1371 <            txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1372 <            #txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1373 <            txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1374 <            txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1375 <            #txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1376 <
1377 <            txt += 'modifyReport_result=$?\n'
1378 <            txt += 'echo modifyReport_result = $modifyReport_result\n'
1379 <            txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1380 <            txt += '    exit_status=1\n'
1381 <            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1382 <            txt += 'else\n'
1383 <            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1368 >            txt += '    echo "ProcessedDataset = $ProcessedDataset"\n'
1369 >            txt += '    echo "FOR_LFN = $FOR_LFN" \n'
1370 >            txt += '    echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1371 >            #txt += '    echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1372 >            #txt += '    $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1373 >            ### FEDE ####
1374 >            txt += '    echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1375 >            txt += '    $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1376 >            ####################################
1377 >            txt += '    modifyReport_result=$?\n'
1378 >            txt += '    if [ $modifyReport_result -ne 0 ]; then\n'
1379 >            txt += '        modifyReport_result=70500\n'
1380 >            txt += '        job_exit_code=$modifyReport_result\n'
1381 >            txt += '        echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1382 >            txt += '        echo "WARNING: Problem with ModifyJobReport"\n'
1383 >            txt += '    else\n'
1384 >            ### FEDE #####
1385 >            #txt += '        mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1386 >            #######################
1387 >            txt += '        mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1388 >            txt += '    fi\n'
1389              txt += 'fi\n'
1446 –        else:
1447 –            txt += 'echo "no data publication required"\n'
1448 –            #txt += 'ProcessedDataset=no_data_to_publish \n'
1449 –            #### FEDE: added slash in LFN ##############
1450 –            #txt += 'FOR_LFN=/local/ \n'
1451 –            #txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1452 –            #txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1390          return txt
1391  
1392      def cleanEnv(self):
1393 <        ### OLI_DANIELE
1457 <        txt = ''
1393 >        txt = '\n#Written by cms_cmssw::cleanEnv\n'
1394          txt += 'if [ $middleware == OSG ]; then\n'
1395          txt += '    cd $RUNTIME_AREA\n'
1396 <        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1396 >        txt += '    echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1397 >        txt += '    echo ">>> Remove working directory: $WORKING_DIR"\n'
1398          txt += '    /bin/rm -rf $WORKING_DIR\n'
1399          txt += '    if [ -d $WORKING_DIR ] ;then\n'
1400 <        txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401 <        txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1402 <        txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1466 <        txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1467 <        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1468 <        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1469 <        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1400 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401 >        txt += '        job_exit_code=10017\n'
1402 >        txt += '        func_exit\n'
1403          txt += '    fi\n'
1404          txt += 'fi\n'
1405          txt += '\n'
# Line 1478 | Line 1411 | class Cmssw(JobType):
1411      def getParams(self):
1412          return self._params
1413  
1481 –    def setTaskid_(self):
1482 –        self._taskId = self.cfg_params['taskId']
1483 –
1484 –    def getTaskid(self):
1485 –        return self._taskId
1486 –
1414      def uniquelist(self, old):
1415          """
1416          remove duplicates from a list
# Line 1493 | Line 1420 | class Cmssw(JobType):
1420              nd[e]=0
1421          return nd.keys()
1422  
1423 <
1497 <    def checkOut(self, limit):
1423 >    def outList(self):
1424          """
1425          check the dimension of the output files
1426          """
1427 <        txt = 'echo "*****************************************"\n'
1428 <        txt += 'echo "** Starting output sandbox limit check **"\n'
1503 <        txt += 'echo "*****************************************"\n'
1504 <        allOutFiles = ""
1427 >        txt = ''
1428 >        txt += 'echo ">>> list of expected files on output sandbox"\n'
1429          listOutFiles = []
1430 <        for fileOut in (self.output_file+self.output_file_sandbox):
1431 <             if fileOut.find('crab_fjr') == -1:
1432 <                 allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1433 <                 listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1434 <        txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1435 <        txt += 'ls -gGhrta;\n'
1436 <        txt += 'sum=0;\n'
1437 <        txt += 'for file in '+str(allOutFiles)+' ; do\n'
1438 <        txt += '    if [ -e $file ]; then\n'
1439 <        txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1440 <        txt += '        sum=`expr $sum + $tt`\n'
1441 <        txt += '    else\n'
1442 <        txt += '        echo "WARNING: output file $file not found!"\n'
1443 <        txt += '    fi\n'
1444 <        txt += 'done\n'
1445 <        txt += 'echo "Total Output dimension: $sum";\n'
1522 <        txt += 'limit='+str(limit)+';\n'
1523 <        txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1524 <        txt += 'if [ $limit -lt $sum ]; then\n'
1525 <        txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1526 <        txt += '    echo "         checking the output file sizes..."\n'
1527 <        """
1528 <        txt += '    dim=0;\n'
1529 <        txt += '    exclude=0;\n'
1530 <        txt += '    for files in '+str(allOutFiles)+' ; do\n'
1531 <        txt += '        sumTemp=0;\n'
1532 <        txt += '        for file2 in '+str(allOutFiles)+' ; do\n'
1533 <        txt += '            if [ $file != $file2 ]; then\n'
1534 <        txt += '                tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1535 <        txt += '                sumTemp=`expr $sumTemp + $tt`;\n'
1536 <        txt += '            fi\n'
1537 <        txt += '        done\n'
1538 <        txt += '        if [ $sumTemp -lt $limit ]; then\n'
1539 <        txt += '            if [ $dim -lt $sumTemp ]; then\n'
1540 <        txt += '                dim=$sumTemp;\n'
1541 <        txt += '                exclude=$file;\n'
1542 <        txt += '            fi\n'
1543 <        txt += '        fi\n'
1544 <        txt += '    done\n'
1545 <        txt += '    echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1546 <        """
1547 <        txt += '    tot=0;\n'
1548 <        txt += '    for file2 in '+str(allOutFiles)+' ; do\n'
1549 <        txt += '        tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1550 <        txt += '        tot=`expr $tot + $tt`;\n'
1551 <        txt += '        if [ $limit -lt $tot ]; then\n'
1552 <        txt += '            tot=`expr $tot - $tt`;\n'
1553 <        txt += '            fileLast=$file;\n'
1554 <        txt += '            break;\n'
1555 <        txt += '        fi\n'
1556 <        txt += '    done\n'
1557 <        txt += '    echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1558 <        txt += '    flag=0;\n'
1559 <        txt += '    for filess in '+str(allOutFiles)+' ; do\n'
1560 <        txt += '        if [ $fileLast = $filess ]; then\n'
1561 <        txt += '            flag=1;\n'
1562 <        txt += '        fi\n'
1563 <        txt += '        if [ $flag -eq 1 ]; then\n'
1564 <        txt += '            rm -f $filess;\n'
1565 <        txt += '        fi\n'
1566 <        txt += '    done\n'
1567 <        txt += '    ls -agGhrt;\n'
1568 <        txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1569 <        txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1570 <        txt += '    exit_status=70000;\n'
1571 <        txt += 'else'
1572 <        txt += '    echo "Total Output dimension $sum is fine.";\n'
1573 <        txt += 'fi\n'
1574 <        txt += 'echo "*****************************************"\n'
1575 <        txt += 'echo "*** Ending output sandbox limit check ***"\n'
1576 <        txt += 'echo "*****************************************"\n'
1577 <        return txt
1430 >        stdout = 'CMSSW_$NJob.stdout'
1431 >        stderr = 'CMSSW_$NJob.stderr'
1432 >        if (self.return_data == 1):
1433 >            for file in (self.output_file+self.output_file_sandbox):
1434 >                listOutFiles.append(self.numberFile_(file, '$NJob'))
1435 >            listOutFiles.append(stdout)
1436 >            listOutFiles.append(stderr)
1437 >        else:
1438 >            for file in (self.output_file_sandbox):
1439 >                listOutFiles.append(self.numberFile_(file, '$NJob'))
1440 >            listOutFiles.append(stdout)
1441 >            listOutFiles.append(stderr)
1442 >        txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1443 >        txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1444 >        txt += 'export filesToCheck\n'
1445 >        return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines