ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.2 by slacapra, Wed Apr 26 15:31:06 2006 UTC vs.
Revision 1.3 by gutsche, Sun May 28 02:27:52 2006 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   import common
6 + import PsetManipulator  
7  
8 < import DataDiscovery
9 < import DataLocation
8 > import DBSInfo_EDM
9 > #from DataDiscovery_EDM import DataDiscovery_EDM
10 > import DataDiscovery_EDM
11 > #from DataLocation_EDM import DataLocation_EDM
12 > import DataLocation_EDM
13   import Scram
14  
15   import os, string, re
# Line 16 | Line 20 | class Cmssw(JobType):
20          common.logger.debug(3,'CMSSW::__init__')
21  
22          self.analisys_common_info = {}
23 +        # Marco.
24 +        self._params = {}
25 +        self.cfg_params = cfg_params
26  
27          log = common.logger
28          
# Line 26 | Line 33 | class Cmssw(JobType):
33          self.executable = ''
34          self.tgz_name = 'default.tgz'
35  
36 +
37          self.version = self.scram.getSWVersion()
38          common.analisys_common_info['sw_version'] = self.version
39 +        ### FEDE
40 +        common.analisys_common_info['copy_input_data'] = 0
41 +        common.analisys_common_info['events_management'] = 1
42  
43          ### collect Data cards
44          try:
45 <            self.owner = cfg_params['CMSSW.owner']
46 <            log.debug(6, "CMSSW::CMSSW(): owner = "+self.owner)
47 <            self.dataset = cfg_params['CMSSW.dataset']
48 <            log.debug(6, "CMSSW::CMSSW(): dataset = "+self.dataset)
45 >         #   self.owner = cfg_params['CMSSW.owner']
46 >         #   log.debug(6, "CMSSW::CMSSW(): owner = "+self.owner)
47 >         #   self.dataset = cfg_params['CMSSW.dataset']
48 >            self.datasetPath = cfg_params['CMSSW.datasetpath']
49 >            log.debug(6, "CMSSW::CMSSW(): datasetPath = "+self.datasetPath)
50          except KeyError:
51 <            msg = "Error: owner and/or dataset not defined "
51 >        #    msg = "Error: owner and/or dataset not defined "
52 >            msg = "Error: datasetpath not defined "  
53              raise CrabException(msg)
41 –
54          self.dataTiers = []
55 <        try:
56 <            tmpDataTiers = string.split(cfg_params['CMSSW.data_tier'],',')
57 <            for tmp in tmpDataTiers:
58 <                tmp=string.strip(tmp)
59 <                self.dataTiers.append(tmp)
60 <                pass
61 <            pass
62 <        except KeyError:
63 <            pass
64 <        log.debug(6, "Cmssw::Cmssw(): dataTiers = "+str(self.dataTiers))
55 > #       try:
56 > #           tmpDataTiers = string.split(cfg_params['CMSSW.data_tier'],',')
57 > #           for tmp in tmpDataTiers:
58 > #               tmp=string.strip(tmp)
59 > #               self.dataTiers.append(tmp)
60 > #               pass
61 > #           pass
62 > #       except KeyError:
63 > #           pass
64 > #       log.debug(6, "Cmssw::Cmssw(): dataTiers = "+str(self.dataTiers))
65  
66          ## now the application
67          try:
# Line 83 | Line 95 | class Cmssw(JobType):
95                      tmp=string.strip(tmp)
96                      self.output_file.append(tmp)
97                      pass
86 –
98              else:
99                  log.message("No output file defined: only stdout/err will be available")
100                  pass
# Line 94 | Line 105 | class Cmssw(JobType):
105  
106          # script_exe file as additional file in inputSandbox
107          try:
108 <           self.scriptExe = cfg_params['CMSSW.script_exe']
108 >           self.scriptExe = cfg_params['USER.script_exe']
109             self.additional_inbox_files.append(self.scriptExe)
110          except KeyError:
111             pass
# Line 109 | Line 120 | class Cmssw(JobType):
120          try:
121              tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
122              for tmp in tmpAddFiles:
123 +                if not os.path.exists(tmp):
124 +                    raise CrabException("Additional input file not found: "+tmp)
125                  tmp=string.strip(tmp)
126                  self.additional_inbox_files.append(tmp)
127                  pass
# Line 117 | Line 130 | class Cmssw(JobType):
130              pass
131  
132          try:
133 +            self.filesPerJob = int(cfg_params['CMSSW.files_per_jobs']) #Daniele
134 +        except KeyError:
135 +            self.filesPerJob = 1
136 +
137 +        ## Max event   will be total_number_of_events ???  Daniele
138 +        try:
139 +            self.maxEv = cfg_params['CMSSW.event_per_job']
140 +        except KeyError:
141 +            self.maxEv = "-1"
142 +        ##  
143 +        try:
144              self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
145          except KeyError:
146 <            msg = 'Must define total_number_of_events and job_number_of_events'
146 >            msg = 'Must define total_number_of_events'
147              raise CrabException(msg)
124 –            
125 – #Marco: FirstEvent is nolonger used inside PSet
126 – #        try:
127 – #            self.first = int(cfg_params['CMSSW.first_event'])
128 – #        except KeyError:
129 – #            self.first = 0
130 – #            pass
131 – #        log.debug(6, "Orca::Orca(): total number of events = "+`self.total_number_of_events`)
132 –        #log.debug(6, "Orca::Orca(): events per job = "+`self.job_number_of_events`)
133 – #        log.debug(6, "Orca::Orca(): first event = "+`self.first`)
148          
149          CEBlackList = []
150          try:
# Line 150 | Line 164 | class Cmssw(JobType):
164          CEWhiteList = []
165          try:
166              tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
153 –            #tmpGood = ['cern']
167              for tmp in tmpGood:
168                  tmp=string.strip(tmp)
156 –                #if (tmp == 'cnaf'): tmp = 'webserver' ########## warning: temp. patch
169                  CEWhiteList.append(tmp)
170          except KeyError:
171              pass
# Line 165 | Line 177 | class Cmssw(JobType):
177  
178          common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
179  
180 +        self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
181 +
182          #DBSDLS-start
183          ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
184          self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
# Line 175 | Line 189 | class Cmssw(JobType):
189  
190          self.tgzNameWithPath = self.getTarBall(self.executable)
191  
192 +        self.jobSplitting()  #Daniele job Splitting
193 +        self.PsetEdit.maxEvent(self.maxEv) #Daniele  
194 +        self.PsetEdit.inputModule("INPUT") #Daniele  
195 +        self.PsetEdit.psetWriter(self.configFilename())
196 +
197      def DataDiscoveryAndLocation(self, cfg_params):
198  
199 <        fun = "CMSSW::DataDiscoveryAndLocation()"
199 >        common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
200 >
201 >        #datasetPath = "/"+self.owner+"/"+self.dataTiers[0]+"/"+self.dataset
202 >        
203 >        datasetPath=self.datasetPath
204 >
205 >        ## TODO
206 >        dataTiersList = ""
207 >        dataTiers = dataTiersList.split(',')
208  
209          ## Contact the DBS
210          try:
211 <            self.pubdata=DataDiscovery.DataDiscovery(self.owner,
185 <                                                     self.dataset,
186 <                                                     self.dataTiers,
187 <                                                     cfg_params)
211 >            self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, dataTiers)
212              self.pubdata.fetchDBSInfo()
213  
214 <        except DataDiscovery.NotExistingDatasetError, ex :
214 >        except DataDiscovery_EDM.NotExistingDatasetError, ex :
215              msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
216              raise CrabException(msg)
217  
218 <        except DataDiscovery.NoDataTierinProvenanceError, ex :
218 >        except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
219              msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
220              raise CrabException(msg)
221 <        except DataDiscovery.DataDiscoveryError, ex:
221 >        except DataDiscovery_EDM.DataDiscoveryError, ex:
222              msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
223              raise CrabException(msg)
224  
225          ## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
226 <        self.DBSPaths=self.pubdata.getDBSPaths()
227 <        common.logger.message("Required data are : ")
228 <        for path in self.DBSPaths:
229 <            common.logger.message(" --> "+path )
226 >        ## self.DBSPaths=self.pubdata.getDBSPaths()
227 >        common.logger.message("Required data are :"+self.datasetPath)
228 >
229 >        filesbyblock=self.pubdata.getFiles()
230 >        self.AllInputFiles=filesbyblock.values()
231 >        self.files = self.AllInputFiles        
232 >
233 >        ## TEMP
234 >    #    self.filesTmp = filesbyblock.values()
235 >    #    self.files = []
236 >    #    locPath='rfio:cmsbose2.bo.infn.it:/flatfiles/SE00/cms/fanfani/ProdTest/'
237 >    #    locPath=''
238 >    #    tmp = []
239 >    #    for file in self.filesTmp[0]:
240 >    #        tmp.append(locPath+file)
241 >    #    self.files.append(tmp)
242 >        ## END TEMP
243  
244          ## get max number of events
245 <        common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
245 >        #common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
246          self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
247          common.logger.message("\nThe number of available events is %s"%self.maxEvents)
248  
212 –        ## get fileblocks corresponding to the required data
213 –        fb=self.pubdata.getFileBlocks()
214 –        common.logger.debug(5,"fileblocks are %s"%fb)
215 –
249          ## Contact the DLS and build a list of sites hosting the fileblocks
250          try:
251 <            dataloc=DataLocation.DataLocation(self.pubdata.getFileBlocks(),cfg_params)
251 >            dataloc=DataLocation_EDM.DataLocation_EDM(filesbyblock.keys(),cfg_params)
252              dataloc.fetchDLSInfo()
253 <        except DataLocation.DataLocationError , ex:
253 >        except DataLocation_EDM.DataLocationError , ex:
254              msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
255              raise CrabException(msg)
256          
# Line 231 | Line 264 | class Cmssw(JobType):
264          if len(sites)==0:
265              msg = 'No sites hosting all the needed data! Exiting... '
266              raise CrabException(msg)
267 +
268          common.logger.message("List of Sites hosting the data : "+str(sites))
269          common.logger.debug(6, "List of Sites: "+str(sites))
270          common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
271          return
272 +    
273 +    def jobSplitting(self):
274 +        """
275 +        first implemntation for job splitting  
276 +        """    
277 +      #  print 'eventi totali '+str(self.maxEvents)
278 +      #  print 'eventi totali richiesti dallo user '+str(self.total_number_of_events)
279 +        #print 'files per job '+str(self.filesPerJob)
280 +        common.logger.message('Required '+str(self.filesPerJob)+' files per job ')
281 +        common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
282 +
283 +        ## TODO: SL need to have (from DBS) a detailed list of how many events per each file
284 +        n_tot_files = (len(self.files[0]))
285 +        ## SL: this is wrong if the files have different number of events
286 +        evPerFile = int(self.maxEvents)/n_tot_files
287 +        
288 +        common.logger.debug(5,'Events per File '+str(evPerFile))
289 +
290 +        ## if asked to process all events, do it
291 +        if self.total_number_of_events == -1:
292 +            self.total_number_of_events=self.maxEvents
293 +            self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
294 +            common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for all available events '+str(self.total_number_of_events)+' events')
295 +        
296 +        else:
297 +            self.total_number_of_files = int(self.total_number_of_events/evPerFile)
298 +            ## SL: if ask for less event than what is computed to be available on a
299 +            ##     file, process the first file anyhow.
300 +            if self.total_number_of_files == 0:
301 +                self.total_number_of_files = self.total_number_of_files + 1
302 +
303 +            common.logger.debug(5,'N files  '+str(self.total_number_of_files))
304 +
305 +            check = 0
306 +            
307 +            ## Compute the number of jobs
308 +            #self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
309 +            self.total_number_of_jobs = int(self.total_number_of_files/self.filesPerJob)
310 +            common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
311 +
312 +            ## is there any remainder?
313 +            check = int(self.total_number_of_files) - (int(self.total_number_of_jobs)*self.filesPerJob)
314 +
315 +            common.logger.debug(5,'Check  '+str(check))
316 +
317 +            if check > 0:
318 +                self.total_number_of_jobs =  self.total_number_of_jobs + 1
319 +                common.logger.message('Warning: last job will be created with '+str(check)+' files')
320 +
321 +            common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs-1)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
322 +            pass
323 +
324 +        list_of_lists = []
325 +        for i in xrange(0, int(n_tot_files), self.filesPerJob):
326 +            list_of_lists.append(self.files[0][i: i+self.filesPerJob])
327 +
328 +        self.list_of_files = list_of_lists
329 +      
330 +        return
331 +
332 +    def split(self, jobParams):
333 +
334 +        common.jobDB.load()
335 +        #### Fabio
336 +        njobs = self.total_number_of_jobs
337 +        filelist = self.list_of_files
338 +        # create the empty structure
339 +        for i in range(njobs):
340 +            jobParams.append("")
341 +        
342 +        for job in range(njobs):
343 +            jobParams[job] = filelist[job]
344 +            common.jobDB.setArguments(job, jobParams[job])
345 +
346 +        common.jobDB.save()
347 +        return
348 +    
349 +    def getJobTypeArguments(self, nj, sched):
350 +        params = common.jobDB.arguments(nj)
351 +        #print params
352 +        parString = "\\{"
353 +        
354 +        for i in range(len(params) - 1):
355 +            parString += '\\\"' + params[i] + '\\\"\,'
356          
357 +        parString += '\\\"' + params[len(params) - 1] + '\\\"\\}'
358 +        return parString
359 +  
360 +    def numberOfJobs(self):
361 +        # Fabio
362 +
363 +        return self.total_number_of_jobs
364 +
365 +
366 +
367      def checkBlackList(self, allSites):
368          if len(self.reCEBlackList)==0: return allSites
369          sites = []
# Line 252 | Line 380 | class Cmssw(JobType):
380              common.logger.debug(3,"No sites found after BlackList")
381          return sites
382  
383 <    def checkWhiteList(self, allsites):
383 >    def checkWhiteList(self, allSites):
384  
385 <        if len(self.reCEWhiteList)==0: return pubDBUrls
385 >        if len(self.reCEWhiteList)==0: return allSites
386          sites = []
387 <        for site in allsites:
260 <            #print 'connecting to the URL ',url
387 >        for site in allSites:
388              good=0
389              for re in self.reCEWhiteList:
390                  if re.search(site):
# Line 327 | Line 454 | class Cmssw(JobType):
454          if os.path.exists(lib):
455              filesToBeTarred.append(libDir)
456  
457 +        ## Now check if module dir is present
458 +        moduleDir = 'module'
459 +        if os.path.isdir(swArea+'/'+moduleDir):
460 +            filesToBeTarred.append(moduleDir)
461 +
462          ## Now check if the Data dir is present
463          dataDir = 'src/Data/'
464          if os.path.isdir(swArea+'/'+dataDir):
# Line 354 | Line 486 | class Cmssw(JobType):
486          the execution environment for the job 'nj'.
487          """
488          # Prepare JobType-independent part
489 <        txt = self.wsSetupCMSEnvironment_()
489 >        txt = ''
490 >  
491 >        ## OLI_Daniele at this level  middleware already known
492 >
493 >        txt += 'if [ $middleware == LCG ]; then \n'
494 >        txt += self.wsSetupCMSLCGEnvironment_()
495 >        txt += 'elif [ $middleware == OSG ]; then\n'
496 >        txt += '    time=`date -u +"%s"`\n'
497 >        txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
498 >        txt += '    echo "Creating working directory: $WORKING_DIR"\n'
499 >        txt += '    /bin/mkdir -p $WORKING_DIR\n'
500 >        txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
501 >        txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be created on on WN `hostname`"\n'
502 >    
503 >        txt += '        echo "JOB_EXIT_STATUS = 1"\n'
504 >        txt += '        exit 1\n'
505 >        txt += '    fi\n'
506 >        txt += '\n'
507 >        txt += '    echo "Change to working directory: $WORKING_DIR"\n'
508 >        txt += '    cd $WORKING_DIR\n'
509 >        txt += self.wsSetupCMSOSGEnvironment_()
510 >        txt += 'fi\n'
511  
512          # Prepare JobType-specific part
513          scram = self.scram.commandName()
# Line 364 | Line 517 | class Cmssw(JobType):
517          txt += 'status=$?\n'
518          txt += 'if [ $status != 0 ] ; then\n'
519          txt += '   echo "SET_EXE_ENV 1 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
520 <        txt += '   echo "JOB_EXIT_STATUS = 5"\n'
521 <        txt += '   echo "SanityCheckCode = 5" | tee -a $RUNTIME_AREA/$repo\n'
520 >        txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
521 >        txt += '   echo "SanityCheckCode = 10034" | tee -a $RUNTIME_AREA/$repo\n'
522          txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
523 <        txt += '   exit 5 \n'
523 >        ## OLI_Daniele
524 >        txt += '    if [ $middleware == OSG ]; then \n'
525 >        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
526 >        txt += '        cd $RUNTIME_AREA\n'
527 >        txt += '        /bin/rm -rf $WORKING_DIR\n'
528 >        txt += '        if [ -d $WORKING_DIR ] ;then\n'
529 >        txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
530 >        txt += '        fi\n'
531 >        txt += '    fi \n'
532 >        txt += '   exit 1 \n'
533          txt += 'fi \n'
534          txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
535          txt += 'cd '+self.version+'\n'
# Line 381 | Line 543 | class Cmssw(JobType):
543          # txt += "## ARGUMNETS: $3 Max Event for this job\n"
544          txt += "\n"
545          txt += "narg=$#\n"
546 <        txt += "if [ $narg -lt 1 ]\n"
546 >        txt += "if [ $narg -lt 2 ]\n"
547          txt += "then\n"
548          txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
549 <        txt += '    echo "JOB_EXIT_STATUS = 1"\n'
550 <        txt += '    echo "SanityCheckCode = 1" | tee -a $RUNTIME_AREA/$repo\n'
549 >        txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
550 >        txt += '    echo "SanityCheckCode = 50113" | tee -a $RUNTIME_AREA/$repo\n'
551          txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
552 +        ## OLI_Daniele
553 +        txt += '    if [ $middleware == OSG ]; then \n'
554 +        txt += '        echo "Remove working directory: $WORKING_DIR"\n'
555 +        txt += '        cd $RUNTIME_AREA\n'
556 +        txt += '        /bin/rm -rf $WORKING_DIR\n'
557 +        txt += '        if [ -d $WORKING_DIR ] ;then\n'
558 +        txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
559 +        txt += '        fi\n'
560 +        txt += '    fi \n'
561          txt += "    exit 1\n"
562          txt += "fi\n"
563          txt += "\n"
564          txt += "NJob=$1\n"
565 <        # txt += "FirstEvent=$2\n"
566 <        # txt += "MaxEvents=$3\n"
565 >        txt += "InputFiles=$2\n"
566 >        txt += "echo \"<$InputFiles>\"\n"
567 >        # txt += "Args = ` cat $2 |  sed -e \'s/\\\\//g\' -e \'s/\"/\\x27/g\' `"
568 >
569 >        ### OLI_DANIELE
570 >        txt += 'if [ $middleware == LCG ]; then \n'
571 >        txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
572 >        txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
573 >        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo\n'
574 >        txt += 'elif [ $middleware == OSG ]; then\n'
575 >
576 >        # OLI: added monitoring for dashbord, use hash of crab.cfg
577 >        if common.scheduler.boss_scheduler_name == 'condor_g':
578 >            # create hash of cfg file
579 >            hash = makeCksum(common.work_space.cfgFileName())
580 >            txt += '    echo "MonitorJobID=`echo ${NJob}_'+hash+'_$GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
581 >            txt += '    echo "SyncGridJobId=`echo $GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
582 >            txt += '    echo "SyncCE=`echo $hostname`" | tee -a $RUNTIME_AREA/$repo\n'
583 >        else :
584 >            txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
585 >            txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
586 >            txt += '    echo "SyncCE=`$EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo\n'
587 >
588 >        txt += 'fi\n'
589 >        txt += 'dumpStatus $RUNTIME_AREA/$repo\n'
590  
591          # Prepare job-specific part
592          job = common.job_list[nj]
593          pset = os.path.basename(job.configFilename())
594          txt += '\n'
595 <        txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
596 <        # txt += 'if [ -e $RUNTIME_AREA/orcarc_$CE ] ; then\n'
597 <        # txt += '  cat $RUNTIME_AREA/orcarc_$CE .orcarc >> .orcarc_tmp\n'
404 <        # txt += '  mv .orcarc_tmp .orcarc\n'
405 <        # txt += 'fi\n'
406 <        # txt += 'if [ -e $RUNTIME_AREA/init_$CE.sh ] ; then\n'
407 <        # txt += '  cp $RUNTIME_AREA/init_$CE.sh init.sh\n'
408 <        # txt += 'fi\n'
595 >        #txt += 'echo sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' \n'
596 >        txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
597 >        #txt += 'sed "s#{\'INPUT\'}#${InputFiles}#" $RUNTIME_AREA/'+pset+' > pset1.cfg\n'
598  
599          if len(self.additional_inbox_files) > 0:
600              for file in self.additional_inbox_files:
# Line 415 | Line 604 | class Cmssw(JobType):
604                  txt += 'fi\n'
605              pass
606  
418 –        # txt += '\n'
419 –        # txt += 'chmod +x ./init.sh\n'
420 –        # txt += './init.sh\n'
421 –        # txt += 'exitStatus=$?\n'
422 –        # txt += 'if [ $exitStatus != 0 ] ; then\n'
423 –        # txt += '  echo "SET_EXE_ENV 1 ==> ERROR StageIn init script failed"\n'
424 –        # txt += '  echo "JOB_EXIT_STATUS = $exitStatus" \n'
425 –        # txt += '  echo "SanityCheckCode = $exitStatus" | tee -a $RUNTIME_AREA/$repo\n'
426 –        # txt += '  dumpStatus $RUNTIME_AREA/$repo\n'
427 –        # txt += '  exit $exitStatus\n'
428 –        # txt += 'fi\n'
429 –        # txt += "echo 'SET_EXE_ENV 0 ==> job setup ok'\n"
607          txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
608  
432 –        # txt += 'echo "FirstEvent=$FirstEvent" >> .orcarc\n'
433 –        # txt += 'echo "MaxEvents=$MaxEvents" >> .orcarc\n'
434 –        # if self.ML:
435 –        #     txt += 'echo "MonalisaJobId=$NJob" >> .orcarc\n'
436 –
609          txt += '\n'
610          txt += 'echo "***** cat pset.cfg *********"\n'
611          txt += 'cat pset.cfg\n'
612          txt += 'echo "****** end pset.cfg ********"\n'
613 +        txt += '\n'
614 +        # txt += 'echo "***** cat pset1.cfg *********"\n'
615 +        # txt += 'cat pset1.cfg\n'
616 +        # txt += 'echo "****** end pset1.cfg ********"\n'
617 +        return txt
618 +
619 +    def wsBuildExe(self, nj):
620 +        """
621 +        Put in the script the commands to build an executable
622 +        or a library.
623 +        """
624 +
625 +        txt = ""
626 +
627 +        if os.path.isfile(self.tgzNameWithPath):
628 +            txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
629 +            txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
630 +            txt += 'untar_status=$? \n'
631 +            txt += 'if [ $untar_status -ne 0 ]; then \n'
632 +            txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
633 +            txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
634 +            txt += '   echo "SanityCheckCode = $untar_status" | tee -a $repo\n'
635 +            txt += '   if [ $middleware == OSG ]; then \n'
636 +            txt += '       echo "Remove working directory: $WORKING_DIR"\n'
637 +            txt += '       cd $RUNTIME_AREA\n'
638 +            txt += '       /bin/rm -rf $WORKING_DIR\n'
639 +            txt += '       if [ -d $WORKING_DIR ] ;then\n'
640 +            txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
641 +            txt += '       fi\n'
642 +            txt += '   fi \n'
643 +            txt += '   \n'
644 +            txt += '   exit $untar_status \n'
645 +            txt += 'else \n'
646 +            txt += '   echo "Successful untar" \n'
647 +            txt += 'fi \n'
648 +            pass
649 +        
650          return txt
651  
652      def modifySteeringCards(self, nj):
# Line 450 | Line 659 | class Cmssw(JobType):
659          return self.executable
660  
661      def executableArgs(self):
662 <        return "-p pset.cfg"
662 >        return " -p pset.cfg"
663  
664      def inputSandbox(self, nj):
665          """
# Line 465 | Line 674 | class Cmssw(JobType):
674          ## config
675          inp_box.append(common.job_list[nj].configFilename())
676          ## additional input files
677 <        for file in self.additional_inbox_files:
678 <            inp_box.append(common.work_space.cwdDir()+file)
470 <        #print "sono inputSandbox, inp_box = ", inp_box
677 >        #for file in self.additional_inbox_files:
678 >        #    inp_box.append(common.work_space.cwdDir()+file)
679          return inp_box
680  
681      def outputSandbox(self, nj):
# Line 490 | Line 698 | class Cmssw(JobType):
698          """
699          Make initial modifications of the user's steering card file.
700          """
493 –        infile = open(self.pset,'r')
494 –            
495 –        outfile = open(common.work_space.jobDir()+self.name()+'.cfg', 'w')
496 –          
497 –        outfile.write('\n\n##### The following cards have been created by CRAB: DO NOT TOUCH #####\n')
498 –
499 –        outfile.write('InputCollections=/System/'+self.owner+'/'+self.dataset+'/'+self.dataset+'\n')
500 –
501 –        infile.close()
502 –        outfile.close()
701          return
702  
703      def wsRenameOutput(self, nj):
# Line 509 | Line 707 | class Cmssw(JobType):
707  
708          txt = '\n'
709          file_list = ''
710 +        check = len(self.output_file)
711 +        i = 0
712          for fileWithSuffix in self.output_file:
713 +            i= i + 1
714              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
715 <            file_list=file_list+output_file_num+' '
715 >            file_list=file_list+output_file_num+''
716              txt += '\n'
717              txt += 'ls \n'
718              txt += '\n'
# Line 519 | Line 720 | class Cmssw(JobType):
720              txt += 'exe_result=$?\n'
721              txt += 'if [ $exe_result -ne 0 ] ; then\n'
722              txt += '   echo "ERROR: No output file to manage"\n'
723 <            txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
724 <            txt += '   echo "SanityCheckCode = $exe_result" | tee -a $RUNTIME_AREA/$repo\n'
725 <            txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
726 <            txt += '   exit $exe_result \n'
723 >            ### OLI_DANIELE
724 >            txt += '    if [ $middleware == OSG ]; then \n'
725 >            txt += '        echo "prepare dummy output file"\n'
726 >            txt += '        cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
727 >            txt += '    fi \n'
728              txt += 'else\n'
729              txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
730              txt += 'fi\n'
731 <            txt += 'cd $RUNTIME_AREA\n'
732 <                      
731 >            if i == check:
732 >                txt += 'cd $RUNTIME_AREA\n'
733 >                pass      
734              pass
735        
736          file_list=file_list[:-1]
737          txt += 'file_list="'+file_list+'"\n'
738 +        ### OLI_DANIELE
739 +        txt += 'if [ $middleware == OSG ]; then\n'  
740 +        txt += '    cd $RUNTIME_AREA\n'
741 +        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
742 +        txt += '    /bin/rm -rf $WORKING_DIR\n'
743 +        txt += '    if [ -d $WORKING_DIR ] ;then\n'
744 +        txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
745 +        txt += '    fi\n'
746 +        txt += 'fi\n'
747 +        txt += '\n'
748          return txt
749  
750      def numberFile_(self, file, txt):
# Line 574 | Line 787 | class Cmssw(JobType):
787              req = req + ')'
788          #print "req = ", req
789          return req
790 +
791 +    def configFilename(self):
792 +        """ return the config filename """
793 +        return self.name()+'.cfg'
794 +
795 +    ### OLI_DANIELE
796 +    def wsSetupCMSOSGEnvironment_(self):
797 +        """
798 +        Returns part of a job script which is prepares
799 +        the execution environment and which is common for all CMS jobs.
800 +        """
801 +        txt = '\n'
802 +        txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
803 +        txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
804 +        txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
805 +        txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
806 +        txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
807 +        txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
808 +        txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
809 +        txt += '   else\n'
810 +        txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
811 +        txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
812 +        txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
813 +        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
814 +        txt += '       exit\n'
815 +        txt += '\n'
816 +        txt += '       echo "Remove working directory: $WORKING_DIR"\n'
817 +        txt += '       cd $RUNTIME_AREA\n'
818 +        txt += '       /bin/rm -rf $WORKING_DIR\n'
819 +        txt += '       if [ -d $WORKING_DIR ] ;then\n'
820 +        txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
821 +        txt += '       fi\n'
822 +        txt += '\n'
823 +        txt += '       exit\n'
824 +        txt += '   fi\n'
825 +        txt += '\n'
826 +        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
827 +        txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
828 +
829 +        return txt
830 +
831 +    ### OLI_DANIELE
832 +    def wsSetupCMSLCGEnvironment_(self):
833 +        """
834 +        Returns part of a job script which is prepares
835 +        the execution environment and which is common for all CMS jobs.
836 +        """
837 +        txt  = '   \n'
838 +        txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
839 +        txt += '      echo "JOB_EXIT_STATUS = 0"\n'
840 +        txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
841 +        txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
842 +        txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
843 +        txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
844 +        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
845 +        txt += '       exit\n'
846 +        txt += '   else\n'
847 +        txt += '       echo "Sourcing environment... "\n'
848 +        txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
849 +        txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
850 +        txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
851 +        txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
852 +        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
853 +        txt += '           exit\n'
854 +        txt += '       fi\n'
855 +        txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
856 +        txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
857 +        txt += '       result=$?\n'
858 +        txt += '       if [ $result -ne 0 ]; then\n'
859 +        txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
860 +        txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
861 +        txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
862 +        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
863 +        txt += '           exit\n'
864 +        txt += '       fi\n'
865 +        txt += '   fi\n'
866 +        txt += '   \n'
867 +        txt += '   string=`cat /etc/redhat-release`\n'
868 +        txt += '   echo $string\n'
869 +        txt += '   if [[ $string = *alhalla* ]]; then\n'
870 +        txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
871 +        txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
872 +        txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
873 +        txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
874 +        txt += '   else\n'
875 +        txt += '       echo "SET_CMS_ENV 1 ==> ERROR OS unknown, LCG environment not initialized"\n'
876 +        txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
877 +        txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
878 +        txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
879 +        txt += '       exit\n'
880 +        txt += '   fi\n'
881 +        txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
882 +        txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
883 +        return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines