ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.73 by gutsche, Sun Apr 8 18:39:51 2007 UTC vs.
Revision 1.106 by slacapra, Mon Jul 23 15:56:29 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   import common
6 – import PsetManipulator  
7 – import DataDiscovery
8 – import DataDiscovery_DBS2
9 – import DataLocation
6   import Scram
7  
8 < import os, string, re, shutil, glob
8 > import os, string, glob
9  
10   class Cmssw(JobType):
11      def __init__(self, cfg_params, ncjobs):
12          JobType.__init__(self, 'CMSSW')
13          common.logger.debug(3,'CMSSW::__init__')
14  
19 –        # Marco.
15          self._params = {}
16          self.cfg_params = cfg_params
17  
18          try:
19              self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
20          except KeyError:
21 <            self.MaxTarBallSize = 100.0
21 >            self.MaxTarBallSize = 9.5
22  
23          # number of jobs requested to be created, limit obj splitting
24          self.ncjobs = ncjobs
# Line 36 | Line 31 | class Cmssw(JobType):
31          self.executable = ''
32          self.executable_arch = self.scram.getArch()
33          self.tgz_name = 'default.tgz'
34 +        self.additional_tgz_name = 'additional.tgz'
35          self.scriptName = 'CMSSW.sh'
36          self.pset = ''      #scrip use case Da  
37          self.datasetPath = '' #scrip use case Da
# Line 51 | Line 47 | class Cmssw(JobType):
47  
48          ## get DBS mode
49          try:
50 <            self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
50 >            self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
51          except KeyError:
52 <            self.use_dbs_2 = 0
52 >            self.use_dbs_1 = 0
53              
54          try:
55              tmp =  cfg_params['CMSSW.datasetpath']
# Line 74 | Line 70 | class Cmssw(JobType):
70              self.setParam_('dataset', 'None')
71              self.setParam_('owner', 'None')
72          else:
73 <            datasetpath_split = self.datasetPath.split("/")
74 <            self.setParam_('dataset', datasetpath_split[1])
75 <            self.setParam_('owner', datasetpath_split[-1])
76 <
73 >            try:
74 >                datasetpath_split = self.datasetPath.split("/")
75 >                # standard style
76 >                if self.use_dbs_1 == 1 :
77 >                    self.setParam_('dataset', datasetpath_split[1])
78 >                    self.setParam_('owner', datasetpath_split[-1])
79 >                else:
80 >                    self.setParam_('dataset', datasetpath_split[1])
81 >                    self.setParam_('owner', datasetpath_split[2])
82 >            except:
83 >                self.setParam_('dataset', self.datasetPath)
84 >                self.setParam_('owner', self.datasetPath)
85 >                
86          self.setTaskid_()
87          self.setParam_('taskId', self.cfg_params['taskId'])
88  
# Line 127 | Line 132 | class Cmssw(JobType):
132                      self.output_file.append(tmp)
133                      pass
134              else:
135 <                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
135 >                log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
136                  pass
137              pass
138          except KeyError:
139 <            log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
139 >            log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
140              pass
141  
142          # script_exe file as additional file in inputSandbox
# Line 157 | Line 162 | class Cmssw(JobType):
162                  tmp = string.strip(tmp)
163                  dirname = ''
164                  if not tmp[0]=="/": dirname = "."
165 <                files = glob.glob(os.path.join(dirname, tmp))
165 >                files = []
166 >                if string.find(tmp,"*")>-1:
167 >                    files = glob.glob(os.path.join(dirname, tmp))
168 >                    if len(files)==0:
169 >                        raise CrabException("No additional input file found with this pattern: "+tmp)
170 >                else:
171 >                    files.append(tmp)
172                  for file in files:
173                      if not os.path.exists(file):
174                          raise CrabException("Additional input file not found: "+file)
175                      pass
176 <                    storedFile = common.work_space.shareDir()+file
177 <                    shutil.copyfile(file, storedFile)
178 <                    self.additional_inbox_files.append(string.strip(storedFile))
176 >                    # fname = string.split(file, '/')[-1]
177 >                    # storedFile = common.work_space.pathForTgz()+'share/'+fname
178 >                    # shutil.copyfile(file, storedFile)
179 >                    self.additional_inbox_files.append(string.strip(file))
180                  pass
181              pass
182              common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
# Line 222 | Line 234 | class Cmssw(JobType):
234          except KeyError:
235              self.sourceSeedVtx = None
236              common.logger.debug(5,"No vertex seed given")
237 +
238 +        try:
239 +            self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
240 +        except KeyError:
241 +            self.sourceSeedG4 = None
242 +            common.logger.debug(5,"No g4 sim hits seed given")
243 +
244 +        try:
245 +            self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
246 +        except KeyError:
247 +            self.sourceSeedMix = None
248 +            common.logger.debug(5,"No mix seed given")
249 +
250          try:
251              self.firstRun = int(cfg_params['CMSSW.first_run'])
252          except KeyError:
253              self.firstRun = None
254              common.logger.debug(5,"No first run given")
255          if self.pset != None: #CarlosDaniele
256 <            self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
256 >            ver = string.split(self.version,"_")
257 >            if (int(ver[1])>=1 and int(ver[2])>=5):
258 >                import PsetManipulator150 as pp
259 >            else:
260 >                import PsetManipulator as pp
261 >            PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
262  
263          #DBSDLS-start
264          ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
# Line 250 | Line 280 | class Cmssw(JobType):
280                  self.jobSplittingForScript()
281              else:
282                  self.jobSplittingNoInput()
283 <        else:
283 >        else:
284              self.jobSplittingByBlocks(blockSites)
285  
286          # modify Pset
# Line 258 | Line 288 | class Cmssw(JobType):
288              try:
289                  if (self.datasetPath): # standard job
290                      # allow to processa a fraction of events in a file
291 <                    self.PsetEdit.inputModule("INPUT")
292 <                    self.PsetEdit.maxEvent("INPUTMAXEVENTS")
293 <                    self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
291 >                    PsetEdit.inputModule("INPUT")
292 >                    PsetEdit.maxEvent("INPUTMAXEVENTS")
293 >                    PsetEdit.skipEvent("INPUTSKIPEVENTS")
294                  else:  # pythia like job
295 <                    self.PsetEdit.maxEvent(self.eventsPerJob)
295 >                    PsetEdit.maxEvent(self.eventsPerJob)
296                      if (self.firstRun):
297 <                        self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
297 >                        PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
298                      if (self.sourceSeed) :
299 <                        self.PsetEdit.pythiaSeed("INPUT")
299 >                        PsetEdit.pythiaSeed("INPUT")
300                          if (self.sourceSeedVtx) :
301 <                            self.PsetEdit.pythiaSeedVtx("INPUTVTX")
301 >                            PsetEdit.vtxSeed("INPUTVTX")
302 >                        if (self.sourceSeedG4) :
303 >                            self.PsetEdit.g4Seed("INPUTG4")
304 >                        if (self.sourceSeedMix) :
305 >                            self.PsetEdit.mixSeed("INPUTMIX")
306                  # add FrameworkJobReport to parameter-set
307 <                self.PsetEdit.addCrabFJR(self.fjrFileName)
308 <                self.PsetEdit.psetWriter(self.configFilename())
307 >                PsetEdit.addCrabFJR(self.fjrFileName)
308 >                PsetEdit.psetWriter(self.configFilename())
309              except:
310                  msg='Error while manipuliating ParameterSet: exiting...'
311                  raise CrabException(msg)
312  
313      def DataDiscoveryAndLocation(self, cfg_params):
314  
315 +        import DataDiscovery
316 +        import DataDiscovery_DBS2
317 +        import DataLocation
318          common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
319  
320          datasetPath=self.datasetPath
321  
322          ## Contact the DBS
323 <        common.logger.message("Contacting DBS...")
323 >        common.logger.message("Contacting Data Discovery Services ...")
324          try:
325  
326 <            if self.use_dbs_2 == 1 :
290 <                self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
291 <            else :
326 >            if self.use_dbs_1 == 1 :
327                  self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
328 +            else :
329 +                self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
330              self.pubdata.fetchDBSInfo()
331  
332          except DataDiscovery.NotExistingDatasetError, ex :
# Line 311 | Line 348 | class Cmssw(JobType):
348              msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
349              raise CrabException(msg)
350  
314 –        ## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
315 –        common.logger.message("Required data are :"+self.datasetPath)
316 –
351          self.filesbyblock=self.pubdata.getFiles()
352          self.eventsbyblock=self.pubdata.getEventsPerBlock()
353          self.eventsbyfile=self.pubdata.getEventsPerFile()
354  
355          ## get max number of events
356          self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
323 –        common.logger.message("The number of available events is %s\n"%self.maxEvents)
357  
325 –        common.logger.message("Contacting DLS...")
358          ## Contact the DLS and build a list of sites hosting the fileblocks
359          try:
360              dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
# Line 340 | Line 372 | class Cmssw(JobType):
372                  allSites.append(oneSite)
373          allSites = self.uniquelist(allSites)
374  
375 <        common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
376 <        common.logger.debug(6, "List of Sites: "+str(allSites))
375 >        # screen output
376 >        common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
377 >
378          return sites
379      
380      def jobSplittingByBlocks(self, blockSites):
# Line 403 | Line 436 | class Cmssw(JobType):
436          jobCount = 0
437          list_of_lists = []
438  
439 +        # list tracking which jobs are in which jobs belong to which block
440 +        jobsOfBlock = {}
441 +
442          # ---- Iterate over the blocks in the dataset until ---- #
443          # ---- we've met the requested total # of events    ---- #
444          while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
445              block = blocks[blockCount]
446              blockCount += 1
447 +            if block not in jobsOfBlock.keys() :
448 +                jobsOfBlock[block] = []
449              
450              if self.eventsbyblock.has_key(block) :
451                  numEventsInBlock = self.eventsbyblock[block]
# Line 457 | Line 495 | class Cmssw(JobType):
495                              common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
496                              self.jobDestination.append(blockSites[block])
497                              common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
498 +                            # fill jobs of block dictionary
499 +                            jobsOfBlock[block].append(jobCount+1)
500                              # reset counter
501                              jobCount = jobCount + 1
502                              totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
# Line 480 | Line 520 | class Cmssw(JobType):
520                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
521                          self.jobDestination.append(blockSites[block])
522                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
523 +                        jobsOfBlock[block].append(jobCount+1)
524                          # reset counter
525                          jobCount = jobCount + 1
526                          totalEventCount = totalEventCount + eventsPerJobRequested
# Line 500 | Line 541 | class Cmssw(JobType):
541                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
542                          self.jobDestination.append(blockSites[block])
543                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
544 +                        jobsOfBlock[block].append(jobCount+1)
545                          # increase counter
546                          jobCount = jobCount + 1
547                          totalEventCount = totalEventCount + eventsPerJobRequested
# Line 517 | Line 559 | class Cmssw(JobType):
559          self.ncjobs = self.total_number_of_jobs = jobCount
560          if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
561              common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
562 <        common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
562 >        common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
563          
564 +        # screen output
565 +        screenOutput = "List of jobs and available destination sites:\n\n"
566 +
567 +        blockCounter = 0
568 +        for block in blocks:
569 +            if block in jobsOfBlock.keys() :
570 +                blockCounter += 1
571 +                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
572 +
573 +        common.logger.message(screenOutput)
574 +
575          self.list_of_args = list_of_lists
576          return
577  
# Line 563 | Line 616 | class Cmssw(JobType):
616              ## Since there is no input, any site is good
617             # self.jobDestination.append(["Any"])
618              self.jobDestination.append([""]) #must be empty to write correctly the xml
619 <            args=''
619 >            args=[]
620              if (self.firstRun):
621                      ## pythia first run
622                  #self.list_of_args.append([(str(self.firstRun)+str(i))])
623 <                args=args+(str(self.firstRun)+str(i))
623 >                args.append(str(self.firstRun)+str(i))
624              else:
625                  ## no first run
626                  #self.list_of_args.append([str(i)])
627 <                args=args+str(i)
627 >                args.append(str(i))
628              if (self.sourceSeed):
629 +                args.append(str(self.sourceSeed)+str(i))
630                  if (self.sourceSeedVtx):
631 <                    ## pythia + vtx random seed
632 <                    #self.list_of_args.append([
633 <                    #                          str(self.sourceSeed)+str(i),
634 <                    #                          str(self.sourceSeedVtx)+str(i)
635 <                    #                          ])
636 <                    args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
637 <                else:
638 <                    ## only pythia random seed
639 <                    #self.list_of_args.append([(str(self.sourceSeed)+str(i))])
640 <                    args=args +str(',')+str(self.sourceSeed)+str(i)
641 <            else:
642 <                ## no random seed
589 <                if str(args)=='': args=args+(str(self.firstRun)+str(i))
590 <            arguments=args.split(',')
591 <            if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
592 <            elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
593 <            else :self.list_of_args.append([str(arguments[0])])
631 >                    ## + vtx random seed
632 >                    args.append(str(self.sourceSeedVtx)+str(i))
633 >                if (self.sourceSeedG4):
634 >                    ## + G4 random seed
635 >                    args.append(str(self.sourceSeedG4)+str(i))
636 >                if (self.sourceSeedMix):    
637 >                    ## + Mix random seed
638 >                    args.append(str(self.sourceSeedMix)+str(i))
639 >                pass
640 >            pass
641 >            self.list_of_args.append(args)
642 >        pass
643              
644 <     #   print self.list_of_args
644 >        # print self.list_of_args
645  
646          return
647  
# Line 687 | Line 736 | class Cmssw(JobType):
736          try: # create tar ball
737              tar = tarfile.open(self.tgzNameWithPath, "w:gz")
738              ## First find the executable
739 <            if (executable != ''):
739 >            if (self.executable != ''):
740                  exeWithPath = self.scram.findFile_(executable)
741                  if ( not exeWithPath ):
742                      raise CrabException('User executable '+executable+' not found')
# Line 697 | Line 746 | class Cmssw(JobType):
746                      # the exe is private, so we must ship
747                      common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
748                      path = swArea+'/'
749 <                    exe = string.replace(exeWithPath, path,'')
750 <                    tar.add(path+exe,executable)
749 >                    # distinguish case when script is in user project area or given by full path somewhere else
750 >                    if exeWithPath.find(path) >= 0 :
751 >                        exe = string.replace(exeWithPath, path,'')
752 >                        tar.add(path+exe,os.path.basename(executable))
753 >                    else :
754 >                        tar.add(exeWithPath,os.path.basename(executable))
755                      pass
756                  else:
757                      # the exe is from release, we'll find it on WN
# Line 729 | Line 782 | class Cmssw(JobType):
782              pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
783              if os.path.isdir(pa):
784                  tar.add(pa,paDir)
785 +
786 +            ### FEDE FOR DBS PUBLICATION
787 +            ## Add PRODCOMMON dir to tar
788 +            prodcommonDir = 'ProdCommon'
789 +            prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
790 +            if os.path.isdir(prodcommonPath):
791 +                tar.add(prodcommonPath,prodcommonDir)
792 +            #############################    
793          
794              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
795              tar.close()
# Line 754 | Line 815 | class Cmssw(JobType):
815          
816          return
817          
818 +    def additionalInputFileTgz(self):
819 +        """
820 +        Put all additional files into a tar ball and return its name
821 +        """
822 +        import tarfile
823 +        tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
824 +        tar = tarfile.open(tarName, "w:gz")
825 +        for file in self.additional_inbox_files:
826 +            tar.add(file,string.split(file,'/')[-1])
827 +        common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
828 +        tar.close()
829 +        return tarName
830 +
831      def wsSetupEnvironment(self, nj):
832          """
833          Returns part of a job script which prepares
# Line 764 | Line 838 | class Cmssw(JobType):
838    
839          ## OLI_Daniele at this level  middleware already known
840  
841 +        txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
842 +        txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
843 +        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
844 +        txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
845          txt += 'if [ $middleware == LCG ]; then \n'
846          txt += self.wsSetupCMSLCGEnvironment_()
847          txt += 'elif [ $middleware == OSG ]; then\n'
# Line 771 | Line 849 | class Cmssw(JobType):
849          txt += '    echo "Created working directory: $WORKING_DIR"\n'
850          txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
851          txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
852 <        txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
853 <        txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
854 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
852 >        txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
853 >        txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
854 >        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
855          txt += '        rm -f $RUNTIME_AREA/$repo \n'
856          txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
857          txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 805 | Line 883 | class Cmssw(JobType):
883          txt += '        cd $RUNTIME_AREA\n'
884          txt += '        /bin/rm -rf $WORKING_DIR\n'
885          txt += '        if [ -d $WORKING_DIR ] ;then\n'
886 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
888 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
886 >        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887 >        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
888 >        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
890          txt += '            rm -f $RUNTIME_AREA/$repo \n'
891          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
892          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 817 | Line 895 | class Cmssw(JobType):
895          txt += '   exit 1 \n'
896          txt += 'fi \n'
897          txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
820 –        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
898          txt += 'cd '+self.version+'\n'
899 +        ########## FEDE FOR DBS2 ######################
900 +        txt += 'SOFTWARE_DIR=`pwd`\n'
901 +        txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
902 +        ###############################################
903          ### needed grep for bug in scramv1 ###
904          txt += scram+' runtime -sh\n'
905          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
# Line 844 | Line 925 | class Cmssw(JobType):
925          txt += '        cd $RUNTIME_AREA\n'
926          txt += '        /bin/rm -rf $WORKING_DIR\n'
927          txt += '        if [ -d $WORKING_DIR ] ;then\n'
928 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
930 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
928 >        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929 >        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
930 >        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
932          txt += '            rm -f $RUNTIME_AREA/$repo \n'
933          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
934          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 859 | Line 940 | class Cmssw(JobType):
940  
941          # Prepare job-specific part
942          job = common.job_list[nj]
943 +        ### FEDE FOR DBS OUTPUT PUBLICATION
944 +        if (self.datasetPath):
945 +            txt += '\n'
946 +            txt += 'DatasetPath='+self.datasetPath+'\n'
947 +
948 +            datasetpath_split = self.datasetPath.split("/")
949 +            
950 +            txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
951 +            txt += 'DataTier='+datasetpath_split[2]+'\n'
952 +            #txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
953 +            txt += 'ApplicationFamily=cmsRun\n'
954 +
955 +        else:
956 +            txt += 'DatasetPath=MCDataTier\n'
957 +            txt += 'PrimaryDataset=null\n'
958 +            txt += 'DataTier=null\n'
959 +            #txt += 'ProcessedDataset=null\n'
960 +            txt += 'ApplicationFamily=MCDataTier\n'
961          if self.pset != None: #CarlosDaniele
962              pset = os.path.basename(job.configFilename())
963              txt += '\n'
964 +            txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
965              if (self.datasetPath): # standard job
966                  #txt += 'InputFiles=$2\n'
967                  txt += 'InputFiles=${args[1]}\n'
968                  txt += 'MaxEvents=${args[2]}\n'
969                  txt += 'SkipEvents=${args[3]}\n'
970                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
971 <                txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
971 >                txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
972                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
973 <                txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
973 >                txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
974                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
975 <                txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
975 >                txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
976              else:  # pythia like job
977 <                if (self.sourceSeed):
978 <                    txt += 'FirstRun=${args[1]}\n'
977 >                seedIndex=1
978 >                if (self.firstRun):
979 >                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
980                      txt += 'echo "FirstRun: <$FirstRun>"\n'
981 <                    txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
982 <                else:
983 <                    txt += '# Copy untouched pset\n'
883 <                    txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
981 >                    txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
982 >                    seedIndex=seedIndex+1
983 >
984                  if (self.sourceSeed):
985 < #                    txt += 'Seed=$2\n'
986 <                    txt += 'Seed=${args[2]}\n'
987 <                    txt += 'echo "Seed: <$Seed>"\n'
988 <                    txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
985 >                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
986 >                    txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
987 >                    seedIndex=seedIndex+1
988 >                    ## the following seeds are not always present
989                      if (self.sourceSeedVtx):
990 < #                        txt += 'VtxSeed=$3\n'
891 <                        txt += 'VtxSeed=${args[3]}\n'
990 >                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
991                          txt += 'echo "VtxSeed: <$VtxSeed>"\n'
992 <                        txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
993 <                    else:
994 <                        txt += 'mv tmp_2.cfg pset.cfg\n'
995 <                else:
996 <                    txt += 'mv tmp_1.cfg pset.cfg\n'
997 <                   # txt += '# Copy untouched pset\n'
998 <                   # txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
999 <
992 >                        txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
993 >                        seedIndex += 1
994 >                    if (self.sourceSeedG4):
995 >                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
996 >                        txt += 'echo "G4Seed: <$G4Seed>"\n'
997 >                        txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
998 >                        seedIndex += 1
999 >                    if (self.sourceSeedMix):
1000 >                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1001 >                        txt += 'echo "MixSeed: <$mixSeed>"\n'
1002 >                        txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1003 >                        seedIndex += 1
1004 >                    pass
1005 >                pass
1006 >            txt += 'mv -f '+pset+' pset.cfg\n'
1007  
1008          if len(self.additional_inbox_files) > 0:
1009 <            for file in self.additional_inbox_files:
1010 <                relFile = file.split("/")[-1]
1011 <                txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
906 <                txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
907 <                txt += '   chmod +x '+relFile+'\n'
908 <                txt += 'fi\n'
1009 >            txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1010 >            txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1011 >            txt += 'fi\n'
1012              pass
1013  
1014          if self.pset != None: #CarlosDaniele
# Line 916 | Line 1019 | class Cmssw(JobType):
1019              txt += 'cat pset.cfg\n'
1020              txt += 'echo "****** end pset.cfg ********"\n'
1021              txt += '\n'
1022 +            ### FEDE FOR DBS OUTPUT PUBLICATION
1023 +            txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1024 +            txt += 'echo "PSETHASH = $PSETHASH" \n'
1025 +            ##############
1026 +            txt += '\n'
1027              # txt += 'echo "***** cat pset1.cfg *********"\n'
1028              # txt += 'cat pset1.cfg\n'
1029              # txt += 'echo "****** end pset1.cfg ********"\n'
# Line 957 | Line 1065 | class Cmssw(JobType):
1065              txt += '   echo "Successful untar" \n'
1066              txt += 'fi \n'
1067              txt += '\n'
1068 <            txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1068 >            txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1069              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1070 <            txt += '   export PYTHONPATH=ProdAgentApi\n'
1070 >            #### FEDE FOR DBS OUTPUT PUBLICATION
1071 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1072 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1073 >            #txt += '   export PYTHONPATH=ProdAgentApi\n'
1074              txt += 'else\n'
1075 <            txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1075 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1076 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1077 >            #txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1078 >            txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1079 >            ###################  
1080              txt += 'fi\n'
1081              txt += '\n'
1082  
# Line 1003 | Line 1118 | class Cmssw(JobType):
1118          if not self.pset is None:
1119              inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1120          ## additional input files
1121 <        for file in self.additional_inbox_files:
1122 <            inp_box.append(file)
1121 >        tgz = self.additionalInputFileTgz()
1122 >        inp_box.append(tgz)
1123          return inp_box
1124  
1125      def outputSandbox(self, nj):
# Line 1038 | Line 1153 | class Cmssw(JobType):
1153              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1154              txt += '\n'
1155              txt += '# check output file\n'
1156 <            txt += 'ls '+fileWithSuffix+'\n'
1157 <            txt += 'ls_result=$?\n'
1158 <            txt += 'if [ $ls_result -ne 0 ] ; then\n'
1159 <            txt += '   echo "ERROR: Problem with output file"\n'
1156 >            # txt += 'ls '+fileWithSuffix+'\n'
1157 >            # txt += 'ls_result=$?\n'
1158 >            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1159 >            txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1160 >            txt += 'else\n'
1161 >            txt += '   exit_status=60302\n'
1162 >            txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1163              if common.scheduler.boss_scheduler_name == 'condor_g':
1164                  txt += '    if [ $middleware == OSG ]; then \n'
1165                  txt += '        echo "prepare dummy output file"\n'
1166                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1167                  txt += '    fi \n'
1050 –            txt += 'else\n'
1051 –            txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1168              txt += 'fi\n'
1169 +        file_list = []
1170 +        for fileWithSuffix in (self.output_file):
1171 +             file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1172 +        txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1173        
1174          txt += 'cd $RUNTIME_AREA\n'
1175 <        txt += 'cd $RUNTIME_AREA\n'
1175 >        #### FEDE this is the cleanEnv function
1176          ### OLI_DANIELE
1177 <        txt += 'if [ $middleware == OSG ]; then\n'  
1178 <        txt += '    cd $RUNTIME_AREA\n'
1179 <        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180 <        txt += '    /bin/rm -rf $WORKING_DIR\n'
1181 <        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182 <        txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183 <        txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1184 <        txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1186 <        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187 <        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188 <        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189 <        txt += '    fi\n'
1190 <        txt += 'fi\n'
1191 <        txt += '\n'
1177 >        #txt += 'if [ $middleware == OSG ]; then\n'  
1178 >        #txt += '    cd $RUNTIME_AREA\n'
1179 >        #txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180 >        #txt += '    /bin/rm -rf $WORKING_DIR\n'
1181 >        #txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182 >        #txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183 >        #txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1184 >        #txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1186 >        #txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187 >        #txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188 >        #txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189 >        #txt += '    fi\n'
1190 >        #txt += 'fi\n'
1191 >        #txt += '\n'
1192  
1073 –        file_list = ''
1074 –        ## Add to filelist only files to be possibly copied to SE
1075 –        for fileWithSuffix in self.output_file:
1076 –            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1077 –            file_list=file_list+output_file_num+' '
1078 –        file_list=file_list[:-1]
1079 –        txt += 'file_list="'+file_list+'"\n'
1193  
1194          return txt
1195  
# Line 1088 | Line 1201 | class Cmssw(JobType):
1201          # take away last extension
1202          name = p[0]
1203          for x in p[1:-1]:
1204 <           name=name+"."+x
1204 >            name=name+"."+x
1205          # add "_txt"
1206          if len(p)>1:
1207 <          ext = p[len(p)-1]
1208 <          result = name + '_' + txt + "." + ext
1207 >            ext = p[len(p)-1]
1208 >            result = name + '_' + txt + "." + ext
1209          else:
1210 <          result = name + '_' + txt
1210 >            result = name + '_' + txt
1211          
1212          return result
1213  
# Line 1107 | Line 1220 | class Cmssw(JobType):
1220              req='Member("VO-cms-' + \
1221                   self.version + \
1222                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1223 +        ## SL add requirement for OS version only if SL4
1224 +        reSL4 = re.compile( r'slc4' )
1225 +        if self.executable_arch and reSL4.search(self.executable_arch):
1226 +            req='Member("VO-cms-' + \
1227 +                 self.executable_arch + \
1228 +                 '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1229  
1230          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1231  
# Line 1126 | Line 1245 | class Cmssw(JobType):
1245          txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1246          txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1247          txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1248 +        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1249          txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1250          txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1251          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1252 +        txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1253          txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1254          txt += '   else\n'
1255          txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
# Line 1144 | Line 1265 | class Cmssw(JobType):
1265          txt += '       cd $RUNTIME_AREA\n'
1266          txt += '       /bin/rm -rf $WORKING_DIR\n'
1267          txt += '       if [ -d $WORKING_DIR ] ;then\n'
1268 <        txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1269 <        txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1270 <        txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1272 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
1273 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1268 >        txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1269 >        txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1270 >        txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271 >        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1272 >        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1273 >        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274 >        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1275          txt += '       fi\n'
1276          txt += '\n'
1277          txt += '       exit 1\n'
# Line 1209 | Line 1330 | class Cmssw(JobType):
1330          txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1331          return txt
1332  
1333 +    ### FEDE FOR DBS OUTPUT PUBLICATION
1334 +    def modifyReport(self, nj):
1335 +        """
1336 +        insert the part of the script that modifies the FrameworkJob Report
1337 +        """
1338 +
1339 +        txt = ''
1340 +        txt += 'echo "Modify Job Report" \n'
1341 +        #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1342 +        ################ FEDE FOR DBS2 #############################################
1343 +        txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1344 +        #############################################################################
1345 +        try:
1346 +            publish_data = int(self.cfg_params['USER.publish_data'])          
1347 +        except KeyError:
1348 +            publish_data = 0
1349 +
1350 +        txt += 'if [ -z "$SE" ]; then\n'
1351 +        txt += '    SE="" \n'
1352 +        txt += 'fi \n'
1353 +        txt += 'if [ -z "$SE_PATH" ]; then\n'
1354 +        txt += '    SE_PATH="" \n'
1355 +        txt += 'fi \n'
1356 +        txt += 'echo "SE = $SE"\n'
1357 +        txt += 'echo "SE_PATH = $SE_PATH"\n'
1358 +
1359 +        if (publish_data == 1):  
1360 +            #processedDataset = self.cfg_params['USER.processed_datasetname']
1361 +            processedDataset = self.cfg_params['USER.publish_data_name']
1362 +            txt += 'ProcessedDataset='+processedDataset+'\n'
1363 +            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1364 +            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1365 +            #### FEDE: added slash in LFN ##############
1366 +            txt += '    FOR_LFN=/copy_problems/ \n'
1367 +            txt += 'else \n'
1368 +            txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1369 +            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1370 +            txt += '    FOR_LFN=/store$tmp \n'
1371 +            txt += 'fi \n'
1372 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1373 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1374 +            txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1375 +            #txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1376 +            txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1377 +            txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1378 +            #txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1379 +      
1380 +            txt += 'modifyReport_result=$?\n'
1381 +            txt += 'echo modifyReport_result = $modifyReport_result\n'
1382 +            txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1383 +            txt += '    exit_status=1\n'
1384 +            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1385 +            txt += 'else\n'
1386 +            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1387 +            txt += 'fi\n'
1388 +        else:
1389 +            txt += 'ProcessedDataset=no_data_to_publish \n'
1390 +            #### FEDE: added slash in LFN ##############
1391 +            txt += 'FOR_LFN=/local/ \n'
1392 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1393 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1394 +        return txt
1395 +
1396 +    def cleanEnv(self):
1397 +        ### OLI_DANIELE
1398 +        txt = ''
1399 +        txt += 'if [ $middleware == OSG ]; then\n'  
1400 +        txt += '    cd $RUNTIME_AREA\n'
1401 +        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1402 +        txt += '    /bin/rm -rf $WORKING_DIR\n'
1403 +        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1404 +        txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1405 +        txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1406 +        txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1407 +        txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1408 +        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1409 +        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1410 +        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1411 +        txt += '    fi\n'
1412 +        txt += 'fi\n'
1413 +        txt += '\n'
1414 +        return txt
1415 +
1416      def setParam_(self, param, value):
1417          self._params[param] = value
1418  
# Line 1221 | Line 1425 | class Cmssw(JobType):
1425      def getTaskid(self):
1426          return self._taskId
1427  
1224 – #######################################################################
1428      def uniquelist(self, old):
1429          """
1430          remove duplicates from a list

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines