ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.157 by spiga, Sun Feb 17 20:13:00 2008 UTC vs.
Revision 1.169 by spiga, Tue Apr 1 14:53:36 2008 UTC

# Line 47 | Line 47 | class Cmssw(JobType):
47          # Try to block creation in case of arch/version mismatch
48          #
49  
50 <        a = string.split(self.version, "_")
51 <
52 <        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53 <            msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
54 <            common.logger.message(msg)
55 <        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56 <            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57 <            raise CrabException(msg)
58 <
59 <        common.taskDB.setDict('codeVersion',self.version)
50 > #        a = string.split(self.version, "_")
51 > #
52 > #        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53 > #            msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
54 > #            common.logger.message(msg)
55 > #        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56 > #            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57 > #            raise CrabException(msg)
58 > #
59 >
60          self.setParam_('application', self.version)
61  
62          ### collect Data cards
# Line 90 | Line 90 | class Cmssw(JobType):
90                  self.setParam_('dataset', self.datasetPath)
91                  self.setParam_('owner', self.datasetPath)
92  
93 <        self.setParam_('taskId', common.taskDB.dict('taskId'))
93 >        self.setParam_('taskId', common._db.queryTask('name')) ## new BL--DS
94  
95          self.dataTiers = []
96  
# Line 202 | Line 202 | class Cmssw(JobType):
202                   msg = 'Must specify  number_of_jobs.'
203                   raise CrabException(msg)
204  
205 <        ## source seed for pythia
205 >        ## New method of dealing with seeds
206 >        self.incrementSeeds = []
207 >        self.preserveSeeds = []
208 >        if cfg_params.has_key('CMSSW.preserve_seeds'):
209 >            tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
210 >            for tmp in tmpList:
211 >                tmp.strip()
212 >                self.preserveSeeds.append(tmp)
213 >        if cfg_params.has_key('CMSSW.increment_seeds'):
214 >            tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
215 >            for tmp in tmpList:
216 >                tmp.strip()
217 >                self.incrementSeeds.append(tmp)
218 >
219 >        ## Old method of dealing with seeds
220 >        ## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
221 >        ## remove
222          self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
223 +        if self.sourceSeed:
224 +          print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
225 +          self.incrementSeeds.append('sourceSeed')
226  
227          self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
228 +        if self.sourceSeedVtx:
229 +          print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
230 +          self.incrementSeeds.append('VtxSmeared')
231  
232          self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
233 +        if self.sourceSeedG4:
234 +          print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
235 +          self.incrementSeeds.append('g4SimHits')
236  
237          self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
238 +        if self.sourceSeedMix:
239 +          print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
240 +          self.incrementSeeds.append('mix')
241  
242          self.firstRun = cfg_params.get('CMSSW.first_run',None)
243  
# Line 248 | Line 276 | class Cmssw(JobType):
276          # modify Pset
277          if self.pset != None: #CarlosDaniele
278              try:
279 <                if (self.datasetPath): # standard job
280 <                    # allow to processa a fraction of events in a file
253 <                    PsetEdit.inputModule("INPUTFILE")
254 <                    PsetEdit.maxEvent(0)
255 <                    PsetEdit.skipEvent(0)
256 <                else:  # pythia like job
257 <                    PsetEdit.maxEvent(self.eventsPerJob)
258 <                    if (self.firstRun):
259 <                        PsetEdit.pythiaFirstRun(0)  #First Run
260 <                    if (self.sourceSeed) :
261 <                        PsetEdit.pythiaSeed(0)
262 <                        if (self.sourceSeedVtx) :
263 <                            PsetEdit.vtxSeed(0)
264 <                        if (self.sourceSeedG4) :
265 <                            PsetEdit.g4Seed(0)
266 <                        if (self.sourceSeedMix) :
267 <                            PsetEdit.mixSeed(0)
268 <                # add FrameworkJobReport to parameter-set
279 >                # Add FrameworkJobReport to parameter-set, set max events.
280 >                # Reset later for data jobs by writeCFG which does all modifications
281                  PsetEdit.addCrabFJR(self.fjrFileName)
282 +                PsetEdit.maxEvent(self.eventsPerJob)
283                  PsetEdit.psetWriter(self.configFilename())
284              except:
285                  msg='Error while manipuliating ParameterSet: exiting...'
# Line 325 | Line 338 | class Cmssw(JobType):
338  
339          return sites
340  
341 <    def setArgsList(self, argsList):
342 <        self.argsList = argsList
341 >  # to Be Removed  DS -- BL
342 >  #  def setArgsList(self, argsList):
343 >  #      self.argsList = argsList
344  
345      def jobSplittingByBlocks(self, blockSites):
346          """
# Line 499 | Line 513 | class Cmssw(JobType):
513                          jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
514                          # remove all but the last file
515                          filesEventCount = self.eventsbyfile[file]
516 <                        parString = ""
503 <                        parString += '\\\"' + file + '\\\"\,'
516 >                        parString = '\\\"' + file + '\\\"\,'
517                      pass # END if
518                  pass # END while (iterate over files in the block)
519          pass # END while (iterate over blocks in the dataset)
# Line 602 | Line 615 | class Cmssw(JobType):
615              if (self.firstRun):
616                  ## pythia first run
617                  args.append(str(self.firstRun)+str(i))
605 –            if (self.sourceSeed):
606 –                args.append(str(self.sourceSeed)+str(i))
607 –                if (self.sourceSeedVtx):
608 –                    ## + vtx random seed
609 –                    args.append(str(self.sourceSeedVtx)+str(i))
610 –                if (self.sourceSeedG4):
611 –                    ## + G4 random seed
612 –                    args.append(str(self.sourceSeedG4)+str(i))
613 –                if (self.sourceSeedMix):
614 –                    ## + Mix random seed
615 –                    args.append(str(self.sourceSeedMix)+str(i))
616 –                pass
617 –            pass
618              self.list_of_args.append(args)
619 –        pass
619  
620          return
621  
# Line 646 | Line 645 | class Cmssw(JobType):
645  
646      def split(self, jobParams):
647  
649 –        common.jobDB.load()
648          #### Fabio
649          njobs = self.total_number_of_jobs
650          arglist = self.list_of_args
# Line 654 | Line 652 | class Cmssw(JobType):
652          for i in range(njobs):
653              jobParams.append("")
654  
655 +        listID=[]
656 +        listField=[]
657          for job in range(njobs):
658              jobParams[job] = arglist[job]
659 <            # print str(arglist[job])
660 <            # print jobParams[job]
661 <            common.jobDB.setArguments(job, jobParams[job])
662 <            common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
663 <            common.jobDB.setDestination(job, self.jobDestination[job])
659 >            listID.append(job+1)
660 >            job_ToSave ={}
661 >            concString = ' '
662 >            argu=''
663 >            if len(jobParams[job]):
664 >                argu +=   concString.join(jobParams[job] )
665 >            job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS
666 >            job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS
667 >            #common._db.updateJob_(job,job_ToSave)## new BL--DS
668 >            listField.append(job_ToSave)
669 >            msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
670 >            +"                     Destination: "+str(self.jobDestination[job])
671 >            common.logger.debug(5,msg)
672 >            #common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
673 >        common._db.updateJob_(listID,listField)## new BL--DS
674 >        ## Pay Attention Here....DS--BL
675 >        self.argsList = (len(jobParams[1])+1)
676  
665 –        common.jobDB.save()
677          return
678 <
679 <    def getJobTypeArguments(self, nj, sched):
680 <        result = ''
681 <        for i in common.jobDB.arguments(nj):
682 <            result=result+str(i)+" "
683 <        return result
678 > #
679 > #    def getJobTypeArguments(self, nj, sched):
680 > #        result = ''
681 > #        jobs=[]
682 > #        jobs.append(nj)
683 > #        for i in common._db.queryJob('arguments',jobs):##  BL--DS
684 > #            result=result+str(i)+" "
685 > #        return result
686  
687      def numberOfJobs(self):
688          # Fabio
# Line 752 | Line 765 | class Cmssw(JobType):
765                      common.logger.debug(5,"data "+root+"/data"+" to be tarred")
766                      tar.add(root+"/data",root[swAreaLen:]+"/data")
767  
755 –            ### Removed ProdAgent Api dependencies ###
756 –            ### Add ProdAgent dir to tar
757 –            #paDir = 'ProdAgentApi'
758 –            #pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
759 –            #if os.path.isdir(pa):
760 –            #    tar.add(pa,paDir)
768  
769              ## Add ProdCommon dir to tar
770              prodcommonDir = 'ProdCommon'
# Line 780 | Line 787 | class Cmssw(JobType):
787          try:
788              tar = tarfile.open(self.MLtgzfile, "w:gz")
789              path=os.environ['CRABDIR'] + '/python/'
790 <            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
790 >            #for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py']:
791 >            ### FEDE ####
792 >            for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']:
793 >            ###############
794                  tar.add(path+file,file)
795              common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
796              tar.close()
# Line 802 | Line 812 | class Cmssw(JobType):
812          tar.close()
813          return tarName
814  
815 <    def wsSetupEnvironment(self, nj):
815 >    def wsSetupEnvironment(self, nj=0):
816          """
817          Returns part of a job script which prepares
818          the execution environment for the job 'nj'.
819          """
820          # Prepare JobType-independent part
821 <        txt = ''
821 >        txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
822          txt += 'echo ">>> setup environment"\n'
823          txt += 'if [ $middleware == LCG ]; then \n'
824          txt += self.wsSetupCMSLCGEnvironment_()
825          txt += 'elif [ $middleware == OSG ]; then\n'
826          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
827          txt += '    if [ ! $? == 0 ] ;then\n'
828 <        txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
829 <        txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
830 <        txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
831 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
832 <        txt += '        exit 1\n'
828 >        #txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
829 >        #txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
830 >        #txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
831 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
832 >        #txt += '        exit 1\n'
833 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
834 >        txt += '        job_exit_code=10016\n'
835 >        txt += '        func_exit\n'
836          txt += '    fi\n'
837          txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
838          txt += '\n'
# Line 839 | Line 852 | class Cmssw(JobType):
852          txt += scram+' project CMSSW '+self.version+'\n'
853          txt += 'status=$?\n'
854          txt += 'if [ $status != 0 ] ; then\n'
855 <        txt += '    echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856 <        txt += '    echo "JOB_EXIT_STATUS = 10034"\n'
857 <        txt += '    echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
858 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
859 <        txt += '    if [ $middleware == OSG ]; then \n'
860 <        txt += '        cd $RUNTIME_AREA\n'
861 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
862 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
863 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
864 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
865 <        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
866 <        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
867 <        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
868 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
869 <        txt += '        fi\n'
870 <        txt += '    fi \n'
871 <        txt += '    exit 1 \n'
855 >        #txt += '    echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856 >        #txt += '    echo "JOB_EXIT_STATUS = 10034"\n'
857 >        #txt += '    echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
858 >        #txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
859 >        txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
860 >        txt += '    job_exit_code=10034\n'
861 >        #txt += '    if [ $middleware == OSG ]; then \n'
862 >        #txt += '        cd $RUNTIME_AREA\n'
863 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
864 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
865 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
866 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
867 >        #txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
868 >        #txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
869 >        #txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
870 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
871 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
872 >        #txt += '            job_exit_code=10017\n'
873 >        #txt += '        fi\n'
874 >        #txt += '    fi \n'
875 >        #txt += '    exit 1 \n'
876 >        txt += '    func_exit\n'
877          txt += 'fi \n'
878          txt += 'cd '+self.version+'\n'
879          ########## FEDE FOR DBS2 ######################
# Line 868 | Line 886 | class Cmssw(JobType):
886          txt += "\n"
887          txt += "## number of arguments (first argument always jobnumber)\n"
888          txt += "\n"
889 <        txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
889 >       # txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
890 >        txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
891          txt += "then\n"
892 <        txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893 <        txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
894 <        txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
895 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
896 <        txt += '    if [ $middleware == OSG ]; then \n'
897 <        txt += '        cd $RUNTIME_AREA\n'
898 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
899 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
900 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
901 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
902 <        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
903 <        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
904 <        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
905 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
906 <        txt += '        fi\n'
907 <        txt += '    fi \n'
908 <        txt += "    exit 1\n"
892 >        #txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893 >        #txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
894 >        #txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
895 >        #txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
896 >        txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
897 >        txt += '    job_exit_code=50113\n'
898 >        #txt += '    if [ $middleware == OSG ]; then \n'
899 >        #txt += '        cd $RUNTIME_AREA\n'
900 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
901 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
902 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
903 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
904 >        #txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
905 >        #txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
906 >        #txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
907 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
908 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
909 >        #txt += '            job_exit_code=10017\n'
910 >        #txt += '        fi\n'
911 >        #txt += '    fi\n'
912 >        #txt += "    exit 1\n"
913 >        txt += "    func_exit\n"
914          txt += "fi\n"
915          txt += "\n"
916  
# Line 908 | Line 932 | class Cmssw(JobType):
932              txt += 'PrimaryDataset=null\n'
933              txt += 'DataTier=null\n'
934              txt += 'ApplicationFamily=MCDataTier\n'
935 <        if self.pset != None: #CarlosDaniele
935 >        if self.pset != None:
936              pset = os.path.basename(job.configFilename())
937              txt += '\n'
938              txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
939              if (self.datasetPath): # standard job
940 <                txt += 'InputFiles=${args[1]}\n'
941 <                txt += 'MaxEvents=${args[2]}\n'
942 <                txt += 'SkipEvents=${args[3]}\n'
940 >                txt += 'InputFiles=${args[1]}; export InputFiles\n'
941 >                txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
942 >                txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
943                  txt += 'echo "Inputfiles:<$InputFiles>"\n'
920 –                txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
944                  txt += 'echo "MaxEvents:<$MaxEvents>"\n'
922 –                txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
945                  txt += 'echo "SkipEvents:<$SkipEvents>"\n'
924 –                txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
946              else:  # pythia like job
947 <                seedIndex=1
947 >                txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
948 >                txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
949 >                txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
950 >                txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
951                  if (self.firstRun):
952 <                    txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
952 >                    txt += 'FirstRun=${args[1]}; export FirstRun\n'
953                      txt += 'echo "FirstRun: <$FirstRun>"\n'
930 –                    txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
931 –                    seedIndex=seedIndex+1
954  
933 –                if (self.sourceSeed):
934 –                    txt += 'Seed=${args['+str(seedIndex)+']}\n'
935 –                    txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
936 –                    seedIndex=seedIndex+1
937 –                    ## the following seeds are not always present
938 –                    if (self.sourceSeedVtx):
939 –                        txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
940 –                        txt += 'echo "VtxSeed: <$VtxSeed>"\n'
941 –                        txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
942 –                        seedIndex += 1
943 –                    if (self.sourceSeedG4):
944 –                        txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
945 –                        txt += 'echo "G4Seed: <$G4Seed>"\n'
946 –                        txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
947 –                        seedIndex += 1
948 –                    if (self.sourceSeedMix):
949 –                        txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
950 –                        txt += 'echo "MixSeed: <$mixSeed>"\n'
951 –                        txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
952 –                        seedIndex += 1
953 –                    pass
954 –                pass
955              txt += 'mv -f '+pset+' pset.cfg\n'
956  
957          if len(self.additional_inbox_files) > 0:
# Line 960 | Line 960 | class Cmssw(JobType):
960              txt += 'fi\n'
961              pass
962  
963 <        if self.pset != None: #CarlosDaniele
963 >        if self.pset != None:
964              txt += '\n'
965              txt += 'echo "***** cat pset.cfg *********"\n'
966              txt += 'cat pset.cfg\n'
967              txt += 'echo "****** end pset.cfg ********"\n'
968              txt += '\n'
969 –            ### FEDE FOR DBS OUTPUT PUBLICATION
969              txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
970              txt += 'echo "PSETHASH = $PSETHASH" \n'
972 –            ##############
971              txt += '\n'
972          return txt
973 <
974 <    def wsBuildExe(self, nj=0):
973 >    #### FEDE #####
974 >    def wsUntarSoftware(self, nj=0):
975          """
976          Put in the script the commands to build an executable
977          or a library.
978          """
979  
980 <        txt = ""
980 >        txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
981  
982          if os.path.isfile(self.tgzNameWithPath):
983              txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
984              txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
985              txt += 'untar_status=$? \n'
986              txt += 'if [ $untar_status -ne 0 ]; then \n'
987 <            txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
988 <            txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
989 <            txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
992 <            txt += '   if [ $middleware == OSG ]; then \n'
993 <            txt += '       cd $RUNTIME_AREA\n'
994 <            txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
995 <            txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
996 <            txt += '       /bin/rm -rf $WORKING_DIR\n'
997 <            txt += '       if [ -d $WORKING_DIR ] ;then\n'
998 <            txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
999 <            txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1000 <            txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1001 <            txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1002 <            txt += '       fi\n'
1003 <            txt += '   fi \n'
1004 <            txt += '   \n'
1005 <            txt += '   exit 1 \n'
987 >            txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
988 >            txt += '   job_exit_code=$untar_status\n'
989 >            txt += '   func_exit\n'
990              txt += 'else \n'
991              txt += '   echo "Successful untar" \n'
992              txt += 'fi \n'
993              txt += '\n'
1010 –            #### Removed ProdAgent API dependencies
994              txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
995              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
996 <            #### FEDE FOR DBS OUTPUT PUBLICATION
1014 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
996 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
997              txt += 'else\n'
998 <            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
998 >            txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
999              txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1018 –            ###################
1000              txt += 'fi\n'
1001              txt += '\n'
1002  
1003              pass
1004  
1005          return txt
1006 +        
1007 +    def wsBuildExe(self, nj=0):
1008 +        """
1009 +        Put in the script the commands to build an executable
1010 +        or a library.
1011 +        """
1012 +
1013 +        txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1014 +        txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1015 +
1016 +        txt += 'mv $RUNTIME_AREA/lib . \n'
1017 +        txt += 'mv $RUNTIME_AREA/module . \n'
1018 +        txt += 'mv $RUNTIME_AREA/ProdCommon . \n'
1019 +        
1020 +
1021 +        #if os.path.isfile(self.tgzNameWithPath):
1022 +        #    txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1023 +        #    txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1024 +        #    txt += 'untar_status=$? \n'
1025 +        #    txt += 'if [ $untar_status -ne 0 ]; then \n'
1026 +        #    txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
1027 +        #    txt += '   job_exit_code=$untar_status\n'
1028 +        #    txt += '   func_exit\n'
1029 +        #    txt += 'else \n'
1030 +        #    txt += '   echo "Successful untar" \n'
1031 +        #    txt += 'fi \n'
1032 +        #    txt += '\n'
1033 +        #    txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
1034 +        txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1035 +        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
1036 +        txt += 'else\n'
1037 +        txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1038 +        txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1039 +        txt += 'fi\n'
1040 +        txt += '\n'
1041 +
1042 +        return txt
1043 +    ############################################################################
1044  
1045      def modifySteeringCards(self, nj):
1046          """
# Line 1036 | Line 1055 | class Cmssw(JobType):
1055              return self.executable
1056  
1057      def executableArgs(self):
1058 +        # FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1059          if self.scriptExe:#CarlosDaniele
1060              return   self.scriptExe + " $NJob"
1061          else:
1042 –            # if >= CMSSW_1_5_X, add -j crab_fjr.xml
1062              version_array = self.scram.getSWVersion().split('_')
1063              major = 0
1064              minor = 0
# Line 1049 | Line 1068 | class Cmssw(JobType):
1068              except:
1069                  msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1070                  raise CrabException(msg)
1071 +
1072 +            ex_args = ""
1073 +
1074 +            # Framework job report
1075              if major >= 1 and minor >= 5 :
1076 <                return " -j " + self.fjrFileName + " -p pset.cfg"
1076 >                #ex_args += " -j " + self.fjrFileName
1077 >            ### FEDE it could be improved!!! ####    
1078 >                ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1079 >            #######################################
1080 >            # Type of cfg file
1081 >            if major >= 2 :
1082 >                ex_args += " -p pset.pycfg"
1083              else:
1084 <                return " -p pset.cfg"
1084 >                ex_args += " -p pset.cfg"
1085 >            return ex_args
1086  
1087      def inputSandbox(self, nj):
1088          """
# Line 1072 | Line 1102 | class Cmssw(JobType):
1102          ## additional input files
1103          tgz = self.additionalInputFileTgz()
1104          inp_box.append(tgz)
1105 +        ## executable
1106 +        wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1107 +        inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1108          return inp_box
1109  
1110      def outputSandbox(self, nj):
# Line 1097 | Line 1130 | class Cmssw(JobType):
1130          Returns part of a job script which renames the produced files.
1131          """
1132  
1133 <        txt = '\n'
1133 >        txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1134          txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1135          txt += 'echo ">>> current directory content:"\n'
1136          txt += 'ls \n'
1137          txt += '\n'
1138  
1139 <        txt += 'output_exit_status=0\n'
1139 >        #txt += 'output_exit_status=0\n'
1140  
1141 <        for fileWithSuffix in (self.output_file_sandbox):
1142 <            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1143 <            txt += '\n'
1144 <            txt += '# check output file\n'
1145 <            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1146 <            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1147 <            txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1148 <            txt += 'else\n'
1149 <            txt += '    exit_status=60302\n'
1150 <            txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1151 <            if common.scheduler.name().upper() == 'CONDOR_G':
1152 <                txt += '    if [ $middleware == OSG ]; then \n'
1153 <                txt += '        echo "prepare dummy output file"\n'
1154 <                txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1155 <                txt += '    fi \n'
1156 <            txt += 'fi\n'
1141 >        ### FEDE #######
1142 >        #for fileWithSuffix in (self.output_file_sandbox):
1143 >        #    output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1144 >        #    txt += '\n'
1145 >        #    txt += '# check output file\n'
1146 >        #    txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1147 >        #    txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1148 >        #    txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1149 >        #    txt += 'else\n'
1150 >        #    txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1151 >        #    txt += '    job_exit_code=60302\n'
1152 >        #    if common.scheduler.name().upper() == 'CONDOR_G':
1153 >        #        txt += '    if [ $middleware == OSG ]; then \n'
1154 >        #        txt += '        echo "prepare dummy output file"\n'
1155 >        #        txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1156 >        #        txt += '    fi \n'
1157 >        #    txt += 'fi\n'
1158  
1159          for fileWithSuffix in (self.output_file):
1160              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
# Line 1134 | Line 1168 | class Cmssw(JobType):
1168                  txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1169                  txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1170              txt += 'else\n'
1171 <            txt += '    exit_status=60302\n'
1172 <            txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1173 <            txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1174 <            txt += '    output_exit_status=$exit_status\n'
1171 >            #txt += '    exit_status=60302\n'
1172 >            #txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1173 >            #txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1174 >            #txt += '    output_exit_status=$exit_status\n'
1175 >            txt += '    job_exit_code=60302\n'
1176 >            txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1177              if common.scheduler.name().upper() == 'CONDOR_G':
1178                  txt += '    if [ $middleware == OSG ]; then \n'
1179                  txt += '        echo "prepare dummy output file"\n'
# Line 1193 | Line 1229 | class Cmssw(JobType):
1229                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1230  
1231          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1232 +        if common.scheduler.name() == "glitecoll":
1233 +            req += ' && other.GlueCEStateStatus == "Production" '
1234  
1235          return req
1236  
# Line 1205 | Line 1243 | class Cmssw(JobType):
1243          Returns part of a job script which is prepares
1244          the execution environment and which is common for all CMS jobs.
1245          """
1246 <        txt = '    echo ">>> setup CMS OSG environment:"\n'
1246 >        txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1247 >        txt += '    echo ">>> setup CMS OSG environment:"\n'
1248          txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1249          txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1250          txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
# Line 1213 | Line 1252 | class Cmssw(JobType):
1252          txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1253          txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1254          txt += '    else\n'
1255 <        txt += '        echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1256 <        txt += '        echo "JOB_EXIT_STATUS = 10020"\n'
1257 <        txt += '        echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1258 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1255 >        #txt += '        echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1256 >        #txt += '        echo "JOB_EXIT_STATUS = 10020"\n'
1257 >        #txt += '        echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1258 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1259 >        txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1260 >        txt += '        job_exit_code=10020\n'
1261 >        #txt += '        cd $RUNTIME_AREA\n'
1262 >        #txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1263 >        #txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
1264 >        #txt += '        /bin/rm -rf $WORKING_DIR\n'
1265 >        #txt += '        if [ -d $WORKING_DIR ] ;then\n'
1266 >        #txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1267 >        #txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1268 >        #txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1269 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1270 >        #txt += '            echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1271 >        #txt += '            job_exit_code=10017\n'
1272 >        #txt += '        fi\n'
1273          txt += '\n'
1274 <        txt += '        cd $RUNTIME_AREA\n'
1275 <        txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1223 <        txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
1224 <        txt += '        /bin/rm -rf $WORKING_DIR\n'
1225 <        txt += '        if [ -d $WORKING_DIR ] ;then\n'
1226 <        txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1227 <        txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1228 <        txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1229 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1230 <        txt += '        fi\n'
1231 <        txt += '\n'
1232 <        txt += '        exit 1\n'
1274 >        #txt += '        exit 1\n'
1275 >        txt += '        func_exit\n'
1276          txt += '    fi\n'
1277          txt += '\n'
1278 <        txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1278 >        txt += '    echo "==> setup cms environment ok"\n'
1279          txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1280  
1281          return txt
# Line 1243 | Line 1286 | class Cmssw(JobType):
1286          Returns part of a job script which is prepares
1287          the execution environment and which is common for all CMS jobs.
1288          """
1289 <        txt = '    echo ">>> setup CMS LCG environment:"\n'
1289 >        txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1290 >        txt += '    echo ">>> setup CMS LCG environment:"\n'
1291          txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1292          txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1293          txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1294          txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1295 <        txt += '        echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1296 <        txt += '        echo "JOB_EXIT_STATUS = 10031" \n'
1297 <        txt += '        echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1298 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1299 <        txt += '        exit 1\n'
1295 >        #txt += '        echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1296 >        #txt += '        echo "JOB_EXIT_STATUS = 10031" \n'
1297 >        #txt += '        echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1298 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1299 >        #txt += '        exit 1\n'
1300 >        txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1301 >        txt += '        job_exit_code=10031\n'
1302 >        txt += '        func_exit\n'
1303          txt += '    else\n'
1304          txt += '        echo "Sourcing environment... "\n'
1305          txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1306 <        txt += '            echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1307 <        txt += '            echo "JOB_EXIT_STATUS = 10020"\n'
1308 <        txt += '            echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1309 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1310 <        txt += '            exit 1\n'
1306 >        #txt += '            echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1307 >        #txt += '            echo "JOB_EXIT_STATUS = 10020"\n'
1308 >        #txt += '            echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1309 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1310 >        #txt += '            exit 1\n'
1311 >        txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1312 >        txt += '            job_exit_code=10020\n'
1313 >        txt += '            func_exit\n'
1314          txt += '        fi\n'
1315          txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1316          txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1317          txt += '        result=$?\n'
1318          txt += '        if [ $result -ne 0 ]; then\n'
1319 <        txt += '            echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1320 <        txt += '            echo "JOB_EXIT_STATUS = 10032"\n'
1321 <        txt += '            echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1322 <        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1323 <        txt += '            exit 1\n'
1319 >        #txt += '            echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1320 >        #txt += '            echo "JOB_EXIT_STATUS = 10032"\n'
1321 >        #txt += '            echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1322 >        #txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1323 >        #txt += '            exit 1\n'
1324 >        txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1325 >        txt += '            job_exit_code=10032\n'
1326 >        txt += '            func_exit\n'
1327          txt += '        fi\n'
1328          txt += '    fi\n'
1329          txt += '    \n'
1330 <        txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1330 >        txt += '    echo "==> setup cms environment ok"\n'
1331          return txt
1332  
1333      ### FEDE FOR DBS OUTPUT PUBLICATION
# Line 1283 | Line 1336 | class Cmssw(JobType):
1336          insert the part of the script that modifies the FrameworkJob Report
1337          """
1338  
1339 <        txt = ''
1339 >        txt = '\n#Written by cms_cmssw::modifyReport\n'
1340          try:
1341              publish_data = int(self.cfg_params['USER.publish_data'])
1342          except KeyError:
1343              publish_data = 0
1344          if (publish_data == 1):
1345 <            txt += 'echo ">>> Modify Job Report:" \n'
1346 <            ################ FEDE FOR DBS2 #############################################
1347 <            #txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1348 <            txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1349 <            #############################################################################
1350 <
1351 <            txt += 'if [ -z "$SE" ]; then\n'
1352 <            txt += '    SE="" \n'
1353 <            txt += 'fi \n'
1354 <            txt += 'if [ -z "$SE_PATH" ]; then\n'
1355 <            txt += '    SE_PATH="" \n'
1356 <            txt += 'fi \n'
1304 <            txt += 'echo "SE = $SE"\n'
1305 <            txt += 'echo "SE_PATH = $SE_PATH"\n'
1345 >            
1346 >            txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1347 >            txt += '    echo ">>> Modify Job Report:" \n'
1348 >            txt += '    chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1349 >            #txt += '    if [ -z "$SE" ]; then\n'
1350 >            #txt += '        SE="" \n'
1351 >            #txt += '    fi \n'
1352 >            #txt += '    if [ -z "$SE_PATH" ]; then\n'
1353 >            #txt += '        SE_PATH="" \n'
1354 >            #txt += '    fi \n'
1355 >            txt += '    echo "SE = $SE"\n'
1356 >            txt += '    echo "SE_PATH = $SE_PATH"\n'
1357  
1358              processedDataset = self.cfg_params['USER.publish_data_name']
1359 <            txt += 'ProcessedDataset='+processedDataset+'\n'
1360 <            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1361 <            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1362 <            #### FEDE: added slash in LFN ##############
1363 <            txt += '    FOR_LFN=/copy_problems/ \n'
1364 <            txt += 'else \n'
1359 >            txt += '    ProcessedDataset='+processedDataset+'\n'
1360 >            #txt += '    if [ "$SE_PATH" == "" ]; then\n'
1361 >            #txt += '        FOR_LFN=/copy_problems/ \n'
1362 >            #txt += '    else \n'
1363 >            #txt += '        tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1364 >            #txt += '        FOR_LFN=/store$tmp \n'
1365 >            #txt += '    fi \n'
1366              txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1315 –            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1367              txt += '    FOR_LFN=/store$tmp \n'
1368 <            txt += 'fi \n'
1369 <            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1370 <            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1371 <            txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1372 <            txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1373 <            txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1374 <
1375 <            txt += 'modifyReport_result=$?\n'
1376 <            txt += 'echo modifyReport_result = $modifyReport_result\n'
1377 <            txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1378 <            txt += '    exit_status=1\n'
1379 <            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1380 <            txt += 'else\n'
1381 <            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1368 >            txt += '    echo "ProcessedDataset = $ProcessedDataset"\n'
1369 >            txt += '    echo "FOR_LFN = $FOR_LFN" \n'
1370 >            txt += '    echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1371 >            #txt += '    echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1372 >            #txt += '    $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1373 >            ### FEDE ####
1374 >            txt += '    echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1375 >            txt += '    $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1376 >            ####################################
1377 >            txt += '    modifyReport_result=$?\n'
1378 >            txt += '    if [ $modifyReport_result -ne 0 ]; then\n'
1379 >            txt += '        modifyReport_result=70500\n'
1380 >            txt += '        job_exit_code=$modifyReport_result\n'
1381 >            txt += '        echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1382 >            txt += '        echo "WARNING: Problem with ModifyJobReport"\n'
1383 >            txt += '    else\n'
1384 >            ### FEDE #####
1385 >            #txt += '        mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1386 >            #######################
1387 >            txt += '        mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1388 >            txt += '    fi\n'
1389              txt += 'fi\n'
1332 –        else:
1333 –            txt += 'echo "no data publication required"\n'
1390          return txt
1391  
1392      def cleanEnv(self):
1393 <        txt = ''
1393 >        txt = '\n#Written by cms_cmssw::cleanEnv\n'
1394          txt += 'if [ $middleware == OSG ]; then\n'
1395          txt += '    cd $RUNTIME_AREA\n'
1396          txt += '    echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1397          txt += '    echo ">>> Remove working directory: $WORKING_DIR"\n'
1398          txt += '    /bin/rm -rf $WORKING_DIR\n'
1399          txt += '    if [ -d $WORKING_DIR ] ;then\n'
1400 <        txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401 <        txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1402 <        txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1347 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1400 >        txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401 >        txt += '        job_exit_code=10017\n'
1402 >        txt += '        func_exit\n'
1403          txt += '    fi\n'
1404          txt += 'fi\n'
1405          txt += '\n'
# Line 1365 | Line 1420 | class Cmssw(JobType):
1420              nd[e]=0
1421          return nd.keys()
1422  
1423 <
1369 <    def checkOut(self, limit):
1423 >    def outList(self):
1424          """
1425          check the dimension of the output files
1426          """
1427 <        txt = 'echo ">>> Starting output sandbox limit check :"\n'
1427 >        txt = ''
1428 >        txt += 'echo ">>> list of expected files on output sandbox"\n'
1429          listOutFiles = []
1430 <        txt += 'stdoutFile=`ls *stdout` \n'
1431 <        txt += 'stderrFile=`ls *stderr` \n'
1430 >        stdout = 'CMSSW_$NJob.stdout'
1431 >        stderr = 'CMSSW_$NJob.stderr'
1432          if (self.return_data == 1):
1433              for file in (self.output_file+self.output_file_sandbox):
1434                  listOutFiles.append(self.numberFile_(file, '$NJob'))
1435 <            listOutFiles.append('$stdoutFile')
1436 <            listOutFiles.append('$stderrFile')
1435 >            listOutFiles.append(stdout)
1436 >            listOutFiles.append(stderr)
1437          else:
1438              for file in (self.output_file_sandbox):
1439                  listOutFiles.append(self.numberFile_(file, '$NJob'))
1440 <            listOutFiles.append('$stdoutFile')
1441 <            listOutFiles.append('$stderrFile')
1442 <  
1388 <        txt += 'echo "OUTPUT files: '+string.join(listOutFiles,' ')+'"\n'
1440 >            listOutFiles.append(stdout)
1441 >            listOutFiles.append(stderr)
1442 >        txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1443          txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1444 <       # txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1445 <        txt += 'ls -gGhrta;\n'
1392 <        txt += 'sum=0;\n'
1393 <        txt += 'for file in $filesToCheck ; do\n'
1394 <        txt += '    if [ -e $file ]; then\n'
1395 <        txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1396 <        txt += '        sum=`expr $sum + $tt`\n'
1397 <        txt += '    else\n'
1398 <        txt += '        echo "WARNING: output file $file not found!"\n'
1399 <        txt += '    fi\n'
1400 <        txt += 'done\n'
1401 <        txt += 'echo "Total Output dimension: $sum";\n'
1402 <        txt += 'limit='+str(limit)+';\n'
1403 <        txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1404 <        txt += 'if [ $limit -lt $sum ]; then\n'
1405 <        txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1406 <        txt += '    echo "         checking the output file sizes..."\n'
1407 <        txt += '    tot=0;\n'
1408 <        txt += '    for filefile in $filesToCheck ; do\n'
1409 <        txt += '        dimFile=`ls -gGrta $filefile | awk \'{ print $3 }\';`\n'
1410 <        txt += '        tot=`expr $tot + $tt`;\n'
1411 <        txt += '        if [ $limit -lt $dimFile ]; then\n'
1412 <        txt += '            echo "deleting file: $filefile";\n'
1413 <        txt += '            rm -f $filefile\n'
1414 <        txt += '        elif [ $limit -lt $tot ]; then\n'
1415 <        txt += '            echo "deleting file: $filefile";\n'
1416 <        txt += '            rm -f $filefile\n'
1417 <        txt += '        else\n'
1418 <        txt += '            echo "saving file: $filefile"\n'
1419 <        txt += '        fi\n'
1420 <        txt += '    done\n'
1421 <
1422 <        txt += '    ls -agGhrt;\n'
1423 <        txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1424 <        txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1425 <        txt += '    exit_status=70000;\n'
1426 <        txt += 'else'
1427 <        txt += '    echo "Total Output dimension $sum is fine.";\n'
1428 <        txt += 'fi\n'
1429 <        txt += 'echo "Ending output sandbox limit check"\n'
1430 <        return txt
1444 >        txt += 'export filesToCheck\n'
1445 >        return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines