ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.92 by gutsche, Tue Jun 19 16:21:49 2007 UTC vs.
Revision 1.111 by farinafa, Tue Aug 7 15:06:49 2007 UTC

# Line 5 | Line 5 | from crab_util import *
5   import common
6   import Scram
7  
8 < import os, string, re, shutil, glob
8 > import os, string, glob
9  
10   class Cmssw(JobType):
11      def __init__(self, cfg_params, ncjobs):
12          JobType.__init__(self, 'CMSSW')
13          common.logger.debug(3,'CMSSW::__init__')
14  
15 –        # Marco.
15          self._params = {}
16          self.cfg_params = cfg_params
17  
# Line 32 | Line 31 | class Cmssw(JobType):
31          self.executable = ''
32          self.executable_arch = self.scram.getArch()
33          self.tgz_name = 'default.tgz'
34 +        self.additional_tgz_name = 'additional.tgz'
35          self.scriptName = 'CMSSW.sh'
36          self.pset = ''      #scrip use case Da  
37          self.datasetPath = '' #scrip use case Da
# Line 173 | Line 173 | class Cmssw(JobType):
173                      if not os.path.exists(file):
174                          raise CrabException("Additional input file not found: "+file)
175                      pass
176 <                    fname = string.split(file, '/')[-1]
177 <                    storedFile = common.work_space.pathForTgz()+'share/'+fname
178 <                    shutil.copyfile(file, storedFile)
179 <                    self.additional_inbox_files.append(string.strip(storedFile))
176 >                    # fname = string.split(file, '/')[-1]
177 >                    # storedFile = common.work_space.pathForTgz()+'share/'+fname
178 >                    # shutil.copyfile(file, storedFile)
179 >                    self.additional_inbox_files.append(string.strip(file))
180                  pass
181              pass
182              common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
# Line 253 | Line 253 | class Cmssw(JobType):
253              self.firstRun = None
254              common.logger.debug(5,"No first run given")
255          if self.pset != None: #CarlosDaniele
256 <            import PsetManipulator  
257 <            PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
256 >            ver = string.split(self.version,"_")
257 >            if (int(ver[1])>=1 and int(ver[2])>=5):
258 >                import PsetManipulator150 as pp
259 >            else:
260 >                import PsetManipulator as pp
261 >            PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
262  
263          #DBSDLS-start
264          ## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
# Line 440 | Line 444 | class Cmssw(JobType):
444          while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
445              block = blocks[blockCount]
446              blockCount += 1
447 +            if block not in jobsOfBlock.keys() :
448 +                jobsOfBlock[block] = []
449              
450              if self.eventsbyblock.has_key(block) :
451                  numEventsInBlock = self.eventsbyblock[block]
# Line 490 | Line 496 | class Cmssw(JobType):
496                              self.jobDestination.append(blockSites[block])
497                              common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
498                              # fill jobs of block dictionary
499 <                            if block in jobsOfBlock.keys() :
494 <                                jobsOfBlock[block].append(jobCount+1)
495 <                            else:
496 <                                jobsOfBlock[block] = [jobCount+1]
499 >                            jobsOfBlock[block].append(jobCount+1)
500                              # reset counter
501                              jobCount = jobCount + 1
502                              totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
# Line 517 | Line 520 | class Cmssw(JobType):
520                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
521                          self.jobDestination.append(blockSites[block])
522                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
523 <                        if block in jobsOfBlock.keys() :
521 <                            jobsOfBlock[block].append(jobCount+1)
522 <                        else:
523 <                            jobsOfBlock[block] = [jobCount+1]
523 >                        jobsOfBlock[block].append(jobCount+1)
524                          # reset counter
525                          jobCount = jobCount + 1
526                          totalEventCount = totalEventCount + eventsPerJobRequested
# Line 541 | Line 541 | class Cmssw(JobType):
541                          common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
542                          self.jobDestination.append(blockSites[block])
543                          common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
544 <                        if block in jobsOfBlock.keys() :
545 <                            jobsOfBlock[block].append(jobCount+1)
546 <                        else:
547 <                            jobsOfBlock[block] = [jobCount+1]
544 >                        jobsOfBlock[block].append(jobCount+1)
545                          # increase counter
546                          jobCount = jobCount + 1
547                          totalEventCount = totalEventCount + eventsPerJobRequested
# Line 568 | Line 565 | class Cmssw(JobType):
565          screenOutput = "List of jobs and available destination sites:\n\n"
566  
567          blockCounter = 0
568 <        for block in jobsOfBlock.keys():
569 <            blockCounter += 1
570 <            screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
568 >        for block in blocks:
569 >            if block in jobsOfBlock.keys() :
570 >                blockCounter += 1
571 >                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
572  
573 <        common.logger.message(screenOutput)
573 >       # common.logger.message(screenOutput)
574  
575          self.list_of_args = list_of_lists
576          return
# Line 784 | Line 782 | class Cmssw(JobType):
782              pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
783              if os.path.isdir(pa):
784                  tar.add(pa,paDir)
785 +
786 +            ### FEDE FOR DBS PUBLICATION
787 +            ## Add PRODCOMMON dir to tar
788 +            prodcommonDir = 'ProdCommon'
789 +            prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
790 +            if os.path.isdir(prodcommonPath):
791 +                tar.add(prodcommonPath,prodcommonDir)
792 +            #############################    
793          
794              common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
795              tar.close()
# Line 809 | Line 815 | class Cmssw(JobType):
815          
816          return
817          
818 +    def additionalInputFileTgz(self):
819 +        """
820 +        Put all additional files into a tar ball and return its name
821 +        """
822 +        import tarfile
823 +        tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
824 +        tar = tarfile.open(tarName, "w:gz")
825 +        for file in self.additional_inbox_files:
826 +            tar.add(file,string.split(file,'/')[-1])
827 +        common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
828 +        tar.close()
829 +        return tarName
830 +
831      def wsSetupEnvironment(self, nj):
832          """
833          Returns part of a job script which prepares
# Line 819 | Line 838 | class Cmssw(JobType):
838    
839          ## OLI_Daniele at this level  middleware already known
840  
841 +        txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
842 +        txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
843 +        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
844 +        txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
845          txt += 'if [ $middleware == LCG ]; then \n'
846          txt += self.wsSetupCMSLCGEnvironment_()
847          txt += 'elif [ $middleware == OSG ]; then\n'
# Line 844 | Line 867 | class Cmssw(JobType):
867          scram = self.scram.commandName()
868          txt += '\n\n'
869          txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
847 –        txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
848 –        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
870          txt += scram+' project CMSSW '+self.version+'\n'
871          txt += 'status=$?\n'
872          txt += 'if [ $status != 0 ] ; then\n'
# Line 862 | Line 883 | class Cmssw(JobType):
883          txt += '        cd $RUNTIME_AREA\n'
884          txt += '        /bin/rm -rf $WORKING_DIR\n'
885          txt += '        if [ -d $WORKING_DIR ] ;then\n'
886 <        txt += '        echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887 <        txt += '        echo "JOB_EXIT_STATUS = 10018"\n'
888 <        txt += '        echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
886 >        txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887 >        txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
888 >        txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
890          txt += '            rm -f $RUNTIME_AREA/$repo \n'
891          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
892          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 875 | Line 896 | class Cmssw(JobType):
896          txt += 'fi \n'
897          txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
898          txt += 'cd '+self.version+'\n'
899 +        ########## FEDE FOR DBS2 ######################
900 +        txt += 'SOFTWARE_DIR=`pwd`\n'
901 +        txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
902 +        ###############################################
903          ### needed grep for bug in scramv1 ###
904          txt += scram+' runtime -sh\n'
905          txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
# Line 900 | Line 925 | class Cmssw(JobType):
925          txt += '        cd $RUNTIME_AREA\n'
926          txt += '        /bin/rm -rf $WORKING_DIR\n'
927          txt += '        if [ -d $WORKING_DIR ] ;then\n'
928 <        txt += '        echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929 <        txt += '        echo "JOB_EXIT_STATUS = 50114"\n'
930 <        txt += '        echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
928 >        txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929 >        txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
930 >        txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931 >        txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
932          txt += '            rm -f $RUNTIME_AREA/$repo \n'
933          txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
934          txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
# Line 915 | Line 940 | class Cmssw(JobType):
940  
941          # Prepare job-specific part
942          job = common.job_list[nj]
943 +        ### FEDE FOR DBS OUTPUT PUBLICATION
944 +        if (self.datasetPath):
945 +            txt += '\n'
946 +            txt += 'DatasetPath='+self.datasetPath+'\n'
947 +
948 +            datasetpath_split = self.datasetPath.split("/")
949 +            
950 +            txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
951 +            txt += 'DataTier='+datasetpath_split[2]+'\n'
952 +            #txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
953 +            txt += 'ApplicationFamily=cmsRun\n'
954 +
955 +        else:
956 +            txt += 'DatasetPath=MCDataTier\n'
957 +            txt += 'PrimaryDataset=null\n'
958 +            txt += 'DataTier=null\n'
959 +            #txt += 'ProcessedDataset=null\n'
960 +            txt += 'ApplicationFamily=MCDataTier\n'
961          if self.pset != None: #CarlosDaniele
962              pset = os.path.basename(job.configFilename())
963              txt += '\n'
964 +            txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
965              if (self.datasetPath): # standard job
966                  #txt += 'InputFiles=$2\n'
967                  txt += 'InputFiles=${args[1]}\n'
# Line 962 | Line 1006 | class Cmssw(JobType):
1006              txt += 'mv -f '+pset+' pset.cfg\n'
1007  
1008          if len(self.additional_inbox_files) > 0:
1009 <            for file in self.additional_inbox_files:
1010 <                relFile = file.split("/")[-1]
1011 <                txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
968 <                txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
969 <                txt += '   chmod +x '+relFile+'\n'
970 <                txt += 'fi\n'
1009 >            txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1010 >            txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1011 >            txt += 'fi\n'
1012              pass
1013  
1014          if self.pset != None: #CarlosDaniele
# Line 978 | Line 1019 | class Cmssw(JobType):
1019              txt += 'cat pset.cfg\n'
1020              txt += 'echo "****** end pset.cfg ********"\n'
1021              txt += '\n'
1022 +            ### FEDE FOR DBS OUTPUT PUBLICATION
1023 +            txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1024 +            txt += 'echo "PSETHASH = $PSETHASH" \n'
1025 +            ##############
1026 +            txt += '\n'
1027 +            # txt += 'echo "***** cat pset1.cfg *********"\n'
1028 +            # txt += 'cat pset1.cfg\n'
1029 +            # txt += 'echo "****** end pset1.cfg ********"\n'
1030          return txt
1031  
1032      def wsBuildExe(self, nj=0):
# Line 1016 | Line 1065 | class Cmssw(JobType):
1065              txt += '   echo "Successful untar" \n'
1066              txt += 'fi \n'
1067              txt += '\n'
1068 <            txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1068 >            txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1069              txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1070 <            txt += '   export PYTHONPATH=ProdAgentApi\n'
1070 >            #### FEDE FOR DBS OUTPUT PUBLICATION
1071 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1072 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1073 >            #txt += '   export PYTHONPATH=ProdAgentApi\n'
1074              txt += 'else\n'
1075 <            txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1075 >            txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1076 >            #txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1077 >            #txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1078 >            txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1079 >            ###################  
1080              txt += 'fi\n'
1081              txt += '\n'
1082  
# Line 1062 | Line 1118 | class Cmssw(JobType):
1118          if not self.pset is None:
1119              inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1120          ## additional input files
1121 <        for file in self.additional_inbox_files:
1122 <            inp_box.append(file)
1121 >        tgz = self.additionalInputFileTgz()
1122 >        inp_box.append(tgz)
1123          return inp_box
1124  
1125      def outputSandbox(self, nj):
# Line 1097 | Line 1153 | class Cmssw(JobType):
1153              output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1154              txt += '\n'
1155              txt += '# check output file\n'
1156 <            txt += 'ls '+fileWithSuffix+'\n'
1157 <            txt += 'ls_result=$?\n'
1158 <            txt += 'if [ $ls_result -ne 0 ] ; then\n'
1159 <            #txt += '   JOB_EXIT_STATUS=60302\n'
1160 <            ### FEDE
1156 >            # txt += 'ls '+fileWithSuffix+'\n'
1157 >            # txt += 'ls_result=$?\n'
1158 >            txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1159 >            txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1160 >            txt += 'else\n'
1161              txt += '   exit_status=60302\n'
1162 <            ####
1107 <            txt += '   echo "ERROR: Problem with output file"\n'
1162 >            txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1163              if common.scheduler.boss_scheduler_name == 'condor_g':
1164                  txt += '    if [ $middleware == OSG ]; then \n'
1165                  txt += '        echo "prepare dummy output file"\n'
1166                  txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1167                  txt += '    fi \n'
1113 –            txt += 'else\n'
1114 –            txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1168              txt += 'fi\n'
1169 +        file_list = []
1170 +        for fileWithSuffix in (self.output_file):
1171 +             file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1172 +        txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1173        
1174          txt += 'cd $RUNTIME_AREA\n'
1175 <        txt += 'cd $RUNTIME_AREA\n'
1175 >        #### FEDE this is the cleanEnv function
1176          ### OLI_DANIELE
1177 <        txt += 'if [ $middleware == OSG ]; then\n'  
1178 <        txt += '    cd $RUNTIME_AREA\n'
1179 <        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180 <        txt += '    /bin/rm -rf $WORKING_DIR\n'
1181 <        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182 <        txt += '    echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183 <        txt += '    echo "JOB_EXIT_STATUS = 60999"\n'
1184 <        txt += '    echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185 <        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
1186 <        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187 <        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188 <        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189 <        txt += '    fi\n'
1190 <        txt += 'fi\n'
1191 <        txt += '\n'
1177 >        #txt += 'if [ $middleware == OSG ]; then\n'  
1178 >        #txt += '    cd $RUNTIME_AREA\n'
1179 >        #txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180 >        #txt += '    /bin/rm -rf $WORKING_DIR\n'
1181 >        #txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182 >        #txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183 >        #txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1184 >        #txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185 >        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1186 >        #txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187 >        #txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188 >        #txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189 >        #txt += '    fi\n'
1190 >        #txt += 'fi\n'
1191 >        #txt += '\n'
1192  
1136 –        file_list = ''
1137 –        ## Add to filelist only files to be possibly copied to SE
1138 –        for fileWithSuffix in self.output_file:
1139 –            output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1140 –            file_list=file_list+output_file_num+' '
1141 –        file_list=file_list[:-1]
1142 –        txt += 'file_list="'+file_list+'"\n'
1193  
1194          return txt
1195  
# Line 1170 | Line 1220 | class Cmssw(JobType):
1220              req='Member("VO-cms-' + \
1221                   self.version + \
1222                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1223 <        # if self.executable_arch:
1224 <        #     req='Member("VO-cms-' + \
1225 <        #          self.executable_arch + \
1226 <        #          '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1223 >        ## SL add requirement for OS version only if SL4
1224 >        #reSL4 = re.compile( r'slc4' )
1225 >        if self.executable_arch: # and reSL4.search(self.executable_arch):
1226 >            req+=' && Member("VO-cms-' + \
1227 >                 self.executable_arch + \
1228 >                 '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1229  
1230          req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1231  
# Line 1213 | Line 1265 | class Cmssw(JobType):
1265          txt += '       cd $RUNTIME_AREA\n'
1266          txt += '       /bin/rm -rf $WORKING_DIR\n'
1267          txt += '       if [ -d $WORKING_DIR ] ;then\n'
1268 <        txt += '        echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1269 <        txt += '        echo "JOB_EXIT_STATUS = 10017"\n'
1270 <        txt += '        echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271 <        txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1272 <        txt += '            rm -f $RUNTIME_AREA/$repo \n'
1273 <        txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274 <        txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1268 >        txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1269 >        txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1270 >        txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271 >        txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1272 >        txt += '           rm -f $RUNTIME_AREA/$repo \n'
1273 >        txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274 >        txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1275          txt += '       fi\n'
1276          txt += '\n'
1277          txt += '       exit 1\n'
# Line 1278 | Line 1330 | class Cmssw(JobType):
1330          txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1331          return txt
1332  
1333 +    ### FEDE FOR DBS OUTPUT PUBLICATION
1334 +    def modifyReport(self, nj):
1335 +        """
1336 +        insert the part of the script that modifies the FrameworkJob Report
1337 +        """
1338 +
1339 +        txt = ''
1340 +        txt += 'echo "Modify Job Report" \n'
1341 +        #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1342 +        ################ FEDE FOR DBS2 #############################################
1343 +        txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1344 +        #############################################################################
1345 +        try:
1346 +            publish_data = int(self.cfg_params['USER.publish_data'])          
1347 +        except KeyError:
1348 +            publish_data = 0
1349 +
1350 +        txt += 'if [ -z "$SE" ]; then\n'
1351 +        txt += '    SE="" \n'
1352 +        txt += 'fi \n'
1353 +        txt += 'if [ -z "$SE_PATH" ]; then\n'
1354 +        txt += '    SE_PATH="" \n'
1355 +        txt += 'fi \n'
1356 +        txt += 'echo "SE = $SE"\n'
1357 +        txt += 'echo "SE_PATH = $SE_PATH"\n'
1358 +
1359 +        if (publish_data == 1):  
1360 +            #processedDataset = self.cfg_params['USER.processed_datasetname']
1361 +            processedDataset = self.cfg_params['USER.publish_data_name']
1362 +            txt += 'ProcessedDataset='+processedDataset+'\n'
1363 +            #### LFN=/store/user/<user>/processedDataset_PSETHASH
1364 +            txt += 'if [ "$SE_PATH" == "" ]; then\n'
1365 +            #### FEDE: added slash in LFN ##############
1366 +            txt += '    FOR_LFN=/copy_problems/ \n'
1367 +            txt += 'else \n'
1368 +            txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1369 +            #####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1370 +            txt += '    FOR_LFN=/store$tmp \n'
1371 +            txt += 'fi \n'
1372 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1373 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1374 +            txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1375 +            #txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1376 +            txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1377 +            txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1378 +            #txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1379 +      
1380 +            txt += 'modifyReport_result=$?\n'
1381 +            txt += 'echo modifyReport_result = $modifyReport_result\n'
1382 +            txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1383 +            txt += '    exit_status=1\n'
1384 +            txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1385 +            txt += 'else\n'
1386 +            txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1387 +            txt += 'fi\n'
1388 +        else:
1389 +            txt += 'ProcessedDataset=no_data_to_publish \n'
1390 +            #### FEDE: added slash in LFN ##############
1391 +            txt += 'FOR_LFN=/local/ \n'
1392 +            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1393 +            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1394 +        return txt
1395 +
1396 +    def cleanEnv(self):
1397 +        ### OLI_DANIELE
1398 +        txt = ''
1399 +        txt += 'if [ $middleware == OSG ]; then\n'  
1400 +        txt += '    cd $RUNTIME_AREA\n'
1401 +        txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1402 +        txt += '    /bin/rm -rf $WORKING_DIR\n'
1403 +        txt += '    if [ -d $WORKING_DIR ] ;then\n'
1404 +        txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1405 +        txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1406 +        txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1407 +        txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1408 +        txt += '        rm -f $RUNTIME_AREA/$repo \n'
1409 +        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1410 +        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1411 +        txt += '    fi\n'
1412 +        txt += 'fi\n'
1413 +        txt += '\n'
1414 +        return txt
1415 +
1416      def setParam_(self, param, value):
1417          self._params[param] = value
1418  
# Line 1290 | Line 1425 | class Cmssw(JobType):
1425      def getTaskid(self):
1426          return self._taskId
1427  
1293 – #######################################################################
1428      def uniquelist(self, old):
1429          """
1430          remove duplicates from a list

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines