ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/cms_cmssw.py
(Generate patch)

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.107 by gutsche, Tue Jul 24 19:49:35 2007 UTC vs.
Revision 1.116.2.2 by fanzago, Tue Oct 9 17:18:11 2007 UTC

# Line 2 | Line 2 | from JobType import JobType
2   from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5 + from BlackWhiteListParser import BlackWhiteListParser
6   import common
7   import Scram
8  
# Line 15 | Line 16 | class Cmssw(JobType):
16          self._params = {}
17          self.cfg_params = cfg_params
18  
19 +        # init BlackWhiteListParser
20 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
21 +
22          try:
23              self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24          except KeyError:
# Line 40 | Line 44 | class Cmssw(JobType):
44          self.fjrFileName = 'crab_fjr.xml'
45  
46          self.version = self.scram.getSWVersion()
47 +        
48 +        #
49 +        # Try to block creation in case of arch/version mismatch
50 +        #
51 +
52 +        a = string.split(self.version, "_")
53 +
54 +        if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
55 +            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
56 +            raise CrabException(msg)
57 +        if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58 +            msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
59 +            raise CrabException(msg)
60 +        
61          common.taskDB.setDict('codeVersion',self.version)
62          self.setParam_('application', self.version)
63  
# Line 73 | Line 91 | class Cmssw(JobType):
91              try:
92                  datasetpath_split = self.datasetPath.split("/")
93                  # standard style
94 +                self.setParam_('datasetFull', self.datasetPath)
95                  if self.use_dbs_1 == 1 :
96                      self.setParam_('dataset', datasetpath_split[1])
97                      self.setParam_('owner', datasetpath_split[-1])
# Line 300 | Line 319 | class Cmssw(JobType):
319                          if (self.sourceSeedVtx) :
320                              PsetEdit.vtxSeed("INPUTVTX")
321                          if (self.sourceSeedG4) :
322 <                            self.PsetEdit.g4Seed("INPUTG4")
322 >                            PsetEdit.g4Seed("INPUTG4")
323                          if (self.sourceSeedMix) :
324 <                            self.PsetEdit.mixSeed("INPUTMIX")
324 >                            PsetEdit.mixSeed("INPUTMIX")
325                  # add FrameworkJobReport to parameter-set
326                  PsetEdit.addCrabFJR(self.fjrFileName)
327                  PsetEdit.psetWriter(self.configFilename())
# Line 568 | Line 587 | class Cmssw(JobType):
587          for block in blocks:
588              if block in jobsOfBlock.keys() :
589                  blockCounter += 1
590 <                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
590 >                screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
591  
592          common.logger.message(screenOutput)
593  
# Line 838 | Line 857 | class Cmssw(JobType):
857    
858          ## OLI_Daniele at this level  middleware already known
859  
841 –        txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
842 –        txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
843 –        txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
844 –        txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
860          txt += 'if [ $middleware == LCG ]; then \n'
861 +        txt += '    echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
862 +        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
863 +        txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
864          txt += self.wsSetupCMSLCGEnvironment_()
865          txt += 'elif [ $middleware == OSG ]; then\n'
866          txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
# Line 861 | Line 879 | class Cmssw(JobType):
879          txt += '    echo "Change to working directory: $WORKING_DIR"\n'
880          txt += '    cd $WORKING_DIR\n'
881          txt += self.wsSetupCMSOSGEnvironment_()
882 +        txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
883 +        txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
884          txt += 'fi\n'
885  
886          # Prepare JobType-specific part
# Line 1169 | Line 1189 | class Cmssw(JobType):
1189              # txt += 'ls '+fileWithSuffix+'\n'
1190              # txt += 'ls_result=$?\n'
1191              txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1192 <            txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1192 >            ###### FEDE FOR OUTPUT DATA PUBLICATION ########
1193 >            txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1194 >            txt += '    cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1195 >            ################################################
1196              txt += 'else\n'
1197 <            txt += '   exit_status=60302\n'
1198 <            txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1197 >            txt += '    exit_status=60302\n'
1198 >            txt += '    echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1199 >            ############# FEDE ADDED CHECK FOR OUTPUT #############
1200 >            ## MATTY's FIX: the exit option was interrupting the execution
1201 >            if fileWithSuffix in self.output_file:
1202 >                txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1203 >                txt += '    output_exit_status=$exit_status\n'
1204 >                txt += '    # exit $exit_status\n'
1205 >            #######################################################    
1206              if common.scheduler.boss_scheduler_name == 'condor_g':
1207                  txt += '    if [ $middleware == OSG ]; then \n'
1208                  txt += '        echo "prepare dummy output file"\n'
# Line 1182 | Line 1212 | class Cmssw(JobType):
1212          file_list = []
1213          for fileWithSuffix in (self.output_file):
1214               file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1215 +            
1216          txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1186 –      
1217          txt += 'cd $RUNTIME_AREA\n'
1188 –        #### FEDE this is the cleanEnv function
1189 –        ### OLI_DANIELE
1190 –        #txt += 'if [ $middleware == OSG ]; then\n'  
1191 –        #txt += '    cd $RUNTIME_AREA\n'
1192 –        #txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1193 –        #txt += '    /bin/rm -rf $WORKING_DIR\n'
1194 –        #txt += '    if [ -d $WORKING_DIR ] ;then\n'
1195 –        #txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1196 –        #txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1197 –        #txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1198 –        #txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1199 –        #txt += '        rm -f $RUNTIME_AREA/$repo \n'
1200 –        #txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1201 –        #txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1202 –        #txt += '    fi\n'
1203 –        #txt += 'fi\n'
1204 –        #txt += '\n'
1205 –
1206 –
1218          return txt
1219  
1220      def numberFile_(self, file, txt):
# Line 1234 | Line 1245 | class Cmssw(JobType):
1245                   self.version + \
1246                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1247          ## SL add requirement for OS version only if SL4
1248 <        reSL4 = re.compile( r'slc4' )
1249 <        if self.executable_arch and reSL4.search(self.executable_arch):
1248 >        #reSL4 = re.compile( r'slc4' )
1249 >        if self.executable_arch: # and reSL4.search(self.executable_arch):
1250              req+=' && Member("VO-cms-' + \
1251                   self.executable_arch + \
1252                   '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
# Line 1350 | Line 1361 | class Cmssw(JobType):
1361          """
1362  
1363          txt = ''
1353 –        txt += 'echo "Modify Job Report" \n'
1354 –        #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1355 –        ################ FEDE FOR DBS2 #############################################
1356 –        txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1357 –        #############################################################################
1364          try:
1365              publish_data = int(self.cfg_params['USER.publish_data'])          
1366          except KeyError:
1367              publish_data = 0
1368 +        if (publish_data == 1):  
1369 +            txt += 'echo "Modify Job Report" \n'
1370 +            #txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1371 +            ################ FEDE FOR DBS2 #############################################
1372 +            txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1373 +            #############################################################################
1374 +            #try:
1375 +            #    publish_data = int(self.cfg_params['USER.publish_data'])          
1376 +            #except KeyError:
1377 +            #    publish_data = 0
1378  
1379 <        txt += 'if [ -z "$SE" ]; then\n'
1380 <        txt += '    SE="" \n'
1381 <        txt += 'fi \n'
1382 <        txt += 'if [ -z "$SE_PATH" ]; then\n'
1383 <        txt += '    SE_PATH="" \n'
1384 <        txt += 'fi \n'
1385 <        txt += 'echo "SE = $SE"\n'
1386 <        txt += 'echo "SE_PATH = $SE_PATH"\n'
1379 >            txt += 'if [ -z "$SE" ]; then\n'
1380 >            txt += '    SE="" \n'
1381 >            txt += 'fi \n'
1382 >            txt += 'if [ -z "$SE_PATH" ]; then\n'
1383 >            txt += '    SE_PATH="" \n'
1384 >            txt += 'fi \n'
1385 >            txt += 'echo "SE = $SE"\n'
1386 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
1387  
1388 <        if (publish_data == 1):  
1388 >        #if (publish_data == 1):  
1389              #processedDataset = self.cfg_params['USER.processed_datasetname']
1390              processedDataset = self.cfg_params['USER.publish_data_name']
1391              txt += 'ProcessedDataset='+processedDataset+'\n'
# Line 1399 | Line 1415 | class Cmssw(JobType):
1415              txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1416              txt += 'fi\n'
1417          else:
1418 <            txt += 'ProcessedDataset=no_data_to_publish \n'
1418 >            txt += 'echo "no data publication required"\n'
1419 >            #txt += 'ProcessedDataset=no_data_to_publish \n'
1420              #### FEDE: added slash in LFN ##############
1421 <            txt += 'FOR_LFN=/local/ \n'
1422 <            txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1423 <            txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1421 >            #txt += 'FOR_LFN=/local/ \n'
1422 >            #txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1423 >            #txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1424          return txt
1425  
1426      def cleanEnv(self):
# Line 1446 | Line 1463 | class Cmssw(JobType):
1463          for e in old:
1464              nd[e]=0
1465          return nd.keys()
1466 +
1467 +
1468 +    def checkOut(self, limit):
1469 +        """
1470 +        check the dimension of the output files
1471 +        """
1472 +        txt = 'echo "*****************************************"\n'
1473 +        txt += 'echo "** Starting output sandbox limit check **"\n'
1474 +        txt += 'echo "*****************************************"\n'
1475 +        allOutFiles = ""
1476 +        listOutFiles = []
1477 +        for fileOut in (self.output_file+self.output_file_sandbox):
1478 +             if fileOut.find('crab_fjr') == -1:
1479 +                 allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1480 +                 listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1481 +        txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1482 +        txt += 'ls -gGhrta;\n'
1483 +        txt += 'sum=0;\n'
1484 +        txt += 'for file in '+str(allOutFiles)+' ; do\n'
1485 +        txt += '    if [ -e $file ]; then\n'
1486 +        txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1487 +        txt += '        sum=`expr $sum + $tt`\n'
1488 +        txt += '    else\n'
1489 +        txt += '        echo "WARNING: output file $file not found!"\n'
1490 +        txt += '    fi\n'
1491 +        txt += 'done\n'
1492 +        txt += 'echo "Total Output dimension: $sum";\n'
1493 +        txt += 'limit='+str(limit)+';\n'
1494 +        txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1495 +        txt += 'if [ $limit -lt $sum ]; then\n'
1496 +        txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1497 +        txt += '    echo "         checking the output file sizes..."\n'
1498 +        """
1499 +        txt += '    dim=0;\n'
1500 +        txt += '    exclude=0;\n'
1501 +        txt += '    for files in '+str(allOutFiles)+' ; do\n'
1502 +        txt += '        sumTemp=0;\n'
1503 +        txt += '        for file2 in '+str(allOutFiles)+' ; do\n'
1504 +        txt += '            if [ $file != $file2 ]; then\n'
1505 +        txt += '                tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1506 +        txt += '                sumTemp=`expr $sumTemp + $tt`;\n'
1507 +        txt += '            fi\n'
1508 +        txt += '        done\n'
1509 +        txt += '        if [ $sumTemp -lt $limit ]; then\n'
1510 +        txt += '            if [ $dim -lt $sumTemp ]; then\n'
1511 +        txt += '                dim=$sumTemp;\n'
1512 +        txt += '                exclude=$file;\n'
1513 +        txt += '            fi\n'
1514 +        txt += '        fi\n'
1515 +        txt += '    done\n'
1516 +        txt += '    echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1517 +        """
1518 +        txt += '    tot=0;\n'
1519 +        txt += '    for file2 in '+str(allOutFiles)+' ; do\n'
1520 +        txt += '        tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n'
1521 +        txt += '        tot=`expr $tot + $tt`;\n'
1522 +        txt += '        if [ $limit -lt $tot ]; then\n'
1523 +        txt += '            tot=`expr $tot - $tt`;\n'
1524 +        txt += '            fileLast=$file;\n'
1525 +        txt += '            break;\n'
1526 +        txt += '        fi\n'
1527 +        txt += '    done\n'
1528 +        txt += '    echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1529 +        txt += '    flag=0;\n'    
1530 +        txt += '    for filess in '+str(allOutFiles)+' ; do\n'
1531 +        txt += '        if [ $fileLast = $filess ]; then\n'
1532 +        txt += '            flag=1;\n'
1533 +        txt += '        fi\n'
1534 +        txt += '        if [ $flag -eq 1 ]; then\n'
1535 +        txt += '            rm -f $filess;\n'
1536 +        txt += '        fi\n'
1537 +        txt += '    done\n'
1538 +        txt += '    ls -agGhrt;\n'
1539 +        txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1540 +        txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1541 +        txt += '    exit_status=70000;\n'
1542 +        txt += 'else'
1543 +        txt += '    echo "Total Output dimension $sum is fine.";\n'
1544 +        txt += 'fi\n'
1545 +        txt += 'echo "*****************************************"\n'
1546 +        txt += 'echo "*** Ending output sandbox limit check ***"\n'
1547 +        txt += 'echo "*****************************************"\n'
1548 +        return txt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines