| 2 |
|
from crab_logger import Logger |
| 3 |
|
from crab_exceptions import * |
| 4 |
|
from crab_util import * |
| 5 |
+ |
from BlackWhiteListParser import BlackWhiteListParser |
| 6 |
|
import common |
| 7 |
|
import Scram |
| 8 |
|
|
| 16 |
|
self._params = {} |
| 17 |
|
self.cfg_params = cfg_params |
| 18 |
|
|
| 19 |
+ |
# init BlackWhiteListParser |
| 20 |
+ |
self.blackWhiteListParser = BlackWhiteListParser(cfg_params) |
| 21 |
+ |
|
| 22 |
|
try: |
| 23 |
|
self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize']) |
| 24 |
|
except KeyError: |
| 586 |
|
for block in blocks: |
| 587 |
|
if block in jobsOfBlock.keys() : |
| 588 |
|
blockCounter += 1 |
| 589 |
< |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block])) |
| 589 |
> |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 590 |
|
|
| 591 |
< |
# common.logger.message(screenOutput) |
| 591 |
> |
common.logger.message(screenOutput) |
| 592 |
|
|
| 593 |
|
self.list_of_args = list_of_lists |
| 594 |
|
return |
| 856 |
|
|
| 857 |
|
## OLI_Daniele at this level middleware already known |
| 858 |
|
|
| 855 |
– |
txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n' |
| 856 |
– |
txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n' |
| 857 |
– |
txt += 'export SCRAM_ARCH='+self.executable_arch+'\n' |
| 858 |
– |
txt += 'export BUILD_ARCH='+self.executable_arch+'\n' |
| 859 |
|
txt += 'if [ $middleware == LCG ]; then \n' |
| 860 |
+ |
txt += ' echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n' |
| 861 |
+ |
txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 862 |
+ |
txt += ' export BUILD_ARCH='+self.executable_arch+'\n' |
| 863 |
|
txt += self.wsSetupCMSLCGEnvironment_() |
| 864 |
|
txt += 'elif [ $middleware == OSG ]; then\n' |
| 865 |
|
txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n' |
| 878 |
|
txt += ' echo "Change to working directory: $WORKING_DIR"\n' |
| 879 |
|
txt += ' cd $WORKING_DIR\n' |
| 880 |
|
txt += self.wsSetupCMSOSGEnvironment_() |
| 881 |
+ |
txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n' |
| 882 |
+ |
txt += ' export SCRAM_ARCH='+self.executable_arch+'\n' |
| 883 |
|
txt += 'fi\n' |
| 884 |
|
|
| 885 |
|
# Prepare JobType-specific part |
| 1118 |
|
def executableArgs(self): |
| 1119 |
|
if self.scriptExe:#CarlosDaniele |
| 1120 |
|
return self.scriptExe + " $NJob" |
| 1121 |
< |
else: |
| 1122 |
< |
return " -p pset.cfg" |
| 1121 |
> |
else: |
| 1122 |
> |
# if >= CMSSW_1_5_X, add -e |
| 1123 |
> |
version_array = self.scram.getSWVersion().split('_') |
| 1124 |
> |
major = 0 |
| 1125 |
> |
minor = 0 |
| 1126 |
> |
try: |
| 1127 |
> |
major = int(version_array[1]) |
| 1128 |
> |
minor = int(version_array[2]) |
| 1129 |
> |
except: |
| 1130 |
> |
msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!" |
| 1131 |
> |
raise CrabException(msg) |
| 1132 |
> |
if major >= 1 and minor >= 5 : |
| 1133 |
> |
return " -e -p pset.cfg" |
| 1134 |
> |
else: |
| 1135 |
> |
return " -p pset.cfg" |
| 1136 |
|
|
| 1137 |
|
def inputSandbox(self, nj): |
| 1138 |
|
""" |
| 1188 |
|
# txt += 'ls '+fileWithSuffix+'\n' |
| 1189 |
|
# txt += 'ls_result=$?\n' |
| 1190 |
|
txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n' |
| 1191 |
< |
txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1191 |
> |
###### FEDE FOR OUTPUT DATA PUBLICATION ######## |
| 1192 |
> |
txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n' |
| 1193 |
> |
txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1194 |
> |
################################################ |
| 1195 |
|
txt += 'else\n' |
| 1196 |
< |
txt += ' exit_status=60302\n' |
| 1197 |
< |
txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n' |
| 1196 |
> |
txt += ' exit_status=60302\n' |
| 1197 |
> |
txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n' |
| 1198 |
> |
############# FEDE ADDED CHECK FOR OUTPUT ############# |
| 1199 |
> |
if fileWithSuffix in self.output_file: |
| 1200 |
> |
txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n' |
| 1201 |
> |
txt += ' exit $exit_status\n' |
| 1202 |
> |
####################################################### |
| 1203 |
|
if common.scheduler.boss_scheduler_name == 'condor_g': |
| 1204 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1205 |
|
txt += ' echo "prepare dummy output file"\n' |
| 1209 |
|
file_list = [] |
| 1210 |
|
for fileWithSuffix in (self.output_file): |
| 1211 |
|
file_list.append(self.numberFile_(fileWithSuffix, '$NJob')) |
| 1212 |
+ |
|
| 1213 |
|
txt += 'file_list="'+string.join(file_list,' ')+'"\n' |
| 1187 |
– |
|
| 1214 |
|
txt += 'cd $RUNTIME_AREA\n' |
| 1189 |
– |
#### FEDE this is the cleanEnv function |
| 1190 |
– |
### OLI_DANIELE |
| 1191 |
– |
#txt += 'if [ $middleware == OSG ]; then\n' |
| 1192 |
– |
#txt += ' cd $RUNTIME_AREA\n' |
| 1193 |
– |
#txt += ' echo "Remove working directory: $WORKING_DIR"\n' |
| 1194 |
– |
#txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1195 |
– |
#txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1196 |
– |
#txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n' |
| 1197 |
– |
#txt += ' echo "JOB_EXIT_STATUS = 60999"\n' |
| 1198 |
– |
#txt += ' echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n' |
| 1199 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1200 |
– |
#txt += ' rm -f $RUNTIME_AREA/$repo \n' |
| 1201 |
– |
#txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
| 1202 |
– |
#txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
| 1203 |
– |
#txt += ' fi\n' |
| 1204 |
– |
#txt += 'fi\n' |
| 1205 |
– |
#txt += '\n' |
| 1206 |
– |
|
| 1207 |
– |
|
| 1215 |
|
return txt |
| 1216 |
|
|
| 1217 |
|
def numberFile_(self, file, txt): |
