| 58 |
|
# raise CrabException(msg) |
| 59 |
|
# |
| 60 |
|
|
| 61 |
– |
self.setParam_('application', self.version) |
| 61 |
|
|
| 62 |
|
### collect Data cards |
| 63 |
|
|
| 73 |
|
self.datasetPath = tmp |
| 74 |
|
self.selectNoInput = 0 |
| 75 |
|
|
| 77 |
– |
# ML monitoring |
| 78 |
– |
# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM |
| 79 |
– |
if not self.datasetPath: |
| 80 |
– |
self.setParam_('dataset', 'None') |
| 81 |
– |
self.setParam_('owner', 'None') |
| 82 |
– |
else: |
| 83 |
– |
## SL what is supposed to fail here? |
| 84 |
– |
try: |
| 85 |
– |
datasetpath_split = self.datasetPath.split("/") |
| 86 |
– |
# standard style |
| 87 |
– |
self.setParam_('datasetFull', self.datasetPath) |
| 88 |
– |
self.setParam_('dataset', datasetpath_split[1]) |
| 89 |
– |
self.setParam_('owner', datasetpath_split[2]) |
| 90 |
– |
except: |
| 91 |
– |
self.setParam_('dataset', self.datasetPath) |
| 92 |
– |
self.setParam_('owner', self.datasetPath) |
| 93 |
– |
|
| 94 |
– |
self.setParam_('taskId', common._db.queryTask('name')) ## new BL--DS |
| 95 |
– |
|
| 76 |
|
self.dataTiers = [] |
| 77 |
|
|
| 78 |
|
## now the application |
| 79 |
|
self.executable = cfg_params.get('CMSSW.executable','cmsRun') |
| 100 |
– |
self.setParam_('exe', self.executable) |
| 80 |
|
log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable) |
| 81 |
|
|
| 82 |
|
if not cfg_params.has_key('CMSSW.pset'): |
| 113 |
|
# script_exe file as additional file in inputSandbox |
| 114 |
|
self.scriptExe = cfg_params.get('USER.script_exe',None) |
| 115 |
|
if self.scriptExe : |
| 116 |
< |
if not os.path.isfile(self.scriptExe): |
| 117 |
< |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 118 |
< |
raise CrabException(msg) |
| 119 |
< |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 116 |
> |
if not os.path.isfile(self.scriptExe): |
| 117 |
> |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 118 |
> |
raise CrabException(msg) |
| 119 |
> |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 120 |
|
|
| 121 |
|
#CarlosDaniele |
| 122 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 123 |
< |
msg ="Error. script_exe not defined" |
| 124 |
< |
raise CrabException(msg) |
| 123 |
> |
msg ="Error. script_exe not defined" |
| 124 |
> |
raise CrabException(msg) |
| 125 |
|
|
| 126 |
|
## additional input files |
| 127 |
|
if cfg_params.has_key('USER.additional_input_files'): |
| 201 |
|
## remove |
| 202 |
|
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 203 |
|
if self.sourceSeed: |
| 204 |
< |
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 205 |
< |
self.incrementSeeds.append('sourceSeed') |
| 204 |
> |
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 205 |
> |
self.incrementSeeds.append('sourceSeed') |
| 206 |
|
|
| 207 |
|
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 208 |
|
if self.sourceSeedVtx: |
| 209 |
< |
print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 210 |
< |
self.incrementSeeds.append('VtxSmeared') |
| 209 |
> |
print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 210 |
> |
self.incrementSeeds.append('VtxSmeared') |
| 211 |
|
|
| 212 |
|
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 213 |
|
if self.sourceSeedG4: |
| 214 |
< |
print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 215 |
< |
self.incrementSeeds.append('g4SimHits') |
| 214 |
> |
print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 215 |
> |
self.incrementSeeds.append('g4SimHits') |
| 216 |
|
|
| 217 |
|
self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None) |
| 218 |
|
if self.sourceSeedMix: |
| 219 |
< |
print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 220 |
< |
self.incrementSeeds.append('mix') |
| 219 |
> |
print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 220 |
> |
self.incrementSeeds.append('mix') |
| 221 |
|
|
| 222 |
|
self.firstRun = cfg_params.get('CMSSW.first_run',None) |
| 223 |
|
|
| 372 |
|
else : |
| 373 |
|
totalNumberOfJobs = self.ncjobs |
| 374 |
|
|
| 396 |
– |
|
| 375 |
|
blocks = blockSites.keys() |
| 376 |
|
blockCount = 0 |
| 377 |
|
# Backup variable in case self.maxEvents counted events in a non-included block |
| 426 |
|
except KeyError: |
| 427 |
|
common.logger.message("File "+str(file)+" has unknown number of events: skipping") |
| 428 |
|
|
| 429 |
< |
|
| 429 |
> |
eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining) |
| 430 |
|
# if less events in file remain than eventsPerJobRequested |
| 431 |
< |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) : |
| 431 |
> |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested): |
| 432 |
|
# if last file in block |
| 433 |
|
if ( fileCount == numFilesInBlock-1 ) : |
| 434 |
|
# end job using last file, use remaining events in block |
| 512 |
|
for block in blocks: |
| 513 |
|
if block in jobsOfBlock.keys() : |
| 514 |
|
blockCounter += 1 |
| 515 |
< |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 515 |
> |
screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]), |
| 516 |
> |
','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block))) |
| 517 |
|
if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0: |
| 518 |
|
noSiteBlock.append( spanRanges(jobsOfBlock[block]) ) |
| 519 |
|
bloskNoSite.append( blockCounter ) |
| 655 |
|
self.argsList = (len(jobParams[1])+1) |
| 656 |
|
|
| 657 |
|
return |
| 679 |
– |
# |
| 680 |
– |
# def getJobTypeArguments(self, nj, sched): |
| 681 |
– |
# result = '' |
| 682 |
– |
# jobs=[] |
| 683 |
– |
# jobs.append(nj) |
| 684 |
– |
# for i in common._db.queryJob('arguments',jobs):## BL--DS |
| 685 |
– |
# result=result+str(i)+" " |
| 686 |
– |
# return result |
| 658 |
|
|
| 659 |
|
def numberOfJobs(self): |
| 660 |
|
# Fabio |
| 896 |
|
txt += 'echo "PSETHASH = $PSETHASH" \n' |
| 897 |
|
txt += '\n' |
| 898 |
|
return txt |
| 899 |
+ |
|
| 900 |
|
def wsUntarSoftware(self, nj=0): |
| 901 |
|
""" |
| 902 |
|
Put in the script the commands to build an executable |
| 939 |
|
txt = '\n#Written by cms_cmssw::wsBuildExe\n' |
| 940 |
|
txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n' |
| 941 |
|
|
| 970 |
– |
<<<<<<< cms_cmssw.py |
| 971 |
– |
txt += 'mv $RUNTIME_AREA/lib . \n' |
| 972 |
– |
txt += 'mv $RUNTIME_AREA/module . \n' |
| 973 |
– |
txt += 'mv $RUNTIME_AREA/ProdCommon . \n' |
| 974 |
– |
======= |
| 942 |
|
txt += 'rm -r lib/ module/ \n' |
| 943 |
|
txt += 'mv $RUNTIME_AREA/lib/ . \n' |
| 944 |
|
txt += 'mv $RUNTIME_AREA/module/ . \n' |
| 945 |
|
txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n' |
| 946 |
|
|
| 980 |
– |
|
| 981 |
– |
#if os.path.isfile(self.tgzNameWithPath): |
| 982 |
– |
# txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n' |
| 983 |
– |
# txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n' |
| 984 |
– |
# txt += 'untar_status=$? \n' |
| 985 |
– |
# txt += 'if [ $untar_status -ne 0 ]; then \n' |
| 986 |
– |
# txt += ' echo "ERROR ==> Untarring .tgz file failed"\n' |
| 987 |
– |
# txt += ' job_exit_code=$untar_status\n' |
| 988 |
– |
# txt += ' func_exit\n' |
| 989 |
– |
# txt += 'else \n' |
| 990 |
– |
# txt += ' echo "Successful untar" \n' |
| 991 |
– |
# txt += 'fi \n' |
| 992 |
– |
# txt += '\n' |
| 993 |
– |
# txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n' |
| 994 |
– |
>>>>>>> 1.170 |
| 947 |
|
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 948 |
|
txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n' |
| 949 |
|
txt += 'else\n' |
| 1045 |
|
txt += 'ls \n' |
| 1046 |
|
txt += '\n' |
| 1047 |
|
|
| 1096 |
– |
#txt += 'output_exit_status=0\n' |
| 1097 |
– |
|
| 1098 |
– |
### FEDE ####### |
| 1099 |
– |
#for fileWithSuffix in (self.output_file_sandbox): |
| 1100 |
– |
# output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1101 |
– |
# txt += '\n' |
| 1102 |
– |
# txt += '# check output file\n' |
| 1103 |
– |
# txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n' |
| 1104 |
– |
# txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n' |
| 1105 |
– |
# txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n' |
| 1106 |
– |
# txt += 'else\n' |
| 1107 |
– |
# txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n' |
| 1108 |
– |
# txt += ' job_exit_code=60302\n' |
| 1109 |
– |
# if common.scheduler.name().upper() == 'CONDOR_G': |
| 1110 |
– |
# txt += ' if [ $middleware == OSG ]; then \n' |
| 1111 |
– |
# txt += ' echo "prepare dummy output file"\n' |
| 1112 |
– |
# txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n' |
| 1113 |
– |
# txt += ' fi \n' |
| 1114 |
– |
# txt += 'fi\n' |
| 1115 |
– |
|
| 1048 |
|
for fileWithSuffix in (self.output_file): |
| 1049 |
|
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1050 |
|
txt += '\n' |
| 1189 |
|
""" |
| 1190 |
|
|
| 1191 |
|
txt = '\n#Written by cms_cmssw::modifyReport\n' |
| 1192 |
< |
try: |
| 1261 |
< |
publish_data = int(self.cfg_params['USER.publish_data']) |
| 1262 |
< |
except KeyError: |
| 1263 |
< |
publish_data = 0 |
| 1192 |
> |
publish_data = int(self.cfg_params.get('USER.publish_data',0)) |
| 1193 |
|
if (publish_data == 1): |
| 1265 |
– |
txt += 'if [ $copy_exit_status -eq 0 ]; then\n' |
| 1266 |
– |
txt += ' echo ">>> Modify Job Report:" \n' |
| 1267 |
– |
txt += ' chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1268 |
– |
txt += ' echo "SE = $SE"\n' |
| 1269 |
– |
txt += ' echo "SE_PATH = $SE_PATH"\n' |
| 1270 |
– |
|
| 1194 |
|
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1195 |
|
LFNBaseName = LFNBase(processedDataset) |
| 1196 |
< |
txt += ' ProcessedDataset='+processedDataset+'\n' |
| 1196 |
> |
|
| 1197 |
> |
txt += 'if [ $copy_exit_status -eq 0 ]; then\n' |
| 1198 |
|
txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName) |
| 1199 |
< |
txt += ' echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1200 |
< |
txt += ' echo "FOR_LFN = $FOR_LFN" \n' |
| 1201 |
< |
txt += ' echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1202 |
< |
txt += ' echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1203 |
< |
txt += ' $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1204 |
< |
txt += ' modifyReport_result=$?\n' |
| 1205 |
< |
txt += ' if [ $modifyReport_result -ne 0 ]; then\n' |
| 1206 |
< |
txt += ' modifyReport_result=70500\n' |
| 1207 |
< |
txt += ' job_exit_code=$modifyReport_result\n' |
| 1208 |
< |
txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n' |
| 1209 |
< |
txt += ' echo "WARNING: Problem with ModifyJobReport"\n' |
| 1210 |
< |
txt += ' else\n' |
| 1211 |
< |
txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1212 |
< |
txt += ' fi\n' |
| 1199 |
> |
txt += 'else\n' |
| 1200 |
> |
txt += ' FOR_LFN=/copy_problems/ \n' |
| 1201 |
> |
txt += ' SE=""\n' |
| 1202 |
> |
txt += ' SE_PATH=""\n' |
| 1203 |
> |
txt += 'fi\n' |
| 1204 |
> |
|
| 1205 |
> |
txt += 'echo ">>> Modify Job Report:" \n' |
| 1206 |
> |
txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n' |
| 1207 |
> |
txt += 'ProcessedDataset='+processedDataset+'\n' |
| 1208 |
> |
txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1209 |
> |
txt += 'echo "SE = $SE"\n' |
| 1210 |
> |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1211 |
> |
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1212 |
> |
txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1213 |
> |
txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1214 |
> |
txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1215 |
> |
txt += 'modifyReport_result=$?\n' |
| 1216 |
> |
txt += 'if [ $modifyReport_result -ne 0 ]; then\n' |
| 1217 |
> |
txt += ' modifyReport_result=70500\n' |
| 1218 |
> |
txt += ' job_exit_code=$modifyReport_result\n' |
| 1219 |
> |
txt += ' echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n' |
| 1220 |
> |
txt += ' echo "WARNING: Problem with ModifyJobReport"\n' |
| 1221 |
> |
txt += 'else\n' |
| 1222 |
> |
txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n' |
| 1223 |
|
txt += 'fi\n' |
| 1224 |
|
return txt |
| 1225 |
|
|
| 1259 |
|
listOutFiles.append(stderr) |
| 1260 |
|
txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n' |
| 1261 |
|
txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n' |
| 1328 |
– |
<<<<<<< cms_cmssw.py |
| 1329 |
– |
txt += 'ls -gGhrta;\n' |
| 1330 |
– |
txt += 'sum=0;\n' |
| 1331 |
– |
txt += 'for file in $filesToCheck ; do\n' |
| 1332 |
– |
txt += ' if [ -e $file ]; then\n' |
| 1333 |
– |
txt += ' tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n' |
| 1334 |
– |
txt += ' sum=`expr $sum + $tt`\n' |
| 1335 |
– |
txt += ' else\n' |
| 1336 |
– |
txt += ' echo "WARNING: output file $file not found!"\n' |
| 1337 |
– |
txt += ' fi\n' |
| 1338 |
– |
txt += 'done\n' |
| 1339 |
– |
txt += 'echo "Total Output dimension: $sum";\n' |
| 1340 |
– |
txt += 'limit='+str(limit)+';\n' |
| 1341 |
– |
txt += 'echo "WARNING: output files size limit is set to: $limit";\n' |
| 1342 |
– |
txt += 'if [ $limit -lt $sum ]; then\n' |
| 1343 |
– |
txt += ' echo "WARNING: output files have to big size - something will be lost;"\n' |
| 1344 |
– |
txt += ' echo " checking the output file sizes..."\n' |
| 1345 |
– |
txt += ' tot=0;\n' |
| 1346 |
– |
txt += ' for filefile in $filesToCheck ; do\n' |
| 1347 |
– |
txt += ' dimFile=`ls -gGrta $filefile | awk \'{ print $3 }\';`\n' |
| 1348 |
– |
txt += ' tot=`expr $tot + $tt`;\n' |
| 1349 |
– |
txt += ' if [ $limit -lt $dimFile ]; then\n' |
| 1350 |
– |
txt += ' echo "deleting file: $filefile";\n' |
| 1351 |
– |
txt += ' rm -f $filefile\n' |
| 1352 |
– |
txt += ' elif [ $limit -lt $tot ]; then\n' |
| 1353 |
– |
txt += ' echo "deleting file: $filefile";\n' |
| 1354 |
– |
txt += ' rm -f $filefile\n' |
| 1355 |
– |
txt += ' else\n' |
| 1356 |
– |
txt += ' echo "saving file: $filefile"\n' |
| 1357 |
– |
txt += ' fi\n' |
| 1358 |
– |
txt += ' done\n' |
| 1359 |
– |
|
| 1360 |
– |
txt += ' ls -agGhrt\n' |
| 1361 |
– |
txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox."\n' |
| 1362 |
– |
txt += ' job_exit_code=70000\n' |
| 1363 |
– |
txt += 'else\n' |
| 1364 |
– |
txt += ' echo "Total Output dimension $sum is fine."\n' |
| 1365 |
– |
txt += 'fi\n' |
| 1366 |
– |
txt += 'echo "Ending output sandbox limit check"\n' |
| 1367 |
– |
return txt |
| 1368 |
– |
======= |
| 1262 |
|
txt += 'export filesToCheck\n' |
| 1370 |
– |
<<<<<<< cms_cmssw.py |
| 1371 |
– |
return txt |
| 1372 |
– |
>>>>>>> 1.169 |
| 1373 |
– |
======= |
| 1263 |
|
return txt |
| 1375 |
– |
>>>>>>> 1.170 |
