| 91 |
|
try: |
| 92 |
|
datasetpath_split = self.datasetPath.split("/") |
| 93 |
|
# standard style |
| 94 |
– |
self.setParam_('datasetFull', self.datasetPath) |
| 94 |
|
if self.use_dbs_1 == 1 : |
| 95 |
|
self.setParam_('dataset', datasetpath_split[1]) |
| 96 |
|
self.setParam_('owner', datasetpath_split[-1]) |
| 318 |
|
if (self.sourceSeedVtx) : |
| 319 |
|
PsetEdit.vtxSeed("INPUTVTX") |
| 320 |
|
if (self.sourceSeedG4) : |
| 321 |
< |
PsetEdit.g4Seed("INPUTG4") |
| 321 |
> |
self.PsetEdit.g4Seed("INPUTG4") |
| 322 |
|
if (self.sourceSeedMix) : |
| 323 |
< |
PsetEdit.mixSeed("INPUTMIX") |
| 323 |
> |
self.PsetEdit.mixSeed("INPUTMIX") |
| 324 |
|
# add FrameworkJobReport to parameter-set |
| 325 |
|
PsetEdit.addCrabFJR(self.fjrFileName) |
| 326 |
|
PsetEdit.psetWriter(self.configFilename()) |
| 1196 |
|
txt += ' exit_status=60302\n' |
| 1197 |
|
txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n' |
| 1198 |
|
############# FEDE ADDED CHECK FOR OUTPUT ############# |
| 1200 |
– |
## MATTY's FIX: the exit option was interrupting the execution |
| 1199 |
|
if fileWithSuffix in self.output_file: |
| 1200 |
|
txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n' |
| 1201 |
< |
txt += ' output_exit_status=$exit_status\n' |
| 1204 |
< |
txt += ' # exit $exit_status\n' |
| 1201 |
> |
txt += ' exit $exit_status\n' |
| 1202 |
|
####################################################### |
| 1203 |
|
if common.scheduler.boss_scheduler_name == 'condor_g': |
| 1204 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1358 |
|
""" |
| 1359 |
|
|
| 1360 |
|
txt = '' |
| 1361 |
+ |
txt += 'echo "Modify Job Report" \n' |
| 1362 |
+ |
#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n' |
| 1363 |
+ |
################ FEDE FOR DBS2 ############################################# |
| 1364 |
+ |
txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n' |
| 1365 |
+ |
############################################################################# |
| 1366 |
|
try: |
| 1367 |
|
publish_data = int(self.cfg_params['USER.publish_data']) |
| 1368 |
|
except KeyError: |
| 1369 |
|
publish_data = 0 |
| 1368 |
– |
if (publish_data == 1): |
| 1369 |
– |
txt += 'echo "Modify Job Report" \n' |
| 1370 |
– |
#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n' |
| 1371 |
– |
################ FEDE FOR DBS2 ############################################# |
| 1372 |
– |
txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n' |
| 1373 |
– |
############################################################################# |
| 1374 |
– |
#try: |
| 1375 |
– |
# publish_data = int(self.cfg_params['USER.publish_data']) |
| 1376 |
– |
#except KeyError: |
| 1377 |
– |
# publish_data = 0 |
| 1370 |
|
|
| 1371 |
< |
txt += 'if [ -z "$SE" ]; then\n' |
| 1372 |
< |
txt += ' SE="" \n' |
| 1373 |
< |
txt += 'fi \n' |
| 1374 |
< |
txt += 'if [ -z "$SE_PATH" ]; then\n' |
| 1375 |
< |
txt += ' SE_PATH="" \n' |
| 1376 |
< |
txt += 'fi \n' |
| 1377 |
< |
txt += 'echo "SE = $SE"\n' |
| 1378 |
< |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1371 |
> |
txt += 'if [ -z "$SE" ]; then\n' |
| 1372 |
> |
txt += ' SE="" \n' |
| 1373 |
> |
txt += 'fi \n' |
| 1374 |
> |
txt += 'if [ -z "$SE_PATH" ]; then\n' |
| 1375 |
> |
txt += ' SE_PATH="" \n' |
| 1376 |
> |
txt += 'fi \n' |
| 1377 |
> |
txt += 'echo "SE = $SE"\n' |
| 1378 |
> |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1379 |
|
|
| 1380 |
< |
#if (publish_data == 1): |
| 1380 |
> |
if (publish_data == 1): |
| 1381 |
|
#processedDataset = self.cfg_params['USER.processed_datasetname'] |
| 1382 |
|
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1383 |
|
txt += 'ProcessedDataset='+processedDataset+'\n' |
| 1407 |
|
txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n' |
| 1408 |
|
txt += 'fi\n' |
| 1409 |
|
else: |
| 1410 |
< |
txt += 'echo "no data publication required"\n' |
| 1419 |
< |
#txt += 'ProcessedDataset=no_data_to_publish \n' |
| 1410 |
> |
txt += 'ProcessedDataset=no_data_to_publish \n' |
| 1411 |
|
#### FEDE: added slash in LFN ############## |
| 1412 |
< |
#txt += 'FOR_LFN=/local/ \n' |
| 1413 |
< |
#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1414 |
< |
#txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1412 |
> |
txt += 'FOR_LFN=/local/ \n' |
| 1413 |
> |
txt += 'echo "ProcessedDataset = $ProcessedDataset"\n' |
| 1414 |
> |
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1415 |
|
return txt |
| 1416 |
|
|
| 1417 |
|
def cleanEnv(self): |
| 1454 |
|
for e in old: |
| 1455 |
|
nd[e]=0 |
| 1456 |
|
return nd.keys() |
| 1466 |
– |
|
| 1467 |
– |
|
| 1468 |
– |
def checkOut(self, limit): |
| 1469 |
– |
""" |
| 1470 |
– |
check the dimension of the output files |
| 1471 |
– |
""" |
| 1472 |
– |
txt = 'echo "*****************************************"\n' |
| 1473 |
– |
txt += 'echo "** Starting output sandbox limit check **"\n' |
| 1474 |
– |
txt += 'echo "*****************************************"\n' |
| 1475 |
– |
allOutFiles = "" |
| 1476 |
– |
listOutFiles = [] |
| 1477 |
– |
for fileOut in (self.output_file+self.output_file_sandbox): |
| 1478 |
– |
if fileOut.find('crab_fjr') == -1: |
| 1479 |
– |
allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') |
| 1480 |
– |
listOutFiles.append(self.numberFile_(fileOut, '$NJob')) |
| 1481 |
– |
txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n' |
| 1482 |
– |
txt += 'ls -gGhrta;\n' |
| 1483 |
– |
txt += 'sum=0;\n' |
| 1484 |
– |
txt += 'for file in '+str(allOutFiles)+' ; do\n' |
| 1485 |
– |
txt += ' if [ -e $file ]; then\n' |
| 1486 |
– |
txt += ' tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n' |
| 1487 |
– |
txt += ' sum=`expr $sum + $tt`\n' |
| 1488 |
– |
txt += ' else\n' |
| 1489 |
– |
txt += ' echo "WARNING: output file $file not found!"\n' |
| 1490 |
– |
txt += ' fi\n' |
| 1491 |
– |
txt += 'done\n' |
| 1492 |
– |
txt += 'echo "Total Output dimension: $sum";\n' |
| 1493 |
– |
txt += 'limit='+str(limit)+';\n' |
| 1494 |
– |
txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n' |
| 1495 |
– |
txt += 'if [ $limit -lt $sum ]; then\n' |
| 1496 |
– |
txt += ' echo "WARNING: output files have to big size - something will be lost;"\n' |
| 1497 |
– |
txt += ' echo " checking the output file sizes..."\n' |
| 1498 |
– |
""" |
| 1499 |
– |
txt += ' dim=0;\n' |
| 1500 |
– |
txt += ' exclude=0;\n' |
| 1501 |
– |
txt += ' for files in '+str(allOutFiles)+' ; do\n' |
| 1502 |
– |
txt += ' sumTemp=0;\n' |
| 1503 |
– |
txt += ' for file2 in '+str(allOutFiles)+' ; do\n' |
| 1504 |
– |
txt += ' if [ $file != $file2 ]; then\n' |
| 1505 |
– |
txt += ' tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n' |
| 1506 |
– |
txt += ' sumTemp=`expr $sumTemp + $tt`;\n' |
| 1507 |
– |
txt += ' fi\n' |
| 1508 |
– |
txt += ' done\n' |
| 1509 |
– |
txt += ' if [ $sumTemp -lt $limit ]; then\n' |
| 1510 |
– |
txt += ' if [ $dim -lt $sumTemp ]; then\n' |
| 1511 |
– |
txt += ' dim=$sumTemp;\n' |
| 1512 |
– |
txt += ' exclude=$file;\n' |
| 1513 |
– |
txt += ' fi\n' |
| 1514 |
– |
txt += ' fi\n' |
| 1515 |
– |
txt += ' done\n' |
| 1516 |
– |
txt += ' echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n' |
| 1517 |
– |
""" |
| 1518 |
– |
txt += ' tot=0;\n' |
| 1519 |
– |
txt += ' for file2 in '+str(allOutFiles)+' ; do\n' |
| 1520 |
– |
txt += ' tt=`ls -gGrta $file2 | awk \'{ print $3 }\';`\n' |
| 1521 |
– |
txt += ' tot=`expr $tot + $tt`;\n' |
| 1522 |
– |
txt += ' if [ $limit -lt $tot ]; then\n' |
| 1523 |
– |
txt += ' tot=`expr $tot - $tt`;\n' |
| 1524 |
– |
txt += ' fileLast=$file;\n' |
| 1525 |
– |
txt += ' break;\n' |
| 1526 |
– |
txt += ' fi\n' |
| 1527 |
– |
txt += ' done\n' |
| 1528 |
– |
txt += ' echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n' |
| 1529 |
– |
txt += ' flag=0;\n' |
| 1530 |
– |
txt += ' for filess in '+str(allOutFiles)+' ; do\n' |
| 1531 |
– |
txt += ' if [ $fileLast = $filess ]; then\n' |
| 1532 |
– |
txt += ' flag=1;\n' |
| 1533 |
– |
txt += ' fi\n' |
| 1534 |
– |
txt += ' if [ $flag -eq 1 ]; then\n' |
| 1535 |
– |
txt += ' rm -f $filess;\n' |
| 1536 |
– |
txt += ' fi\n' |
| 1537 |
– |
txt += ' done\n' |
| 1538 |
– |
txt += ' ls -agGhrt;\n' |
| 1539 |
– |
txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n' |
| 1540 |
– |
txt += ' echo "JOB_EXIT_STATUS = 70000";\n' |
| 1541 |
– |
txt += ' exit_status=70000;\n' |
| 1542 |
– |
txt += 'else' |
| 1543 |
– |
txt += ' echo "Total Output dimension $sum is fine.";\n' |
| 1544 |
– |
txt += 'fi\n' |
| 1545 |
– |
txt += 'echo "*****************************************"\n' |
| 1546 |
– |
txt += 'echo "*** Ending output sandbox limit check ***"\n' |
| 1547 |
– |
txt += 'echo "*****************************************"\n' |
| 1548 |
– |
return txt |
