| 2 |
|
from crab_logger import Logger |
| 3 |
|
from crab_exceptions import * |
| 4 |
|
from crab_util import * |
| 5 |
< |
from BlackWhiteListParser import BlackWhiteListParser |
| 5 |
> |
from BlackWhiteListParser import SEBlackWhiteListParser |
| 6 |
|
import common |
| 7 |
|
import Scram |
| 8 |
|
from LFNBaseName import * |
| 20 |
|
self._params = {} |
| 21 |
|
self.cfg_params = cfg_params |
| 22 |
|
# init BlackWhiteListParser |
| 23 |
< |
self.blackWhiteListParser = BlackWhiteListParser(cfg_params) |
| 23 |
> |
self.blackWhiteListParser = SEBlackWhiteListParser(cfg_params) |
| 24 |
|
|
| 25 |
|
self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5)) |
| 26 |
|
|
| 197 |
|
tmp.strip() |
| 198 |
|
self.incrementSeeds.append(tmp) |
| 199 |
|
|
| 200 |
< |
## Old method of dealing with seeds |
| 201 |
< |
## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then |
| 202 |
< |
## remove |
| 203 |
< |
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 204 |
< |
if self.sourceSeed: |
| 205 |
< |
print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 206 |
< |
self.incrementSeeds.append('sourceSeed') |
| 207 |
< |
self.incrementSeeds.append('theSource') |
| 208 |
< |
|
| 200 |
> |
## FUTURE: Can remove in CRAB 2.4.0 |
| 201 |
> |
self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None) |
| 202 |
|
self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None) |
| 203 |
< |
if self.sourceSeedVtx: |
| 211 |
< |
print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 212 |
< |
self.incrementSeeds.append('VtxSmeared') |
| 213 |
< |
|
| 214 |
< |
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 215 |
< |
if self.sourceSeedG4: |
| 216 |
< |
print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 217 |
< |
self.incrementSeeds.append('g4SimHits') |
| 218 |
< |
|
| 203 |
> |
self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None) |
| 204 |
|
self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None) |
| 205 |
< |
if self.sourceSeedMix: |
| 206 |
< |
print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds." |
| 207 |
< |
self.incrementSeeds.append('mix') |
| 205 |
> |
if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix: |
| 206 |
> |
msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds' |
| 207 |
> |
raise CrabException(msg) |
| 208 |
|
|
| 209 |
|
self.firstRun = cfg_params.get('CMSSW.first_run',None) |
| 210 |
|
|
| 226 |
– |
|
| 211 |
|
# Copy/return |
| 212 |
|
self.copy_data = int(cfg_params.get('USER.copy_data',0)) |
| 213 |
|
self.return_data = int(cfg_params.get('USER.return_data',0)) |
| 876 |
|
txt += 'DatasetPath='+self.datasetPath+'\n' |
| 877 |
|
|
| 878 |
|
datasetpath_split = self.datasetPath.split("/") |
| 879 |
< |
|
| 879 |
> |
### FEDE FOR NEW LFN ### |
| 880 |
> |
self.primaryDataset = datasetpath_split[1] |
| 881 |
> |
######################## |
| 882 |
|
txt += 'PrimaryDataset='+datasetpath_split[1]+'\n' |
| 883 |
|
txt += 'DataTier='+datasetpath_split[2]+'\n' |
| 884 |
|
txt += 'ApplicationFamily=cmsRun\n' |
| 885 |
|
|
| 886 |
|
else: |
| 887 |
|
txt += 'DatasetPath=MCDataTier\n' |
| 888 |
+ |
### FEDE FOR NEW LFN ### |
| 889 |
+ |
self.primaryDataset = 'null' |
| 890 |
+ |
######################## |
| 891 |
|
txt += 'PrimaryDataset=null\n' |
| 892 |
|
txt += 'DataTier=null\n' |
| 893 |
|
txt += 'ApplicationFamily=MCDataTier\n' |
| 1113 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1114 |
|
|
| 1115 |
|
req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)' |
| 1116 |
< |
if common.scheduler.name() == "glitecoll": |
| 1116 |
> |
if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"): |
| 1117 |
|
req += ' && other.GlueCEStateStatus == "Production" ' |
| 1118 |
|
|
| 1119 |
|
return req |
| 1192 |
|
publish_data = int(self.cfg_params.get('USER.publish_data',0)) |
| 1193 |
|
if (publish_data == 1): |
| 1194 |
|
processedDataset = self.cfg_params['USER.publish_data_name'] |
| 1195 |
< |
### FEDE for publication with LSF and CAF schedulers #### |
| 1196 |
< |
print "common.scheduler.name().upper() = ", common.scheduler.name().upper() |
| 1195 |
> |
if (self.primaryDataset == 'null'): |
| 1196 |
> |
self.primaryDataset = processedDataset |
| 1197 |
|
if (common.scheduler.name().upper() == "CAF" or common.scheduler.name().upper() == "LSF"): |
| 1198 |
< |
print "chiamo LFNBaseName con localUser = true" |
| 1199 |
< |
LFNBaseName = LFNBase(processedDataset, LocalUser=True) |
| 1200 |
< |
else : |
| 1201 |
< |
LFNBaseName = LFNBase(processedDataset) |
| 1202 |
< |
#### |
| 1198 |
> |
### FEDE FOR NEW LFN ### |
| 1199 |
> |
LFNBaseName = LFNBase(self.primaryDataset, processedDataset, LocalUser=True) |
| 1200 |
> |
self.user = getUserName(LocalUser=True) |
| 1201 |
> |
######################## |
| 1202 |
> |
else : |
| 1203 |
> |
### FEDE FOR NEW LFN ### |
| 1204 |
> |
LFNBaseName = LFNBase(self.primaryDataset, processedDataset) |
| 1205 |
> |
self.user = getUserName() |
| 1206 |
> |
######################## |
| 1207 |
|
|
| 1208 |
|
txt += 'if [ $copy_exit_status -eq 0 ]; then\n' |
| 1209 |
< |
txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName) |
| 1209 |
> |
### FEDE FOR NEW LFN ### |
| 1210 |
> |
#txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName) |
| 1211 |
> |
txt += ' FOR_LFN=%s/${PSETHASH}/\n'%(LFNBaseName) |
| 1212 |
> |
######################## |
| 1213 |
|
txt += 'else\n' |
| 1214 |
|
txt += ' FOR_LFN=/copy_problems/ \n' |
| 1215 |
|
txt += ' SE=""\n' |
| 1224 |
|
txt += 'echo "SE_PATH = $SE_PATH"\n' |
| 1225 |
|
txt += 'echo "FOR_LFN = $FOR_LFN" \n' |
| 1226 |
|
txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n' |
| 1227 |
< |
txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1228 |
< |
txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1227 |
> |
### FEDE FOR NEW LFN ### |
| 1228 |
> |
txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n' |
| 1229 |
> |
txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n' |
| 1230 |
> |
######################## |
| 1231 |
|
txt += 'modifyReport_result=$?\n' |
| 1232 |
|
txt += 'if [ $modifyReport_result -ne 0 ]; then\n' |
| 1233 |
|
txt += ' modifyReport_result=70500\n' |
| 1265 |
|
txt += ' fi\n' |
| 1266 |
|
#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap |
| 1267 |
|
|
| 1268 |
< |
if (self.datasetPath and not self.dataset_pu ): |
| 1268 |
> |
if (self.datasetPath and not (self.dataset_pu or self.useParent) : |
| 1269 |
|
# VERIFY PROCESSED DATA |
| 1270 |
|
txt += ' if [ $executable_exit_status -eq 0 ];then\n' |
| 1271 |
|
txt += ' echo ">>> Verify list of processed files:"\n' |
