| 4 |
|
from crab_util import * |
| 5 |
|
import common |
| 6 |
|
import PsetManipulator |
| 7 |
< |
import DataDiscovery |
| 8 |
< |
import DataDiscovery_DBS2 |
| 9 |
< |
import DataLocation |
| 7 |
> |
# import DataDiscovery |
| 8 |
> |
# import DataDiscovery_DBS2 |
| 9 |
> |
# import DataLocation |
| 10 |
|
import Scram |
| 11 |
|
|
| 12 |
|
import os, string, re, shutil, glob |
| 23 |
|
try: |
| 24 |
|
self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize']) |
| 25 |
|
except KeyError: |
| 26 |
< |
self.MaxTarBallSize = 100.0 |
| 26 |
> |
self.MaxTarBallSize = 9.5 |
| 27 |
|
|
| 28 |
|
# number of jobs requested to be created, limit obj splitting |
| 29 |
|
self.ncjobs = ncjobs |
| 51 |
|
|
| 52 |
|
## get DBS mode |
| 53 |
|
try: |
| 54 |
< |
self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2']) |
| 54 |
> |
self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1']) |
| 55 |
|
except KeyError: |
| 56 |
< |
self.use_dbs_2 = 0 |
| 56 |
> |
self.use_dbs_1 = 0 |
| 57 |
|
|
| 58 |
|
try: |
| 59 |
|
tmp = cfg_params['CMSSW.datasetpath'] |
| 75 |
|
self.setParam_('owner', 'None') |
| 76 |
|
else: |
| 77 |
|
datasetpath_split = self.datasetPath.split("/") |
| 78 |
< |
self.setParam_('dataset', datasetpath_split[1]) |
| 79 |
< |
self.setParam_('owner', datasetpath_split[-1]) |
| 80 |
< |
|
| 78 |
> |
if self.use_dbs_1 == 1 : |
| 79 |
> |
self.setParam_('dataset', datasetpath_split[1]) |
| 80 |
> |
self.setParam_('owner', datasetpath_split[-1]) |
| 81 |
> |
else: |
| 82 |
> |
self.setParam_('dataset', datasetpath_split[1]) |
| 83 |
> |
self.setParam_('owner', datasetpath_split[2]) |
| 84 |
|
self.setTaskid_() |
| 85 |
|
self.setParam_('taskId', self.cfg_params['taskId']) |
| 86 |
|
|
| 160 |
|
tmp = string.strip(tmp) |
| 161 |
|
dirname = '' |
| 162 |
|
if not tmp[0]=="/": dirname = "." |
| 163 |
< |
files = glob.glob(os.path.join(dirname, tmp)) |
| 163 |
> |
files = [] |
| 164 |
> |
if string.find(tmp,"*")>-1: |
| 165 |
> |
files = glob.glob(os.path.join(dirname, tmp)) |
| 166 |
> |
if len(files)==0: |
| 167 |
> |
raise CrabException("No additional input file found with this pattern: "+tmp) |
| 168 |
> |
else: |
| 169 |
> |
files.append(tmp) |
| 170 |
|
for file in files: |
| 171 |
|
if not os.path.exists(file): |
| 172 |
|
raise CrabException("Additional input file not found: "+file) |
| 173 |
|
pass |
| 174 |
< |
storedFile = common.work_space.shareDir()+file |
| 174 |
> |
fname = string.split(file, '/')[-1] |
| 175 |
> |
storedFile = common.work_space.pathForTgz()+'share/'+fname |
| 176 |
|
shutil.copyfile(file, storedFile) |
| 177 |
|
self.additional_inbox_files.append(string.strip(storedFile)) |
| 178 |
|
pass |
| 288 |
|
|
| 289 |
|
def DataDiscoveryAndLocation(self, cfg_params): |
| 290 |
|
|
| 291 |
+ |
import DataDiscovery |
| 292 |
+ |
import DataDiscovery_DBS2 |
| 293 |
+ |
import DataLocation |
| 294 |
|
common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()") |
| 295 |
|
|
| 296 |
|
datasetPath=self.datasetPath |
| 299 |
|
common.logger.message("Contacting DBS...") |
| 300 |
|
try: |
| 301 |
|
|
| 302 |
< |
if self.use_dbs_2 == 1 : |
| 290 |
< |
self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params) |
| 291 |
< |
else : |
| 302 |
> |
if self.use_dbs_1 == 1 : |
| 303 |
|
self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params) |
| 304 |
+ |
else : |
| 305 |
+ |
self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params) |
| 306 |
|
self.pubdata.fetchDBSInfo() |
| 307 |
|
|
| 308 |
|
except DataDiscovery.NotExistingDatasetError, ex : |
| 700 |
|
try: # create tar ball |
| 701 |
|
tar = tarfile.open(self.tgzNameWithPath, "w:gz") |
| 702 |
|
## First find the executable |
| 703 |
< |
if (executable != ''): |
| 703 |
> |
if (self.executable != ''): |
| 704 |
|
exeWithPath = self.scram.findFile_(executable) |
| 705 |
|
if ( not exeWithPath ): |
| 706 |
|
raise CrabException('User executable '+executable+' not found') |
| 710 |
|
# the exe is private, so we must ship |
| 711 |
|
common.logger.debug(5,"Exe "+exeWithPath+" to be tarred") |
| 712 |
|
path = swArea+'/' |
| 713 |
< |
exe = string.replace(exeWithPath, path,'') |
| 714 |
< |
tar.add(path+exe,executable) |
| 713 |
> |
# distinguish case when script is in user project area or given by full path somewhere else |
| 714 |
> |
if exeWithPath.find(path) >= 0 : |
| 715 |
> |
exe = string.replace(exeWithPath, path,'') |
| 716 |
> |
tar.add(path+exe,os.path.basename(executable)) |
| 717 |
> |
else : |
| 718 |
> |
tar.add(exeWithPath,os.path.basename(executable)) |
| 719 |
|
pass |
| 720 |
|
else: |
| 721 |
|
# the exe is from release, we'll find it on WN |
| 806 |
|
scram = self.scram.commandName() |
| 807 |
|
txt += '\n\n' |
| 808 |
|
txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n' |
| 809 |
+ |
txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n' |
| 810 |
+ |
txt += 'export SCRAM_ARCH='+self.executable_arch+'\n' |
| 811 |
|
txt += scram+' project CMSSW '+self.version+'\n' |
| 812 |
|
txt += 'status=$?\n' |
| 813 |
|
txt += 'if [ $status != 0 ] ; then\n' |
| 836 |
|
txt += ' exit 1 \n' |
| 837 |
|
txt += 'fi \n' |
| 838 |
|
txt += 'echo "CMSSW_VERSION = '+self.version+'"\n' |
| 820 |
– |
txt += 'export SCRAM_ARCH='+self.executable_arch+'\n' |
| 839 |
|
txt += 'cd '+self.version+'\n' |
| 840 |
|
### needed grep for bug in scramv1 ### |
| 841 |
|
txt += scram+' runtime -sh\n' |
