| 4 |
|
from crab_util import * |
| 5 |
|
import common |
| 6 |
|
import PsetManipulator |
| 7 |
– |
|
| 7 |
|
import DataDiscovery |
| 8 |
|
import DataDiscovery_DBS2 |
| 9 |
|
import DataLocation |
| 10 |
|
import Scram |
| 11 |
|
|
| 12 |
< |
import os, string, re, shutil |
| 12 |
> |
import os, string, re, shutil, glob |
| 13 |
|
|
| 14 |
|
class Cmssw(JobType): |
| 15 |
|
def __init__(self, cfg_params, ncjobs): |
| 20 |
|
self._params = {} |
| 21 |
|
self.cfg_params = cfg_params |
| 22 |
|
|
| 23 |
+ |
try: |
| 24 |
+ |
self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize']) |
| 25 |
+ |
except KeyError: |
| 26 |
+ |
self.MaxTarBallSize = 100.0 |
| 27 |
+ |
|
| 28 |
|
# number of jobs requested to be created, limit obj splitting |
| 29 |
|
self.ncjobs = ncjobs |
| 30 |
|
|
| 34 |
|
self.additional_inbox_files = [] |
| 35 |
|
self.scriptExe = '' |
| 36 |
|
self.executable = '' |
| 37 |
+ |
self.executable_arch = self.scram.getArch() |
| 38 |
|
self.tgz_name = 'default.tgz' |
| 39 |
|
self.scriptName = 'CMSSW.sh' |
| 40 |
|
self.pset = '' #scrip use case Da |
| 50 |
|
### collect Data cards |
| 51 |
|
|
| 52 |
|
## get DBS mode |
| 48 |
– |
self.use_dbs_2 = 0 |
| 53 |
|
try: |
| 54 |
|
self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2']) |
| 55 |
|
except KeyError: |
| 144 |
|
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 145 |
|
except KeyError: |
| 146 |
|
self.scriptExe = '' |
| 147 |
+ |
|
| 148 |
|
#CarlosDaniele |
| 149 |
|
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 150 |
< |
msg ="WARNING. script_exe not defined" |
| 150 |
> |
msg ="Error. script_exe not defined" |
| 151 |
|
raise CrabException(msg) |
| 152 |
|
|
| 153 |
|
## additional input files |
| 154 |
|
try: |
| 155 |
|
tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',') |
| 156 |
< |
common.logger.debug(5,"Additional input files: "+str(tmpAddFiles)) |
| 157 |
< |
for tmpFile in tmpAddFiles: |
| 158 |
< |
tmpFile = string.strip(tmpFile) |
| 159 |
< |
if not os.path.exists(tmpFile): |
| 160 |
< |
raise CrabException("Additional input file not found: "+tmpFile) |
| 156 |
> |
for tmp in tmpAddFiles: |
| 157 |
> |
tmp = string.strip(tmp) |
| 158 |
> |
dirname = '' |
| 159 |
> |
if not tmp[0]=="/": dirname = "." |
| 160 |
> |
files = glob.glob(os.path.join(dirname, tmp)) |
| 161 |
> |
for file in files: |
| 162 |
> |
if not os.path.exists(file): |
| 163 |
> |
raise CrabException("Additional input file not found: "+file) |
| 164 |
|
pass |
| 165 |
< |
storedFile = common.work_space.shareDir()+ tmpFile |
| 166 |
< |
shutil.copyfile(tmpFile, storedFile) |
| 167 |
< |
self.additional_inbox_files.append(string.strip(storedFile)) |
| 165 |
> |
storedFile = common.work_space.shareDir()+file |
| 166 |
> |
shutil.copyfile(file, storedFile) |
| 167 |
> |
self.additional_inbox_files.append(string.strip(storedFile)) |
| 168 |
|
pass |
| 161 |
– |
common.logger.debug(5,"Inbox files so far : "+str(self.additional_inbox_files)) |
| 169 |
|
pass |
| 170 |
+ |
common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files)) |
| 171 |
|
except KeyError: |
| 172 |
|
pass |
| 173 |
|
|
| 409 |
|
block = blocks[blockCount] |
| 410 |
|
blockCount += 1 |
| 411 |
|
|
| 404 |
– |
|
| 412 |
|
if self.eventsbyblock.has_key(block) : |
| 413 |
|
numEventsInBlock = self.eventsbyblock[block] |
| 414 |
|
common.logger.debug(5,'Events in Block File '+str(numEventsInBlock)) |
| 687 |
|
try: # create tar ball |
| 688 |
|
tar = tarfile.open(self.tgzNameWithPath, "w:gz") |
| 689 |
|
## First find the executable |
| 690 |
< |
if (self.executable != ''): |
| 690 |
> |
if (executable != ''): |
| 691 |
|
exeWithPath = self.scram.findFile_(executable) |
| 692 |
|
if ( not exeWithPath ): |
| 693 |
|
raise CrabException('User executable '+executable+' not found') |
| 698 |
|
common.logger.debug(5,"Exe "+exeWithPath+" to be tarred") |
| 699 |
|
path = swArea+'/' |
| 700 |
|
exe = string.replace(exeWithPath, path,'') |
| 701 |
< |
tar.add(path+exe,exe) |
| 701 |
> |
tar.add(path+exe,executable) |
| 702 |
|
pass |
| 703 |
|
else: |
| 704 |
|
# the exe is from release, we'll find it on WN |
| 734 |
|
tar.close() |
| 735 |
|
except : |
| 736 |
|
raise CrabException('Could not create tar-ball') |
| 737 |
< |
|
| 737 |
> |
|
| 738 |
> |
## check for tarball size |
| 739 |
> |
tarballinfo = os.stat(self.tgzNameWithPath) |
| 740 |
> |
if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) : |
| 741 |
> |
raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.') |
| 742 |
> |
|
| 743 |
|
## create tar-ball with ML stuff |
| 744 |
|
self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz' |
| 745 |
|
try: |
| 817 |
|
txt += ' exit 1 \n' |
| 818 |
|
txt += 'fi \n' |
| 819 |
|
txt += 'echo "CMSSW_VERSION = '+self.version+'"\n' |
| 820 |
+ |
txt += 'export SCRAM_ARCH='+self.executable_arch+'\n' |
| 821 |
|
txt += 'cd '+self.version+'\n' |
| 822 |
|
### needed grep for bug in scramv1 ### |
| 823 |
|
txt += scram+' runtime -sh\n' |
| 976 |
|
""" |
| 977 |
|
|
| 978 |
|
def executableName(self): |
| 979 |
< |
if self.pset == None: #CarlosDaniele |
| 979 |
> |
if self.scriptExe: #CarlosDaniele |
| 980 |
|
return "sh " |
| 981 |
|
else: |
| 982 |
|
return self.executable |
| 983 |
|
|
| 984 |
|
def executableArgs(self): |
| 985 |
< |
if self.pset == None:#CarlosDaniele |
| 985 |
> |
if self.scriptExe:#CarlosDaniele |
| 986 |
|
return self.scriptExe + " $NJob" |
| 987 |
|
else: |
| 988 |
|
return " -p pset.cfg" |
| 1000 |
|
if os.path.isfile(self.MLtgzfile): |
| 1001 |
|
inp_box.append(self.MLtgzfile) |
| 1002 |
|
## config |
| 1003 |
< |
if not self.pset is None: #CarlosDaniele |
| 1003 |
> |
if not self.pset is None: |
| 1004 |
|
inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename()) |
| 1005 |
|
## additional input files |
| 1006 |
< |
#for file in self.additional_inbox_files: |
| 1007 |
< |
# inp_box.append(common.work_space.cwdDir()+file) |
| 1006 |
> |
for file in self.additional_inbox_files: |
| 1007 |
> |
inp_box.append(file) |
| 1008 |
|
return inp_box |
| 1009 |
|
|
| 1010 |
|
def outputSandbox(self, nj): |
| 1205 |
|
txt += ' fi\n' |
| 1206 |
|
txt += ' fi\n' |
| 1207 |
|
txt += ' \n' |
| 1195 |
– |
txt += ' string=`cat /etc/redhat-release`\n' |
| 1196 |
– |
txt += ' echo $string\n' |
| 1197 |
– |
txt += ' if [[ $string = *alhalla* ]]; then\n' |
| 1198 |
– |
txt += ' echo "SCRAM_ARCH= $SCRAM_ARCH"\n' |
| 1199 |
– |
txt += ' elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n' |
| 1200 |
– |
txt += ' export SCRAM_ARCH=slc3_ia32_gcc323\n' |
| 1201 |
– |
txt += ' echo "SCRAM_ARCH= $SCRAM_ARCH"\n' |
| 1202 |
– |
txt += ' else\n' |
| 1203 |
– |
txt += ' echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n' |
| 1204 |
– |
txt += ' echo "JOB_EXIT_STATUS = 10033"\n' |
| 1205 |
– |
txt += ' echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n' |
| 1206 |
– |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1207 |
– |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
| 1208 |
– |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
| 1209 |
– |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
| 1210 |
– |
txt += ' exit 1\n' |
| 1211 |
– |
txt += ' fi\n' |
| 1208 |
|
txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n' |
| 1209 |
|
txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n' |
| 1210 |
|
return txt |
