| 2 |
|
from crab_logger import Logger |
| 3 |
|
from crab_exceptions import * |
| 4 |
|
from crab_util import * |
| 5 |
– |
import math |
| 5 |
|
import common |
| 6 |
|
import PsetManipulator |
| 7 |
< |
|
| 8 |
< |
import DBSInfo_EDM |
| 9 |
< |
import DataDiscovery_EDM |
| 11 |
< |
import DataLocation_EDM |
| 7 |
> |
import DataDiscovery |
| 8 |
> |
import DataDiscovery_DBS2 |
| 9 |
> |
import DataLocation |
| 10 |
|
import Scram |
| 11 |
|
|
| 12 |
< |
import os, string, re |
| 12 |
> |
import os, string, re, shutil, glob |
| 13 |
|
|
| 14 |
|
class Cmssw(JobType): |
| 15 |
< |
def __init__(self, cfg_params): |
| 15 |
> |
def __init__(self, cfg_params, ncjobs): |
| 16 |
|
JobType.__init__(self, 'CMSSW') |
| 17 |
|
common.logger.debug(3,'CMSSW::__init__') |
| 18 |
|
|
| 21 |
– |
self.analisys_common_info = {} |
| 19 |
|
# Marco. |
| 20 |
|
self._params = {} |
| 21 |
|
self.cfg_params = cfg_params |
| 22 |
+ |
|
| 23 |
+ |
try: |
| 24 |
+ |
self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize']) |
| 25 |
+ |
except KeyError: |
| 26 |
+ |
self.MaxTarBallSize = 100.0 |
| 27 |
+ |
|
| 28 |
+ |
# number of jobs requested to be created, limit obj splitting |
| 29 |
+ |
self.ncjobs = ncjobs |
| 30 |
+ |
|
| 31 |
|
log = common.logger |
| 32 |
|
|
| 33 |
|
self.scram = Scram.Scram(cfg_params) |
| 28 |
– |
scramArea = '' |
| 34 |
|
self.additional_inbox_files = [] |
| 35 |
|
self.scriptExe = '' |
| 36 |
|
self.executable = '' |
| 37 |
+ |
self.executable_arch = self.scram.getArch() |
| 38 |
|
self.tgz_name = 'default.tgz' |
| 39 |
+ |
self.scriptName = 'CMSSW.sh' |
| 40 |
+ |
self.pset = '' #scrip use case Da |
| 41 |
+ |
self.datasetPath = '' #scrip use case Da |
| 42 |
|
|
| 43 |
+ |
# set FJR file name |
| 44 |
+ |
self.fjrFileName = 'crab_fjr.xml' |
| 45 |
|
|
| 46 |
|
self.version = self.scram.getSWVersion() |
| 47 |
+ |
common.taskDB.setDict('codeVersion',self.version) |
| 48 |
|
self.setParam_('application', self.version) |
| 37 |
– |
common.analisys_common_info['sw_version'] = self.version |
| 38 |
– |
### FEDE |
| 39 |
– |
common.analisys_common_info['copy_input_data'] = 0 |
| 40 |
– |
common.analisys_common_info['events_management'] = 1 |
| 49 |
|
|
| 50 |
|
### collect Data cards |
| 51 |
+ |
|
| 52 |
+ |
## get DBS mode |
| 53 |
+ |
try: |
| 54 |
+ |
self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2']) |
| 55 |
+ |
except KeyError: |
| 56 |
+ |
self.use_dbs_2 = 0 |
| 57 |
+ |
|
| 58 |
|
try: |
| 59 |
|
tmp = cfg_params['CMSSW.datasetpath'] |
| 60 |
|
log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp) |
| 75 |
|
self.setParam_('owner', 'None') |
| 76 |
|
else: |
| 77 |
|
datasetpath_split = self.datasetPath.split("/") |
| 78 |
< |
self.setParam_('dataset', datasetpath_split[1]) |
| 79 |
< |
self.setParam_('owner', datasetpath_split[-1]) |
| 78 |
> |
if self.use_dbs_2 == 1 : |
| 79 |
> |
self.setParam_('dataset', datasetpath_split[1]) |
| 80 |
> |
self.setParam_('owner', datasetpath_split[2]) |
| 81 |
> |
else : |
| 82 |
> |
self.setParam_('dataset', datasetpath_split[1]) |
| 83 |
> |
self.setParam_('owner', datasetpath_split[-1]) |
| 84 |
|
|
| 85 |
|
self.setTaskid_() |
| 86 |
|
self.setParam_('taskId', self.cfg_params['taskId']) |
| 104 |
|
try: |
| 105 |
|
self.pset = cfg_params['CMSSW.pset'] |
| 106 |
|
log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset) |
| 107 |
< |
if (not os.path.exists(self.pset)): |
| 108 |
< |
raise CrabException("User defined PSet file "+self.pset+" does not exist") |
| 107 |
> |
if self.pset.lower() != 'none' : |
| 108 |
> |
if (not os.path.exists(self.pset)): |
| 109 |
> |
raise CrabException("User defined PSet file "+self.pset+" does not exist") |
| 110 |
> |
else: |
| 111 |
> |
self.pset = None |
| 112 |
|
except KeyError: |
| 113 |
|
raise CrabException("PSet file missing. Cannot run cmsRun ") |
| 114 |
|
|
| 115 |
|
# output files |
| 116 |
+ |
## stuff which must be returned always via sandbox |
| 117 |
+ |
self.output_file_sandbox = [] |
| 118 |
+ |
|
| 119 |
+ |
# add fjr report by default via sandbox |
| 120 |
+ |
self.output_file_sandbox.append(self.fjrFileName) |
| 121 |
+ |
|
| 122 |
+ |
# other output files to be returned via sandbox or copied to SE |
| 123 |
|
try: |
| 124 |
|
self.output_file = [] |
| 96 |
– |
|
| 125 |
|
tmp = cfg_params['CMSSW.output_file'] |
| 126 |
|
if tmp != '': |
| 127 |
|
tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',') |
| 131 |
|
self.output_file.append(tmp) |
| 132 |
|
pass |
| 133 |
|
else: |
| 134 |
< |
log.message("No output file defined: only stdout/err will be available") |
| 134 |
> |
log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available") |
| 135 |
|
pass |
| 136 |
|
pass |
| 137 |
|
except KeyError: |
| 138 |
< |
log.message("No output file defined: only stdout/err will be available") |
| 138 |
> |
log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available") |
| 139 |
|
pass |
| 140 |
|
|
| 141 |
|
# script_exe file as additional file in inputSandbox |
| 142 |
|
try: |
| 143 |
|
self.scriptExe = cfg_params['USER.script_exe'] |
| 116 |
– |
self.additional_inbox_files.append(self.scriptExe) |
| 144 |
|
if self.scriptExe != '': |
| 145 |
|
if not os.path.isfile(self.scriptExe): |
| 146 |
< |
msg ="WARNING. file "+self.scriptExe+" not found" |
| 146 |
> |
msg ="ERROR. file "+self.scriptExe+" not found" |
| 147 |
|
raise CrabException(msg) |
| 148 |
+ |
self.additional_inbox_files.append(string.strip(self.scriptExe)) |
| 149 |
|
except KeyError: |
| 150 |
< |
pass |
| 151 |
< |
|
| 150 |
> |
self.scriptExe = '' |
| 151 |
> |
|
| 152 |
> |
#CarlosDaniele |
| 153 |
> |
if self.datasetPath == None and self.pset == None and self.scriptExe == '' : |
| 154 |
> |
msg ="Error. script_exe not defined" |
| 155 |
> |
raise CrabException(msg) |
| 156 |
> |
|
| 157 |
|
## additional input files |
| 158 |
|
try: |
| 159 |
|
tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',') |
| 160 |
|
for tmp in tmpAddFiles: |
| 161 |
< |
if not os.path.exists(tmp): |
| 162 |
< |
raise CrabException("Additional input file not found: "+tmp) |
| 163 |
< |
self.additional_inbox_files.append(string.strip(tmp)) |
| 161 |
> |
tmp = string.strip(tmp) |
| 162 |
> |
dirname = '' |
| 163 |
> |
if not tmp[0]=="/": dirname = "." |
| 164 |
> |
files = glob.glob(os.path.join(dirname, tmp)) |
| 165 |
> |
for file in files: |
| 166 |
> |
if not os.path.exists(file): |
| 167 |
> |
raise CrabException("Additional input file not found: "+file) |
| 168 |
> |
pass |
| 169 |
> |
storedFile = common.work_space.shareDir()+file |
| 170 |
> |
shutil.copyfile(file, storedFile) |
| 171 |
> |
self.additional_inbox_files.append(string.strip(storedFile)) |
| 172 |
|
pass |
| 173 |
|
pass |
| 174 |
+ |
common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files)) |
| 175 |
|
except KeyError: |
| 176 |
|
pass |
| 177 |
|
|
| 205 |
|
self.total_number_of_events = 0 |
| 206 |
|
self.selectTotalNumberEvents = 0 |
| 207 |
|
|
| 208 |
< |
if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ): |
| 209 |
< |
msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.' |
| 210 |
< |
raise CrabException(msg) |
| 208 |
> |
if self.pset != None: #CarlosDaniele |
| 209 |
> |
if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ): |
| 210 |
> |
msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.' |
| 211 |
> |
raise CrabException(msg) |
| 212 |
> |
else: |
| 213 |
> |
if (self.selectNumberOfJobs == 0): |
| 214 |
> |
msg = 'Must specify number_of_jobs.' |
| 215 |
> |
raise CrabException(msg) |
| 216 |
|
|
| 217 |
|
## source seed for pythia |
| 218 |
|
try: |
| 226 |
|
except KeyError: |
| 227 |
|
self.sourceSeedVtx = None |
| 228 |
|
common.logger.debug(5,"No vertex seed given") |
| 229 |
< |
|
| 230 |
< |
self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset |
| 229 |
> |
try: |
| 230 |
> |
self.firstRun = int(cfg_params['CMSSW.first_run']) |
| 231 |
> |
except KeyError: |
| 232 |
> |
self.firstRun = None |
| 233 |
> |
common.logger.debug(5,"No first run given") |
| 234 |
> |
if self.pset != None: #CarlosDaniele |
| 235 |
> |
self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset |
| 236 |
|
|
| 237 |
|
#DBSDLS-start |
| 238 |
|
## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code |
| 249 |
|
self.tgzNameWithPath = self.getTarBall(self.executable) |
| 250 |
|
|
| 251 |
|
## Select Splitting |
| 252 |
< |
if self.selectNoInput: self.jobSplittingNoInput() |
| 253 |
< |
else: self.jobSplittingByBlocks(blockSites) |
| 252 |
> |
if self.selectNoInput: |
| 253 |
> |
if self.pset == None: #CarlosDaniele |
| 254 |
> |
self.jobSplittingForScript() |
| 255 |
> |
else: |
| 256 |
> |
self.jobSplittingNoInput() |
| 257 |
> |
else: |
| 258 |
> |
self.jobSplittingByBlocks(blockSites) |
| 259 |
|
|
| 260 |
|
# modify Pset |
| 261 |
< |
try: |
| 262 |
< |
if (self.datasetPath): # standard job |
| 263 |
< |
# always process all events in a file |
| 264 |
< |
self.PsetEdit.maxEvent("-1") |
| 265 |
< |
self.PsetEdit.inputModule("INPUT") |
| 266 |
< |
|
| 267 |
< |
else: # pythia like job |
| 268 |
< |
self.PsetEdit.maxEvent(self.eventsPerJob) |
| 269 |
< |
if (self.sourceSeed) : |
| 270 |
< |
self.PsetEdit.pythiaSeed("INPUT") |
| 271 |
< |
if (self.sourceSeedVtx) : |
| 272 |
< |
self.PsetEdit.pythiaSeedVtx("INPUTVTX") |
| 273 |
< |
self.PsetEdit.psetWriter(self.configFilename()) |
| 274 |
< |
except: |
| 275 |
< |
msg='Error while manipuliating ParameterSet: exiting...' |
| 276 |
< |
raise CrabException(msg) |
| 261 |
> |
if self.pset != None: #CarlosDaniele |
| 262 |
> |
try: |
| 263 |
> |
if (self.datasetPath): # standard job |
| 264 |
> |
# allow to processa a fraction of events in a file |
| 265 |
> |
self.PsetEdit.inputModule("INPUT") |
| 266 |
> |
self.PsetEdit.maxEvent("INPUTMAXEVENTS") |
| 267 |
> |
self.PsetEdit.skipEvent("INPUTSKIPEVENTS") |
| 268 |
> |
else: # pythia like job |
| 269 |
> |
self.PsetEdit.maxEvent(self.eventsPerJob) |
| 270 |
> |
if (self.firstRun): |
| 271 |
> |
self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN") #First Run |
| 272 |
> |
if (self.sourceSeed) : |
| 273 |
> |
self.PsetEdit.pythiaSeed("INPUT") |
| 274 |
> |
if (self.sourceSeedVtx) : |
| 275 |
> |
self.PsetEdit.pythiaSeedVtx("INPUTVTX") |
| 276 |
> |
# add FrameworkJobReport to parameter-set |
| 277 |
> |
self.PsetEdit.addCrabFJR(self.fjrFileName) |
| 278 |
> |
self.PsetEdit.psetWriter(self.configFilename()) |
| 279 |
> |
except: |
| 280 |
> |
msg='Error while manipuliating ParameterSet: exiting...' |
| 281 |
> |
raise CrabException(msg) |
| 282 |
|
|
| 283 |
|
def DataDiscoveryAndLocation(self, cfg_params): |
| 284 |
|
|
| 286 |
|
|
| 287 |
|
datasetPath=self.datasetPath |
| 288 |
|
|
| 227 |
– |
## TODO |
| 228 |
– |
dataTiersList = "" |
| 229 |
– |
dataTiers = dataTiersList.split(',') |
| 230 |
– |
|
| 289 |
|
## Contact the DBS |
| 290 |
+ |
common.logger.message("Contacting DBS...") |
| 291 |
|
try: |
| 292 |
< |
self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params) |
| 292 |
> |
|
| 293 |
> |
if self.use_dbs_2 == 1 : |
| 294 |
> |
self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params) |
| 295 |
> |
else : |
| 296 |
> |
self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params) |
| 297 |
|
self.pubdata.fetchDBSInfo() |
| 298 |
|
|
| 299 |
< |
except DataDiscovery_EDM.NotExistingDatasetError, ex : |
| 299 |
> |
except DataDiscovery.NotExistingDatasetError, ex : |
| 300 |
|
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 301 |
|
raise CrabException(msg) |
| 302 |
< |
|
| 240 |
< |
except DataDiscovery_EDM.NoDataTierinProvenanceError, ex : |
| 302 |
> |
except DataDiscovery.NoDataTierinProvenanceError, ex : |
| 303 |
|
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 304 |
|
raise CrabException(msg) |
| 305 |
< |
except DataDiscovery_EDM.DataDiscoveryError, ex: |
| 306 |
< |
msg = 'ERROR ***: failed Data Discovery in DBS %s'%ex.getErrorMessage() |
| 305 |
> |
except DataDiscovery.DataDiscoveryError, ex: |
| 306 |
> |
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 307 |
> |
raise CrabException(msg) |
| 308 |
> |
except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex : |
| 309 |
> |
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 310 |
> |
raise CrabException(msg) |
| 311 |
> |
except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex : |
| 312 |
> |
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 313 |
> |
raise CrabException(msg) |
| 314 |
> |
except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex: |
| 315 |
> |
msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage() |
| 316 |
|
raise CrabException(msg) |
| 317 |
|
|
| 318 |
|
## get list of all required data in the form of dbs paths (dbs path = /dataset/datatier/owner) |
| 248 |
– |
## self.DBSPaths=self.pubdata.getDBSPaths() |
| 319 |
|
common.logger.message("Required data are :"+self.datasetPath) |
| 320 |
|
|
| 321 |
|
self.filesbyblock=self.pubdata.getFiles() |
| 324 |
|
|
| 325 |
|
## get max number of events |
| 326 |
|
self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py |
| 327 |
< |
common.logger.message("\nThe number of available events is %s"%self.maxEvents) |
| 327 |
> |
common.logger.message("The number of available events is %s\n"%self.maxEvents) |
| 328 |
|
|
| 329 |
+ |
common.logger.message("Contacting DLS...") |
| 330 |
|
## Contact the DLS and build a list of sites hosting the fileblocks |
| 331 |
|
try: |
| 332 |
< |
dataloc=DataLocation_EDM.DataLocation_EDM(self.filesbyblock.keys(),cfg_params) |
| 332 |
> |
dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params) |
| 333 |
|
dataloc.fetchDLSInfo() |
| 334 |
< |
except DataLocation_EDM.DataLocationError , ex: |
| 334 |
> |
except DataLocation.DataLocationError , ex: |
| 335 |
|
msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage() |
| 336 |
|
raise CrabException(msg) |
| 337 |
|
|
| 339 |
|
sites = dataloc.getSites() |
| 340 |
|
allSites = [] |
| 341 |
|
listSites = sites.values() |
| 342 |
< |
for list in listSites: |
| 343 |
< |
for oneSite in list: |
| 342 |
> |
for listSite in listSites: |
| 343 |
> |
for oneSite in listSite: |
| 344 |
|
allSites.append(oneSite) |
| 345 |
|
allSites = self.uniquelist(allSites) |
| 346 |
|
|
| 380 |
|
else: |
| 381 |
|
eventsRemaining = totalEventsRequested |
| 382 |
|
|
| 383 |
+ |
# If user requested more events per job than are in the dataset |
| 384 |
+ |
if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents): |
| 385 |
+ |
eventsPerJobRequested = self.maxEvents |
| 386 |
+ |
|
| 387 |
|
# For user info at end |
| 388 |
|
totalEventCount = 0 |
| 389 |
|
|
| 393 |
|
if (self.selectNumberOfJobs): |
| 394 |
|
common.logger.message("May not create the exact number_of_jobs requested.") |
| 395 |
|
|
| 396 |
+ |
if ( self.ncjobs == 'all' ) : |
| 397 |
+ |
totalNumberOfJobs = 999999999 |
| 398 |
+ |
else : |
| 399 |
+ |
totalNumberOfJobs = self.ncjobs |
| 400 |
+ |
|
| 401 |
+ |
|
| 402 |
|
blocks = blockSites.keys() |
| 403 |
|
blockCount = 0 |
| 404 |
|
# Backup variable in case self.maxEvents counted events in a non-included block |
| 409 |
|
|
| 410 |
|
# ---- Iterate over the blocks in the dataset until ---- # |
| 411 |
|
# ---- we've met the requested total # of events ---- # |
| 412 |
< |
while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) ): |
| 412 |
> |
while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)): |
| 413 |
|
block = blocks[blockCount] |
| 414 |
< |
|
| 415 |
< |
|
| 416 |
< |
evInBlock = self.eventsbyblock[block] |
| 417 |
< |
common.logger.debug(5,'Events in Block File '+str(evInBlock)) |
| 418 |
< |
|
| 338 |
< |
#Correct - switch to this when DBS up |
| 339 |
< |
#numEventsInBlock = self.eventsbyblock[block] |
| 340 |
< |
numEventsInBlock = evInBlock |
| 414 |
> |
blockCount += 1 |
| 415 |
> |
|
| 416 |
> |
if self.eventsbyblock.has_key(block) : |
| 417 |
> |
numEventsInBlock = self.eventsbyblock[block] |
| 418 |
> |
common.logger.debug(5,'Events in Block File '+str(numEventsInBlock)) |
| 419 |
|
|
| 420 |
< |
files = self.filesbyblock[block] |
| 421 |
< |
numFilesInBlock = len(files) |
| 422 |
< |
if (numFilesInBlock <= 0): |
| 423 |
< |
continue |
| 424 |
< |
fileCount = 0 |
| 425 |
< |
|
| 426 |
< |
# ---- New block => New job ---- # |
| 427 |
< |
parString = "\\{" |
| 428 |
< |
jobEventCount = 0 |
| 420 |
> |
files = self.filesbyblock[block] |
| 421 |
> |
numFilesInBlock = len(files) |
| 422 |
> |
if (numFilesInBlock <= 0): |
| 423 |
> |
continue |
| 424 |
> |
fileCount = 0 |
| 425 |
> |
|
| 426 |
> |
# ---- New block => New job ---- # |
| 427 |
> |
parString = "\\{" |
| 428 |
> |
# counter for number of events in files currently worked on |
| 429 |
> |
filesEventCount = 0 |
| 430 |
> |
# flag if next while loop should touch new file |
| 431 |
> |
newFile = 1 |
| 432 |
> |
# job event counter |
| 433 |
> |
jobSkipEventCount = 0 |
| 434 |
|
|
| 435 |
< |
# ---- Iterate over the files in the block until we've met the requested ---- # |
| 436 |
< |
# ---- total # of events or we've gone over all the files in this block ---- # |
| 437 |
< |
while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ): |
| 438 |
< |
file = files[fileCount] |
| 439 |
< |
fileCount = fileCount + 1 |
| 440 |
< |
numEventsInFile = self.eventsbyfile[file] |
| 441 |
< |
common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events") |
| 442 |
< |
# Add file to current job |
| 443 |
< |
parString += '\\\"' + file + '\\\"\,' |
| 444 |
< |
jobEventCount = jobEventCount + numEventsInFile |
| 445 |
< |
totalEventCount = totalEventCount + numEventsInFile |
| 446 |
< |
eventsRemaining = eventsRemaining - numEventsInFile |
| 447 |
< |
if (jobEventCount >= eventsPerJobRequested): |
| 448 |
< |
# ---- This job has at least CMSSW.events_per_job => End of job ---- # |
| 449 |
< |
# Don't need the last \, |
| 450 |
< |
fullString = parString[:-2] |
| 451 |
< |
fullString += '\\}' |
| 452 |
< |
list_of_lists.append([fullString]) |
| 453 |
< |
common.logger.message("Job "+str(jobCount+1)+" can run over "+str(jobEventCount)+" events.") |
| 454 |
< |
|
| 455 |
< |
#self.jobDestination[jobCount] = blockSites[block] |
| 456 |
< |
self.jobDestination.append(blockSites[block]) |
| 457 |
< |
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 458 |
< |
if ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ): |
| 459 |
< |
# ---- Still need CMSSW.total_number_of_events ---- # |
| 460 |
< |
# ---- and not about to jump into a new block ---- # |
| 461 |
< |
# ---- => New job ---- # |
| 435 |
> |
# ---- Iterate over the files in the block until we've met the requested ---- # |
| 436 |
> |
# ---- total # of events or we've gone over all the files in this block ---- # |
| 437 |
> |
while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ): |
| 438 |
> |
file = files[fileCount] |
| 439 |
> |
if newFile : |
| 440 |
> |
try: |
| 441 |
> |
numEventsInFile = self.eventsbyfile[file] |
| 442 |
> |
common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events") |
| 443 |
> |
# increase filesEventCount |
| 444 |
> |
filesEventCount += numEventsInFile |
| 445 |
> |
# Add file to current job |
| 446 |
> |
parString += '\\\"' + file + '\\\"\,' |
| 447 |
> |
newFile = 0 |
| 448 |
> |
except KeyError: |
| 449 |
> |
common.logger.message("File "+str(file)+" has unknown number of events: skipping") |
| 450 |
> |
|
| 451 |
> |
|
| 452 |
> |
# if less events in file remain than eventsPerJobRequested |
| 453 |
> |
if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) : |
| 454 |
> |
# if last file in block |
| 455 |
> |
if ( fileCount == numFilesInBlock-1 ) : |
| 456 |
> |
# end job using last file, use remaining events in block |
| 457 |
> |
# close job and touch new file |
| 458 |
> |
fullString = parString[:-2] |
| 459 |
> |
fullString += '\\}' |
| 460 |
> |
list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)]) |
| 461 |
> |
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).") |
| 462 |
> |
self.jobDestination.append(blockSites[block]) |
| 463 |
> |
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 464 |
> |
# reset counter |
| 465 |
> |
jobCount = jobCount + 1 |
| 466 |
> |
totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount |
| 467 |
> |
eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount |
| 468 |
> |
jobSkipEventCount = 0 |
| 469 |
> |
# reset file |
| 470 |
> |
parString = "\\{" |
| 471 |
> |
filesEventCount = 0 |
| 472 |
> |
newFile = 1 |
| 473 |
> |
fileCount += 1 |
| 474 |
> |
else : |
| 475 |
> |
# go to next file |
| 476 |
> |
newFile = 1 |
| 477 |
> |
fileCount += 1 |
| 478 |
> |
# if events in file equal to eventsPerJobRequested |
| 479 |
> |
elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) : |
| 480 |
> |
# close job and touch new file |
| 481 |
> |
fullString = parString[:-2] |
| 482 |
> |
fullString += '\\}' |
| 483 |
> |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 484 |
> |
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.") |
| 485 |
> |
self.jobDestination.append(blockSites[block]) |
| 486 |
> |
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 487 |
> |
# reset counter |
| 488 |
> |
jobCount = jobCount + 1 |
| 489 |
> |
totalEventCount = totalEventCount + eventsPerJobRequested |
| 490 |
> |
eventsRemaining = eventsRemaining - eventsPerJobRequested |
| 491 |
> |
jobSkipEventCount = 0 |
| 492 |
> |
# reset file |
| 493 |
|
parString = "\\{" |
| 494 |
< |
jobEventCount = 0 |
| 494 |
> |
filesEventCount = 0 |
| 495 |
> |
newFile = 1 |
| 496 |
> |
fileCount += 1 |
| 497 |
> |
|
| 498 |
> |
# if more events in file remain than eventsPerJobRequested |
| 499 |
> |
else : |
| 500 |
> |
# close job but don't touch new file |
| 501 |
> |
fullString = parString[:-2] |
| 502 |
> |
fullString += '\\}' |
| 503 |
> |
list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)]) |
| 504 |
> |
common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.") |
| 505 |
> |
self.jobDestination.append(blockSites[block]) |
| 506 |
> |
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 507 |
> |
# increase counter |
| 508 |
|
jobCount = jobCount + 1 |
| 509 |
+ |
totalEventCount = totalEventCount + eventsPerJobRequested |
| 510 |
+ |
eventsRemaining = eventsRemaining - eventsPerJobRequested |
| 511 |
+ |
# calculate skip events for last file |
| 512 |
+ |
# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest |
| 513 |
+ |
jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file]) |
| 514 |
+ |
# remove all but the last file |
| 515 |
+ |
filesEventCount = self.eventsbyfile[file] |
| 516 |
+ |
parString = "\\{" |
| 517 |
+ |
parString += '\\\"' + file + '\\\"\,' |
| 518 |
|
pass # END if |
| 519 |
< |
pass # END if |
| 384 |
< |
pass # END while (iterate over files in the block) |
| 385 |
< |
if (jobEventCount < eventsPerJobRequested): |
| 386 |
< |
# ---- Job ending prematurely due to end of block => End of job ---- # |
| 387 |
< |
# Don't need the last \, |
| 388 |
< |
fullString = parString[:-2] |
| 389 |
< |
fullString += '\\}' |
| 390 |
< |
list_of_lists.append([fullString]) |
| 391 |
< |
common.logger.message("Job "+str(jobCount+1)+" can run over "+str(jobEventCount)+" events.") |
| 392 |
< |
#self.jobDestination[jobCount] = blockSites[block] |
| 393 |
< |
self.jobDestination.append(blockSites[block]) |
| 394 |
< |
common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount])) |
| 395 |
< |
pass # END if |
| 396 |
< |
blockCount = blockCount + 1 |
| 397 |
< |
jobCount = jobCount + 1 |
| 519 |
> |
pass # END while (iterate over files in the block) |
| 520 |
|
pass # END while (iterate over blocks in the dataset) |
| 521 |
< |
self.total_number_of_jobs = jobCount |
| 522 |
< |
if (eventsRemaining > 0): |
| 521 |
> |
self.ncjobs = self.total_number_of_jobs = jobCount |
| 522 |
> |
if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ): |
| 523 |
|
common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.") |
| 524 |
|
common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n") |
| 525 |
|
|
| 540 |
|
raise CrabException(msg) |
| 541 |
|
|
| 542 |
|
if (self.selectEventsPerJob): |
| 543 |
< |
self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob) |
| 543 |
> |
if (self.selectTotalNumberEvents): |
| 544 |
> |
self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob) |
| 545 |
> |
elif(self.selectNumberOfJobs) : |
| 546 |
> |
self.total_number_of_jobs =self.theNumberOfJobs |
| 547 |
> |
self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob) |
| 548 |
> |
|
| 549 |
|
elif (self.selectNumberOfJobs) : |
| 550 |
|
self.total_number_of_jobs = self.theNumberOfJobs |
| 551 |
|
self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs) |
| 552 |
< |
|
| 552 |
> |
|
| 553 |
|
common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs)) |
| 554 |
|
|
| 555 |
|
# is there any remainder? |
| 565 |
|
self.list_of_args = [] |
| 566 |
|
for i in range(self.total_number_of_jobs): |
| 567 |
|
## Since there is no input, any site is good |
| 568 |
< |
self.jobDestination.append(["Any"]) |
| 568 |
> |
# self.jobDestination.append(["Any"]) |
| 569 |
> |
self.jobDestination.append([""]) #must be empty to write correctly the xml |
| 570 |
> |
args='' |
| 571 |
> |
if (self.firstRun): |
| 572 |
> |
## pythia first run |
| 573 |
> |
#self.list_of_args.append([(str(self.firstRun)+str(i))]) |
| 574 |
> |
args=args+(str(self.firstRun)+str(i)) |
| 575 |
> |
else: |
| 576 |
> |
## no first run |
| 577 |
> |
#self.list_of_args.append([str(i)]) |
| 578 |
> |
args=args+str(i) |
| 579 |
|
if (self.sourceSeed): |
| 580 |
|
if (self.sourceSeedVtx): |
| 581 |
|
## pythia + vtx random seed |
| 582 |
< |
self.list_of_args.append([ |
| 583 |
< |
str(self.sourceSeed)+str(i), |
| 584 |
< |
str(self.sourceSeedVtx)+str(i) |
| 585 |
< |
]) |
| 582 |
> |
#self.list_of_args.append([ |
| 583 |
> |
# str(self.sourceSeed)+str(i), |
| 584 |
> |
# str(self.sourceSeedVtx)+str(i) |
| 585 |
> |
# ]) |
| 586 |
> |
args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i) |
| 587 |
|
else: |
| 588 |
|
## only pythia random seed |
| 589 |
< |
self.list_of_args.append([(str(self.sourceSeed)+str(i))]) |
| 589 |
> |
#self.list_of_args.append([(str(self.sourceSeed)+str(i))]) |
| 590 |
> |
args=args +str(',')+str(self.sourceSeed)+str(i) |
| 591 |
|
else: |
| 592 |
|
## no random seed |
| 593 |
< |
self.list_of_args.append([str(i)]) |
| 594 |
< |
#print self.list_of_args |
| 593 |
> |
if str(args)=='': args=args+(str(self.firstRun)+str(i)) |
| 594 |
> |
arguments=args.split(',') |
| 595 |
> |
if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])]) |
| 596 |
> |
elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])]) |
| 597 |
> |
else :self.list_of_args.append([str(arguments[0])]) |
| 598 |
> |
|
| 599 |
> |
# print self.list_of_args |
| 600 |
> |
|
| 601 |
> |
return |
| 602 |
> |
|
| 603 |
|
|
| 604 |
+ |
def jobSplittingForScript(self):#CarlosDaniele |
| 605 |
+ |
""" |
| 606 |
+ |
Perform job splitting based on number of job |
| 607 |
+ |
""" |
| 608 |
+ |
common.logger.debug(5,'Splitting per job') |
| 609 |
+ |
common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ') |
| 610 |
+ |
|
| 611 |
+ |
self.total_number_of_jobs = self.theNumberOfJobs |
| 612 |
+ |
|
| 613 |
+ |
common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs)) |
| 614 |
+ |
|
| 615 |
+ |
common.logger.message(str(self.total_number_of_jobs)+' jobs can be created') |
| 616 |
+ |
|
| 617 |
+ |
# argument is seed number.$i |
| 618 |
+ |
self.list_of_args = [] |
| 619 |
+ |
for i in range(self.total_number_of_jobs): |
| 620 |
+ |
## Since there is no input, any site is good |
| 621 |
+ |
# self.jobDestination.append(["Any"]) |
| 622 |
+ |
self.jobDestination.append([""]) |
| 623 |
+ |
## no random seed |
| 624 |
+ |
self.list_of_args.append([str(i)]) |
| 625 |
|
return |
| 626 |
|
|
| 627 |
|
def split(self, jobParams): |
| 661 |
|
""" |
| 662 |
|
|
| 663 |
|
# if it exist, just return it |
| 664 |
< |
self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name |
| 664 |
> |
# |
| 665 |
> |
# Marco. Let's start to use relative path for Boss XML files |
| 666 |
> |
# |
| 667 |
> |
self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name |
| 668 |
|
if os.path.exists(self.tgzNameWithPath): |
| 669 |
|
return self.tgzNameWithPath |
| 670 |
|
|
| 678 |
|
# First of all declare the user Scram area |
| 679 |
|
swArea = self.scram.getSWArea_() |
| 680 |
|
#print "swArea = ", swArea |
| 681 |
< |
swVersion = self.scram.getSWVersion() |
| 682 |
< |
#print "swVersion = ", swVersion |
| 681 |
> |
# swVersion = self.scram.getSWVersion() |
| 682 |
> |
# print "swVersion = ", swVersion |
| 683 |
|
swReleaseTop = self.scram.getReleaseTop_() |
| 684 |
|
#print "swReleaseTop = ", swReleaseTop |
| 685 |
|
|
| 687 |
|
if swReleaseTop == '' or swArea == swReleaseTop: |
| 688 |
|
return |
| 689 |
|
|
| 690 |
< |
filesToBeTarred = [] |
| 691 |
< |
## First find the executable |
| 692 |
< |
if (self.executable != ''): |
| 693 |
< |
exeWithPath = self.scram.findFile_(executable) |
| 694 |
< |
# print exeWithPath |
| 695 |
< |
if ( not exeWithPath ): |
| 696 |
< |
raise CrabException('User executable '+executable+' not found') |
| 697 |
< |
|
| 698 |
< |
## then check if it's private or not |
| 699 |
< |
if exeWithPath.find(swReleaseTop) == -1: |
| 700 |
< |
# the exe is private, so we must ship |
| 701 |
< |
common.logger.debug(5,"Exe "+exeWithPath+" to be tarred") |
| 702 |
< |
path = swArea+'/' |
| 703 |
< |
exe = string.replace(exeWithPath, path,'') |
| 704 |
< |
filesToBeTarred.append(exe) |
| 705 |
< |
pass |
| 706 |
< |
else: |
| 707 |
< |
# the exe is from release, we'll find it on WN |
| 708 |
< |
pass |
| 709 |
< |
|
| 710 |
< |
## Now get the libraries: only those in local working area |
| 711 |
< |
libDir = 'lib' |
| 712 |
< |
lib = swArea+'/' +libDir |
| 713 |
< |
common.logger.debug(5,"lib "+lib+" to be tarred") |
| 714 |
< |
if os.path.exists(lib): |
| 715 |
< |
filesToBeTarred.append(libDir) |
| 716 |
< |
|
| 717 |
< |
## Now check if module dir is present |
| 718 |
< |
moduleDir = 'module' |
| 719 |
< |
if os.path.isdir(swArea+'/'+moduleDir): |
| 720 |
< |
filesToBeTarred.append(moduleDir) |
| 721 |
< |
|
| 722 |
< |
## Now check if the Data dir is present |
| 723 |
< |
dataDir = 'src/Data/' |
| 724 |
< |
if os.path.isdir(swArea+'/'+dataDir): |
| 725 |
< |
filesToBeTarred.append(dataDir) |
| 726 |
< |
|
| 727 |
< |
## Create the tar-ball |
| 728 |
< |
if len(filesToBeTarred)>0: |
| 729 |
< |
cwd = os.getcwd() |
| 730 |
< |
os.chdir(swArea) |
| 731 |
< |
tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' ' |
| 732 |
< |
for line in filesToBeTarred: |
| 733 |
< |
tarcmd = tarcmd + line + ' ' |
| 734 |
< |
cout = runCommand(tarcmd) |
| 735 |
< |
if not cout: |
| 736 |
< |
raise CrabException('Could not create tar-ball') |
| 737 |
< |
os.chdir(cwd) |
| 738 |
< |
else: |
| 739 |
< |
common.logger.debug(5,"No files to be to be tarred") |
| 690 |
> |
import tarfile |
| 691 |
> |
try: # create tar ball |
| 692 |
> |
tar = tarfile.open(self.tgzNameWithPath, "w:gz") |
| 693 |
> |
## First find the executable |
| 694 |
> |
if (executable != ''): |
| 695 |
> |
exeWithPath = self.scram.findFile_(executable) |
| 696 |
> |
if ( not exeWithPath ): |
| 697 |
> |
raise CrabException('User executable '+executable+' not found') |
| 698 |
> |
|
| 699 |
> |
## then check if it's private or not |
| 700 |
> |
if exeWithPath.find(swReleaseTop) == -1: |
| 701 |
> |
# the exe is private, so we must ship |
| 702 |
> |
common.logger.debug(5,"Exe "+exeWithPath+" to be tarred") |
| 703 |
> |
path = swArea+'/' |
| 704 |
> |
# distinguish case when script is in user project area or given by full path somewhere else |
| 705 |
> |
if exeWithPath.find(path) >= 0 : |
| 706 |
> |
exe = string.replace(exeWithPath, path,'') |
| 707 |
> |
tar.add(path+exe,os.path.basename(executable)) |
| 708 |
> |
else : |
| 709 |
> |
tar.add(exeWithPath,os.path.basename(executable)) |
| 710 |
> |
pass |
| 711 |
> |
else: |
| 712 |
> |
# the exe is from release, we'll find it on WN |
| 713 |
> |
pass |
| 714 |
> |
|
| 715 |
> |
## Now get the libraries: only those in local working area |
| 716 |
> |
libDir = 'lib' |
| 717 |
> |
lib = swArea+'/' +libDir |
| 718 |
> |
common.logger.debug(5,"lib "+lib+" to be tarred") |
| 719 |
> |
if os.path.exists(lib): |
| 720 |
> |
tar.add(lib,libDir) |
| 721 |
> |
|
| 722 |
> |
## Now check if module dir is present |
| 723 |
> |
moduleDir = 'module' |
| 724 |
> |
module = swArea + '/' + moduleDir |
| 725 |
> |
if os.path.isdir(module): |
| 726 |
> |
tar.add(module,moduleDir) |
| 727 |
> |
|
| 728 |
> |
## Now check if any data dir(s) is present |
| 729 |
> |
swAreaLen=len(swArea) |
| 730 |
> |
for root, dirs, files in os.walk(swArea): |
| 731 |
> |
if "data" in dirs: |
| 732 |
> |
common.logger.debug(5,"data "+root+"/data"+" to be tarred") |
| 733 |
> |
tar.add(root+"/data",root[swAreaLen:]+"/data") |
| 734 |
> |
|
| 735 |
> |
## Add ProdAgent dir to tar |
| 736 |
> |
paDir = 'ProdAgentApi' |
| 737 |
> |
pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi' |
| 738 |
> |
if os.path.isdir(pa): |
| 739 |
> |
tar.add(pa,paDir) |
| 740 |
> |
|
| 741 |
> |
common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames())) |
| 742 |
> |
tar.close() |
| 743 |
> |
except : |
| 744 |
> |
raise CrabException('Could not create tar-ball') |
| 745 |
> |
|
| 746 |
> |
## check for tarball size |
| 747 |
> |
tarballinfo = os.stat(self.tgzNameWithPath) |
| 748 |
> |
if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) : |
| 749 |
> |
raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.') |
| 750 |
> |
|
| 751 |
> |
## create tar-ball with ML stuff |
| 752 |
> |
self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz' |
| 753 |
> |
try: |
| 754 |
> |
tar = tarfile.open(self.MLtgzfile, "w:gz") |
| 755 |
> |
path=os.environ['CRABDIR'] + '/python/' |
| 756 |
> |
for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']: |
| 757 |
> |
tar.add(path+file,file) |
| 758 |
> |
common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames())) |
| 759 |
> |
tar.close() |
| 760 |
> |
except : |
| 761 |
> |
raise CrabException('Could not create ML files tar-ball') |
| 762 |
|
|
| 763 |
|
return |
| 764 |
|
|
| 775 |
|
txt += 'if [ $middleware == LCG ]; then \n' |
| 776 |
|
txt += self.wsSetupCMSLCGEnvironment_() |
| 777 |
|
txt += 'elif [ $middleware == OSG ]; then\n' |
| 778 |
< |
txt += ' time=`date -u +"%s"`\n' |
| 779 |
< |
txt += ' WORKING_DIR=$OSG_WN_TMP/cms_$time\n' |
| 587 |
< |
txt += ' echo "Creating working directory: $WORKING_DIR"\n' |
| 588 |
< |
txt += ' /bin/mkdir -p $WORKING_DIR\n' |
| 778 |
> |
txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n' |
| 779 |
> |
txt += ' echo "Created working directory: $WORKING_DIR"\n' |
| 780 |
|
txt += ' if [ ! -d $WORKING_DIR ] ;then\n' |
| 781 |
|
txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n' |
| 782 |
|
txt += ' echo "JOB_EXIT_STATUS = 10016"\n' |
| 825 |
|
txt += ' exit 1 \n' |
| 826 |
|
txt += 'fi \n' |
| 827 |
|
txt += 'echo "CMSSW_VERSION = '+self.version+'"\n' |
| 828 |
+ |
txt += 'export SCRAM_ARCH='+self.executable_arch+'\n' |
| 829 |
|
txt += 'cd '+self.version+'\n' |
| 830 |
|
### needed grep for bug in scramv1 ### |
| 831 |
+ |
txt += scram+' runtime -sh\n' |
| 832 |
|
txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n' |
| 833 |
+ |
txt += 'echo $PATH\n' |
| 834 |
|
|
| 835 |
|
# Handle the arguments: |
| 836 |
|
txt += "\n" |
| 867 |
|
|
| 868 |
|
# Prepare job-specific part |
| 869 |
|
job = common.job_list[nj] |
| 870 |
< |
pset = os.path.basename(job.configFilename()) |
| 871 |
< |
txt += '\n' |
| 872 |
< |
if (self.datasetPath): # standard job |
| 873 |
< |
#txt += 'InputFiles=$2\n' |
| 874 |
< |
txt += 'InputFiles=${args[1]}\n' |
| 875 |
< |
txt += 'echo "Inputfiles:<$InputFiles>"\n' |
| 876 |
< |
txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n' |
| 877 |
< |
else: # pythia like job |
| 878 |
< |
if (self.sourceSeed): |
| 879 |
< |
# txt += 'Seed=$2\n' |
| 880 |
< |
txt += 'Seed=${args[1]}\n' |
| 881 |
< |
txt += 'echo "Seed: <$Seed>"\n' |
| 882 |
< |
txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n' |
| 883 |
< |
if (self.sourceSeedVtx): |
| 884 |
< |
# txt += 'VtxSeed=$3\n' |
| 885 |
< |
txt += 'VtxSeed=${args[2]}\n' |
| 886 |
< |
txt += 'echo "VtxSeed: <$VtxSeed>"\n' |
| 887 |
< |
txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n' |
| 870 |
> |
if self.pset != None: #CarlosDaniele |
| 871 |
> |
pset = os.path.basename(job.configFilename()) |
| 872 |
> |
txt += '\n' |
| 873 |
> |
if (self.datasetPath): # standard job |
| 874 |
> |
#txt += 'InputFiles=$2\n' |
| 875 |
> |
txt += 'InputFiles=${args[1]}\n' |
| 876 |
> |
txt += 'MaxEvents=${args[2]}\n' |
| 877 |
> |
txt += 'SkipEvents=${args[3]}\n' |
| 878 |
> |
txt += 'echo "Inputfiles:<$InputFiles>"\n' |
| 879 |
> |
txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n' |
| 880 |
> |
txt += 'echo "MaxEvents:<$MaxEvents>"\n' |
| 881 |
> |
txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n' |
| 882 |
> |
txt += 'echo "SkipEvents:<$SkipEvents>"\n' |
| 883 |
> |
txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n' |
| 884 |
> |
else: # pythia like job |
| 885 |
> |
if (self.sourceSeed): |
| 886 |
> |
txt += 'FirstRun=${args[1]}\n' |
| 887 |
> |
txt += 'echo "FirstRun: <$FirstRun>"\n' |
| 888 |
> |
txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n' |
| 889 |
|
else: |
| 890 |
< |
txt += 'mv tmp.cfg pset.cfg\n' |
| 891 |
< |
else: |
| 892 |
< |
txt += '# Copy untouched pset\n' |
| 893 |
< |
txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n' |
| 890 |
> |
txt += '# Copy untouched pset\n' |
| 891 |
> |
txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n' |
| 892 |
> |
if (self.sourceSeed): |
| 893 |
> |
# txt += 'Seed=$2\n' |
| 894 |
> |
txt += 'Seed=${args[2]}\n' |
| 895 |
> |
txt += 'echo "Seed: <$Seed>"\n' |
| 896 |
> |
txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n' |
| 897 |
> |
if (self.sourceSeedVtx): |
| 898 |
> |
# txt += 'VtxSeed=$3\n' |
| 899 |
> |
txt += 'VtxSeed=${args[3]}\n' |
| 900 |
> |
txt += 'echo "VtxSeed: <$VtxSeed>"\n' |
| 901 |
> |
txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n' |
| 902 |
> |
else: |
| 903 |
> |
txt += 'mv tmp_2.cfg pset.cfg\n' |
| 904 |
> |
else: |
| 905 |
> |
txt += 'mv tmp_1.cfg pset.cfg\n' |
| 906 |
> |
# txt += '# Copy untouched pset\n' |
| 907 |
> |
# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n' |
| 908 |
|
|
| 909 |
|
|
| 910 |
|
if len(self.additional_inbox_files) > 0: |
| 916 |
|
txt += 'fi\n' |
| 917 |
|
pass |
| 918 |
|
|
| 919 |
< |
txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n' |
| 920 |
< |
|
| 921 |
< |
txt += '\n' |
| 922 |
< |
txt += 'echo "***** cat pset.cfg *********"\n' |
| 923 |
< |
txt += 'cat pset.cfg\n' |
| 924 |
< |
txt += 'echo "****** end pset.cfg ********"\n' |
| 925 |
< |
txt += '\n' |
| 926 |
< |
# txt += 'echo "***** cat pset1.cfg *********"\n' |
| 927 |
< |
# txt += 'cat pset1.cfg\n' |
| 928 |
< |
# txt += 'echo "****** end pset1.cfg ********"\n' |
| 919 |
> |
if self.pset != None: #CarlosDaniele |
| 920 |
> |
txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n' |
| 921 |
> |
|
| 922 |
> |
txt += '\n' |
| 923 |
> |
txt += 'echo "***** cat pset.cfg *********"\n' |
| 924 |
> |
txt += 'cat pset.cfg\n' |
| 925 |
> |
txt += 'echo "****** end pset.cfg ********"\n' |
| 926 |
> |
txt += '\n' |
| 927 |
> |
# txt += 'echo "***** cat pset1.cfg *********"\n' |
| 928 |
> |
# txt += 'cat pset1.cfg\n' |
| 929 |
> |
# txt += 'echo "****** end pset1.cfg ********"\n' |
| 930 |
|
return txt |
| 931 |
|
|
| 932 |
< |
def wsBuildExe(self, nj): |
| 932 |
> |
def wsBuildExe(self, nj=0): |
| 933 |
|
""" |
| 934 |
|
Put in the script the commands to build an executable |
| 935 |
|
or a library. |
| 964 |
|
txt += 'else \n' |
| 965 |
|
txt += ' echo "Successful untar" \n' |
| 966 |
|
txt += 'fi \n' |
| 967 |
+ |
txt += '\n' |
| 968 |
+ |
txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n' |
| 969 |
+ |
txt += 'if [ -z "$PYTHONPATH" ]; then\n' |
| 970 |
+ |
txt += ' export PYTHONPATH=ProdAgentApi\n' |
| 971 |
+ |
txt += 'else\n' |
| 972 |
+ |
txt += ' export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n' |
| 973 |
+ |
txt += 'fi\n' |
| 974 |
+ |
txt += '\n' |
| 975 |
+ |
|
| 976 |
|
pass |
| 977 |
|
|
| 978 |
|
return txt |
| 984 |
|
""" |
| 985 |
|
|
| 986 |
|
def executableName(self): |
| 987 |
< |
return self.executable |
| 987 |
> |
if self.scriptExe: #CarlosDaniele |
| 988 |
> |
return "sh " |
| 989 |
> |
else: |
| 990 |
> |
return self.executable |
| 991 |
|
|
| 992 |
|
def executableArgs(self): |
| 993 |
< |
return " -p pset.cfg" |
| 993 |
> |
if self.scriptExe:#CarlosDaniele |
| 994 |
> |
return self.scriptExe + " $NJob" |
| 995 |
> |
else: |
| 996 |
> |
return " -p pset.cfg" |
| 997 |
|
|
| 998 |
|
def inputSandbox(self, nj): |
| 999 |
|
""" |
| 1000 |
|
Returns a list of filenames to be put in JDL input sandbox. |
| 1001 |
|
""" |
| 1002 |
|
inp_box = [] |
| 1003 |
< |
# dict added to delete duplicate from input sandbox file list |
| 1004 |
< |
seen = {} |
| 1003 |
> |
# # dict added to delete duplicate from input sandbox file list |
| 1004 |
> |
# seen = {} |
| 1005 |
|
## code |
| 1006 |
|
if os.path.isfile(self.tgzNameWithPath): |
| 1007 |
|
inp_box.append(self.tgzNameWithPath) |
| 1008 |
+ |
if os.path.isfile(self.MLtgzfile): |
| 1009 |
+ |
inp_box.append(self.MLtgzfile) |
| 1010 |
|
## config |
| 1011 |
< |
inp_box.append(common.job_list[nj].configFilename()) |
| 1011 |
> |
if not self.pset is None: |
| 1012 |
> |
inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename()) |
| 1013 |
|
## additional input files |
| 1014 |
< |
#for file in self.additional_inbox_files: |
| 1015 |
< |
# inp_box.append(common.work_space.cwdDir()+file) |
| 1014 |
> |
for file in self.additional_inbox_files: |
| 1015 |
> |
inp_box.append(file) |
| 1016 |
|
return inp_box |
| 1017 |
|
|
| 1018 |
|
def outputSandbox(self, nj): |
| 1021 |
|
""" |
| 1022 |
|
out_box = [] |
| 1023 |
|
|
| 796 |
– |
stdout=common.job_list[nj].stdout() |
| 797 |
– |
stderr=common.job_list[nj].stderr() |
| 798 |
– |
|
| 1024 |
|
## User Declared output files |
| 1025 |
< |
for out in self.output_file: |
| 1025 |
> |
for out in (self.output_file+self.output_file_sandbox): |
| 1026 |
|
n_out = nj + 1 |
| 1027 |
|
out_box.append(self.numberFile_(out,str(n_out))) |
| 1028 |
|
return out_box |
| 804 |
– |
return [] |
| 1029 |
|
|
| 1030 |
|
def prepareSteeringCards(self): |
| 1031 |
|
""" |
| 1041 |
|
txt = '\n' |
| 1042 |
|
txt += '# directory content\n' |
| 1043 |
|
txt += 'ls \n' |
| 1044 |
< |
file_list = '' |
| 1045 |
< |
for fileWithSuffix in self.output_file: |
| 1044 |
> |
|
| 1045 |
> |
for fileWithSuffix in (self.output_file+self.output_file_sandbox): |
| 1046 |
|
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 823 |
– |
file_list=file_list+output_file_num+' ' |
| 1047 |
|
txt += '\n' |
| 1048 |
|
txt += '# check output file\n' |
| 1049 |
|
txt += 'ls '+fileWithSuffix+'\n' |
| 1050 |
|
txt += 'ls_result=$?\n' |
| 828 |
– |
#txt += 'exe_result=$?\n' |
| 1051 |
|
txt += 'if [ $ls_result -ne 0 ] ; then\n' |
| 1052 |
|
txt += ' echo "ERROR: Problem with output file"\n' |
| 831 |
– |
#txt += ' echo "JOB_EXIT_STATUS = $exe_result"\n' |
| 832 |
– |
#txt += ' echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n' |
| 833 |
– |
#txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 834 |
– |
### OLI_DANIELE |
| 1053 |
|
if common.scheduler.boss_scheduler_name == 'condor_g': |
| 1054 |
|
txt += ' if [ $middleware == OSG ]; then \n' |
| 1055 |
|
txt += ' echo "prepare dummy output file"\n' |
| 1060 |
|
txt += 'fi\n' |
| 1061 |
|
|
| 1062 |
|
txt += 'cd $RUNTIME_AREA\n' |
| 845 |
– |
file_list=file_list[:-1] |
| 846 |
– |
txt += 'file_list="'+file_list+'"\n' |
| 1063 |
|
txt += 'cd $RUNTIME_AREA\n' |
| 1064 |
|
### OLI_DANIELE |
| 1065 |
|
txt += 'if [ $middleware == OSG ]; then\n' |
| 1077 |
|
txt += ' fi\n' |
| 1078 |
|
txt += 'fi\n' |
| 1079 |
|
txt += '\n' |
| 1080 |
+ |
|
| 1081 |
+ |
file_list = '' |
| 1082 |
+ |
## Add to filelist only files to be possibly copied to SE |
| 1083 |
+ |
for fileWithSuffix in self.output_file: |
| 1084 |
+ |
output_file_num = self.numberFile_(fileWithSuffix, '$NJob') |
| 1085 |
+ |
file_list=file_list+output_file_num+' ' |
| 1086 |
+ |
file_list=file_list[:-1] |
| 1087 |
+ |
txt += 'file_list="'+file_list+'"\n' |
| 1088 |
+ |
|
| 1089 |
|
return txt |
| 1090 |
|
|
| 1091 |
|
def numberFile_(self, file, txt): |
| 1100 |
|
# add "_txt" |
| 1101 |
|
if len(p)>1: |
| 1102 |
|
ext = p[len(p)-1] |
| 878 |
– |
#result = name + '_' + str(txt) + "." + ext |
| 1103 |
|
result = name + '_' + txt + "." + ext |
| 1104 |
|
else: |
| 881 |
– |
#result = name + '_' + str(txt) |
| 1105 |
|
result = name + '_' + txt |
| 1106 |
|
|
| 1107 |
|
return result |
| 1108 |
|
|
| 1109 |
< |
def getRequirements(self, nj): |
| 1109 |
> |
def getRequirements(self, nj=[]): |
| 1110 |
|
""" |
| 1111 |
|
return job requirements to add to jdl files |
| 1112 |
|
""" |
| 1113 |
|
req = '' |
| 1114 |
< |
if common.analisys_common_info['sw_version']: |
| 1114 |
> |
if self.version: |
| 1115 |
|
req='Member("VO-cms-' + \ |
| 1116 |
< |
common.analisys_common_info['sw_version'] + \ |
| 1116 |
> |
self.version + \ |
| 1117 |
|
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
| 1118 |
|
|
| 1119 |
|
req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)' |
| 1120 |
|
|
| 898 |
– |
## here we should get the requirement for job nj |
| 899 |
– |
sites = common.jobDB.destination(nj) |
| 900 |
– |
|
| 901 |
– |
# check for "Any" site, in case no requirement for site |
| 902 |
– |
if len(sites)>0 and sites[0]!="Any": |
| 903 |
– |
req = req + ' && anyMatch(other.storage.CloseSEs, (' |
| 904 |
– |
for site in sites: |
| 905 |
– |
#req = req + 'other.GlueCEInfoHostName == "' + site + '" || ' |
| 906 |
– |
req = req + 'target.GlueSEUniqueID=="' + site + '" || ' |
| 907 |
– |
pass |
| 908 |
– |
# remove last || |
| 909 |
– |
req = req[0:-4] |
| 910 |
– |
req = req + '))' |
| 911 |
– |
|
| 1121 |
|
return req |
| 1122 |
|
|
| 1123 |
|
def configFilename(self): |
| 1135 |
|
txt += ' if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n' |
| 1136 |
|
txt += ' # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n' |
| 1137 |
|
txt += ' source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n' |
| 1138 |
< |
txt += ' elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n' |
| 1139 |
< |
txt += ' # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n' |
| 1140 |
< |
txt += ' source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n' |
| 1138 |
> |
txt += ' elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n' |
| 1139 |
> |
txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n' |
| 1140 |
> |
txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n' |
| 1141 |
|
txt += ' else\n' |
| 1142 |
< |
txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n' |
| 1142 |
> |
txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1143 |
|
txt += ' echo "JOB_EXIT_STATUS = 10020"\n' |
| 1144 |
|
txt += ' echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n' |
| 1145 |
|
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1152 |
|
txt += ' cd $RUNTIME_AREA\n' |
| 1153 |
|
txt += ' /bin/rm -rf $WORKING_DIR\n' |
| 1154 |
|
txt += ' if [ -d $WORKING_DIR ] ;then\n' |
| 1155 |
< |
txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n' |
| 1155 |
> |
txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n' |
| 1156 |
|
txt += ' echo "JOB_EXIT_STATUS = 10017"\n' |
| 1157 |
|
txt += ' echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n' |
| 1158 |
|
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1213 |
|
txt += ' fi\n' |
| 1214 |
|
txt += ' fi\n' |
| 1215 |
|
txt += ' \n' |
| 1007 |
– |
txt += ' string=`cat /etc/redhat-release`\n' |
| 1008 |
– |
txt += ' echo $string\n' |
| 1009 |
– |
txt += ' if [[ $string = *alhalla* ]]; then\n' |
| 1010 |
– |
txt += ' echo "SCRAM_ARCH= $SCRAM_ARCH"\n' |
| 1011 |
– |
txt += ' elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n' |
| 1012 |
– |
txt += ' export SCRAM_ARCH=slc3_ia32_gcc323\n' |
| 1013 |
– |
txt += ' echo "SCRAM_ARCH= $SCRAM_ARCH"\n' |
| 1014 |
– |
txt += ' else\n' |
| 1015 |
– |
txt += ' echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n' |
| 1016 |
– |
txt += ' echo "JOB_EXIT_STATUS = 10033"\n' |
| 1017 |
– |
txt += ' echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n' |
| 1018 |
– |
txt += ' dumpStatus $RUNTIME_AREA/$repo\n' |
| 1019 |
– |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
| 1020 |
– |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
| 1021 |
– |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
| 1022 |
– |
txt += ' exit 1\n' |
| 1023 |
– |
txt += ' fi\n' |
| 1216 |
|
txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n' |
| 1217 |
|
txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n' |
| 1218 |
|
return txt |
