ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.155 by slacapra, Tue Feb 12 15:20:47 2008 UTC vs.
Revision 1.254 by ewv, Thu Oct 30 16:25:24 2008 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	<	from BlackWhiteListParser import BlackWhiteListParser
5	>	from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
6		import common
7		import Scram
8
9		import os, string, glob
10
11		class Cmssw(JobType):
12	<	def __init__(self, cfg_params, ncjobs):
12	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15	+	self.skip_blocks = skip_blocks
16
17		self.argsList = []
18
19		self._params = {}
20		self.cfg_params = cfg_params
20	–	# init BlackWhiteListParser
21	–	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
21
22	<	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
22	>	# init BlackWhiteListParser
23	>	seWhiteList = cfg_params.get('EDG.se_white_list',[])
24	>	seBlackList = cfg_params.get('EDG.se_black_list',[])
25	>	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
26	>
27	>	### Temporary patch to automatically skip the ISB size check:
28	>	server=self.cfg_params.get('CRAB.server_name',None)
29	>	size = 9.5
30	>	if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
31	>	### D.S.
32	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
33
34		# number of jobs requested to be created, limit obj splitting
35		self.ncjobs = ncjobs
#	Line 33 | Line 42 | class Cmssw(JobType):
42		self.executable = ''
43		self.executable_arch = self.scram.getArch()
44		self.tgz_name = 'default.tgz'
36	–	self.additional_tgz_name = 'additional.tgz'
45		self.scriptName = 'CMSSW.sh'
46	<	self.pset = ''      #scrip use case Da
47	<	self.datasetPath = '' #scrip use case Da
46	>	self.pset = ''
47	>	self.datasetPath = ''
48
49		# set FJR file name
50		self.fjrFileName = 'crab_fjr.xml'
51
52		self.version = self.scram.getSWVersion()
53	<
54	<	#
55	<	# Try to block creation in case of arch/version mismatch
56	<	#
57	<
58	<	a = string.split(self.version, "_")
59	<
60	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
61	<	msg = "Warning: You are using %s version of CMSSW  with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
62	<	common.logger.message(msg)
55	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
53	>	version_array = self.version.split('_')
54	>	self.CMSSW_major = 0
55	>	self.CMSSW_minor = 0
56	>	self.CMSSW_patch = 0
57	>	try:
58	>	self.CMSSW_major = int(version_array[1])
59	>	self.CMSSW_minor = int(version_array[2])
60	>	self.CMSSW_patch = int(version_array[3])
61	>	except:
62	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
63		raise CrabException(msg)
64
59	–	common.taskDB.setDict('codeVersion',self.version)
60	–	self.setParam_('application', self.version)
61	–
65		### collect Data cards
66
67		if not cfg_params.has_key('CMSSW.datasetpath'):
68		msg = "Error: datasetpath not defined "
69		raise CrabException(msg)
70	+
71	+	### Temporary: added to remove input file control in the case of PU
72	+	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
73	+
74		tmp =  cfg_params['CMSSW.datasetpath']
75		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
76	<	if string.lower(tmp)=='none':
76	>
77	>	if tmp =='':
78	>	msg = "Error: datasetpath not defined "
79	>	raise CrabException(msg)
80	>	elif string.lower(tmp)=='none':
81		self.datasetPath = None
82		self.selectNoInput = 1
83		else:
84		self.datasetPath = tmp
85		self.selectNoInput = 0
86
76	–	# ML monitoring
77	–	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
78	–	if not self.datasetPath:
79	–	self.setParam_('dataset', 'None')
80	–	self.setParam_('owner', 'None')
81	–	else:
82	–	## SL what is supposed to fail here?
83	–	try:
84	–	datasetpath_split = self.datasetPath.split("/")
85	–	# standard style
86	–	self.setParam_('datasetFull', self.datasetPath)
87	–	self.setParam_('dataset', datasetpath_split[1])
88	–	self.setParam_('owner', datasetpath_split[2])
89	–	except:
90	–	self.setParam_('dataset', self.datasetPath)
91	–	self.setParam_('owner', self.datasetPath)
92	–
93	–	self.setParam_('taskId', common.taskDB.dict('taskId'))
94	–
87		self.dataTiers = []
88	<
88	>	self.debugWrap = ''
89	>	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
90	>	if self.debug_wrapper: self.debugWrap='--debug'
91		## now the application
92		self.executable = cfg_params.get('CMSSW.executable','cmsRun')
99	–	self.setParam_('exe', self.executable)
93		log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
94
95		if not cfg_params.has_key('CMSSW.pset'):
#	Line 117 | Line 110 | class Cmssw(JobType):
110		self.output_file_sandbox.append(self.fjrFileName)
111
112		# other output files to be returned via sandbox or copied to SE
113	+	outfileflag = False
114		self.output_file = []
115		tmp = cfg_params.get('CMSSW.output_file',None)
116		if tmp :
117	<	tmpOutFiles = string.split(tmp,',')
118	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
119	<	for tmp in tmpOutFiles:
120	<	tmp=string.strip(tmp)
127	<	self.output_file.append(tmp)
128	<	pass
129	<	else:
130	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
131	<	pass
117	>	self.output_file = [x.strip() for x in tmp.split(',')]
118	>	outfileflag = True #output found
119	>	#else:
120	>	#    log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
121
122		# script_exe file as additional file in inputSandbox
123		self.scriptExe = cfg_params.get('USER.script_exe',None)
124		if self.scriptExe :
125	<	if not os.path.isfile(self.scriptExe):
126	<	msg ="ERROR. file "+self.scriptExe+" not found"
127	<	raise CrabException(msg)
128	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
125	>	if not os.path.isfile(self.scriptExe):
126	>	msg ="ERROR. file "+self.scriptExe+" not found"
127	>	raise CrabException(msg)
128	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
129
141	–	#CarlosDaniele
130		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
131	<	msg ="Error. script_exe  not defined"
132	<	raise CrabException(msg)
131	>	msg ="Error. script_exe  not defined"
132	>	raise CrabException(msg)
133	>
134	>	# use parent files...
135	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
136
137		## additional input files
138		if cfg_params.has_key('USER.additional_input_files'):
#	Line 161 | Line 152 | class Cmssw(JobType):
152		if not os.path.exists(file):
153		raise CrabException("Additional input file not found: "+file)
154		pass
164	–	# fname = string.split(file, '/')[-1]
165	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
166	–	# shutil.copyfile(file, storedFile)
155		self.additional_inbox_files.append(string.strip(file))
156		pass
157		pass
#	Line 189 | Line 177 | class Cmssw(JobType):
177		if cfg_params.has_key('CMSSW.total_number_of_events'):
178		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
179		self.selectTotalNumberEvents = 1
180	+	if self.selectNumberOfJobs  == 1:
181	+	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
182	+	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
183	+	raise CrabException(msg)
184		else:
185		self.total_number_of_events = 0
186		self.selectTotalNumberEvents = 0
187
188	<	if self.pset != None: #CarlosDaniele
188	>	if self.pset != None:
189		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
190		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
191		raise CrabException(msg)
#	Line 202 | Line 194 | class Cmssw(JobType):
194		msg = 'Must specify  number_of_jobs.'
195		raise CrabException(msg)
196
197	<	## source seed for pythia
198	<	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
197	>	## New method of dealing with seeds
198	>	self.incrementSeeds = []
199	>	self.preserveSeeds = []
200	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
201	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
202	>	for tmp in tmpList:
203	>	tmp.strip()
204	>	self.preserveSeeds.append(tmp)
205	>	if cfg_params.has_key('CMSSW.increment_seeds'):
206	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
207	>	for tmp in tmpList:
208	>	tmp.strip()
209	>	self.incrementSeeds.append(tmp)
210
211	+	## FUTURE: Can remove in CRAB 2.4.0
212	+	self.sourceSeed    = cfg_params.get('CMSSW.pythia_seed',None)
213		self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
214	<
210	<	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
211	<
214	>	self.sourceSeedG4  = cfg_params.get('CMSSW.g4_seed',None)
215		self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
216	+	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
217	+	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
218	+	raise CrabException(msg)
219
220		self.firstRun = cfg_params.get('CMSSW.first_run',None)
221
216	–	if self.pset != None: #CarlosDaniele
217	–	import PsetManipulator as pp
218	–	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
219	–
222		# Copy/return
221	–
223		self.copy_data = int(cfg_params.get('USER.copy_data',0))
224		self.return_data = int(cfg_params.get('USER.return_data',0))
225
#	Line 234 | Line 235 | class Cmssw(JobType):
235		blockSites = self.DataDiscoveryAndLocation(cfg_params)
236		#DBSDLS-end
237
237	–	self.tgzNameWithPath = self.getTarBall(self.executable)
238	–
238		## Select Splitting
239		if self.selectNoInput:
240	<	if self.pset == None: #CarlosDaniele
240	>	if self.pset == None:
241		self.jobSplittingForScript()
242		else:
243		self.jobSplittingNoInput()
244	+	elif (cfg_params.get('CMSSW.noblockboundary',0)):
245	+	self.jobSplittingNoBlockBoundary(blockSites)
246		else:
247		self.jobSplittingByBlocks(blockSites)
248
249	<	# modify Pset
250	<	if self.pset != None: #CarlosDaniele
251	<	try:
252	<	if (self.datasetPath): # standard job
253	<	# allow to processa a fraction of events in a file
254	<	PsetEdit.inputModule("INPUTFILE")
255	<	PsetEdit.maxEvent(0)
256	<	PsetEdit.skipEvent(0)
257	<	else:  # pythia like job
249	>	# modify Pset only the first time
250	>	if isNew:
251	>	if self.pset != None:
252	>	import PsetManipulator as pp
253	>	PsetEdit = pp.PsetManipulator(self.pset)
254	>	try:
255	>	# Add FrameworkJobReport to parameter-set, set max events.
256	>	# Reset later for data jobs by writeCFG which does all modifications
257	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
258		PsetEdit.maxEvent(self.eventsPerJob)
259	<	if (self.firstRun):
260	<	PsetEdit.pythiaFirstRun(0)  #First Run
261	<	if (self.sourceSeed) :
262	<	PsetEdit.pythiaSeed(0)
263	<	if (self.sourceSeedVtx) :
264	<	PsetEdit.vtxSeed(0)
265	<	if (self.sourceSeedG4) :
266	<	PsetEdit.g4Seed(0)
267	<	if (self.sourceSeedMix) :
268	<	PsetEdit.mixSeed(0)
269	<	# add FrameworkJobReport to parameter-set
270	<	PsetEdit.addCrabFJR(self.fjrFileName)
271	<	PsetEdit.psetWriter(self.configFilename())
272	<	except:
273	<	msg='Error while manipuliating ParameterSet: exiting...'
274	<	raise CrabException(msg)
259	>	PsetEdit.psetWriter(self.configFilename())
260	>	## If present, add TFileService to output files
261	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
262	>	tfsOutput = PsetEdit.getTFileService()
263	>	if tfsOutput:
264	>	if tfsOutput in self.output_file:
265	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
266	>	else:
267	>	outfileflag = True #output found
268	>	self.output_file.append(tfsOutput)
269	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
270	>	pass
271	>	pass
272	>	## If present and requested, add PoolOutputModule to output files
273	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
274	>	edmOutput = PsetEdit.getPoolOutputModule()
275	>	if edmOutput:
276	>	if edmOutput in self.output_file:
277	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
278	>	else:
279	>	self.output_file.append(edmOutput)
280	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
281	>	pass
282	>	pass
283	>	except CrabException:
284	>	msg='Error while manipulating ParameterSet: exiting...'
285	>	raise CrabException(msg)
286	>	## Prepare inputSandbox TarBall (only the first time)
287	>	self.tgzNameWithPath = self.getTarBall(self.executable)
288
289		def DataDiscoveryAndLocation(self, cfg_params):
290
#	Line 283 | Line 297 | class Cmssw(JobType):
297		## Contact the DBS
298		common.logger.message("Contacting Data Discovery Services ...")
299		try:
300	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
300	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
301		self.pubdata.fetchDBSInfo()
302
303		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 299 | Line 313 | class Cmssw(JobType):
313		self.filesbyblock=self.pubdata.getFiles()
314		self.eventsbyblock=self.pubdata.getEventsPerBlock()
315		self.eventsbyfile=self.pubdata.getEventsPerFile()
316	+	self.parentFiles=self.pubdata.getParent()
317
318		## get max number of events
319	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
319	>	self.maxEvents=self.pubdata.getMaxEvents()
320
321		## Contact the DLS and build a list of sites hosting the fileblocks
322		try:
#	Line 325 | Line 340 | class Cmssw(JobType):
340
341		return sites
342
328	–	def setArgsList(self, argsList):
329	–	self.argsList = argsList
330	–
343		def jobSplittingByBlocks(self, blockSites):
344		"""
345		Perform job splitting. Jobs run over an integer number of files
#	Line 378 | Line 390 | class Cmssw(JobType):
390		else :
391		totalNumberOfJobs = self.ncjobs
392
381	–
393		blocks = blockSites.keys()
394		blockCount = 0
395		# Backup variable in case self.maxEvents counted events in a non-included block
#	Line 419 | Line 430 | class Cmssw(JobType):
430
431		# ---- Iterate over the files in the block until we've met the requested ---- #
432		# ---- total # of events or we've gone over all the files in this block  ---- #
433	+	pString=''
434		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
435		file = files[fileCount]
436	+	if self.useParent:
437	+	parent = self.parentFiles[file]
438	+	for f in parent :
439	+	pString += '\\\"' + f + '\\\"\,'
440	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
441	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
442		if newFile :
443		try:
444		numEventsInFile = self.eventsbyfile[file]
#	Line 433 | Line 451 | class Cmssw(JobType):
451		except KeyError:
452		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
453
454	<
454	>	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
455		# if less events in file remain than eventsPerJobRequested
456	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
456	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
457		# if last file in block
458		if ( fileCount == numFilesInBlock-1 ) :
459		# end job using last file, use remaining events in block
460		# close job and touch new file
461		fullString = parString[:-2]
462	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
462	>	if self.useParent:
463	>	fullParentString = pString[:-2]
464	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
465	>	else:
466	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
467		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
468		self.jobDestination.append(blockSites[block])
469		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 453 | Line 475 | class Cmssw(JobType):
475		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
476		jobSkipEventCount = 0
477		# reset file
478	+	pString = ""
479		parString = ""
480		filesEventCount = 0
481		newFile = 1
#	Line 465 | Line 488 | class Cmssw(JobType):
488		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
489		# close job and touch new file
490		fullString = parString[:-2]
491	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
491	>	if self.useParent:
492	>	fullParentString = pString[:-2]
493	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
494	>	else:
495	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
496		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
497		self.jobDestination.append(blockSites[block])
498		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 476 | Line 503 | class Cmssw(JobType):
503		eventsRemaining = eventsRemaining - eventsPerJobRequested
504		jobSkipEventCount = 0
505		# reset file
506	+	pString = ""
507		parString = ""
508		filesEventCount = 0
509		newFile = 1
#	Line 485 | Line 513 | class Cmssw(JobType):
513		else :
514		# close job but don't touch new file
515		fullString = parString[:-2]
516	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
516	>	if self.useParent:
517	>	fullParentString = pString[:-2]
518	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
519	>	else:
520	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
521		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
522		self.jobDestination.append(blockSites[block])
523		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 499 | Line 531 | class Cmssw(JobType):
531		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
532		# remove all but the last file
533		filesEventCount = self.eventsbyfile[file]
534	<	parString = ""
535	<	parString += '\\\"' + file + '\\\"\,'
534	>	if self.useParent:
535	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
536	>	parString = '\\\"' + file + '\\\"\,'
537		pass # END if
538		pass # END while (iterate over files in the block)
539		pass # END while (iterate over blocks in the dataset)
#	Line 520 | Line 553 | class Cmssw(JobType):
553		for block in blocks:
554		if block in jobsOfBlock.keys() :
555		blockCounter += 1
556	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
556	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
557	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
558		if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
559		noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
560		bloskNoSite.append( blockCounter )
#	Line 554 | Line 588 | class Cmssw(JobType):
588		self.list_of_args = list_of_lists
589		return
590
591	+	def jobSplittingNoBlockBoundary(self,blockSites):
592	+	"""
593	+	"""
594	+	# ---- Handle the possible job splitting configurations ---- #
595	+	if (self.selectTotalNumberEvents):
596	+	totalEventsRequested = self.total_number_of_events
597	+	if (self.selectEventsPerJob):
598	+	eventsPerJobRequested = self.eventsPerJob
599	+	if (self.selectNumberOfJobs):
600	+	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
601	+
602	+	# If user requested all the events in the dataset
603	+	if (totalEventsRequested == -1):
604	+	eventsRemaining=self.maxEvents
605	+	# If user requested more events than are in the dataset
606	+	elif (totalEventsRequested > self.maxEvents):
607	+	eventsRemaining = self.maxEvents
608	+	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
609	+	# If user requested less events than are in the dataset
610	+	else:
611	+	eventsRemaining = totalEventsRequested
612	+
613	+	# If user requested more events per job than are in the dataset
614	+	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
615	+	eventsPerJobRequested = self.maxEvents
616	+
617	+	# For user info at end
618	+	totalEventCount = 0
619	+
620	+	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
621	+	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
622	+
623	+	if (self.selectNumberOfJobs):
624	+	common.logger.message("May not create the exact number_of_jobs requested.")
625	+
626	+	if ( self.ncjobs == 'all' ) :
627	+	totalNumberOfJobs = 999999999
628	+	else :
629	+	totalNumberOfJobs = self.ncjobs
630	+
631	+	blocks = blockSites.keys()
632	+	blockCount = 0
633	+	# Backup variable in case self.maxEvents counted events in a non-included block
634	+	numBlocksInDataset = len(blocks)
635	+
636	+	jobCount = 0
637	+	list_of_lists = []
638	+
639	+	#AF
640	+	#AF do not reset input files and event count on block boundary
641	+	#AF
642	+	parString=""
643	+	filesEventCount = 0
644	+	#AF
645	+
646	+	# list tracking which jobs are in which jobs belong to which block
647	+	jobsOfBlock = {}
648	+	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
649	+	block = blocks[blockCount]
650	+	blockCount += 1
651	+	if block not in jobsOfBlock.keys() :
652	+	jobsOfBlock[block] = []
653	+
654	+	if self.eventsbyblock.has_key(block) :
655	+	numEventsInBlock = self.eventsbyblock[block]
656	+	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
657	+	files = self.filesbyblock[block]
658	+	numFilesInBlock = len(files)
659	+	if (numFilesInBlock <= 0):
660	+	continue
661	+	fileCount = 0
662	+	#AF
663	+	#AF do not reset input files and event count of block boundary
664	+	#AF
665	+	## ---- New block => New job ---- #
666	+	#parString = ""
667	+	# counter for number of events in files currently worked on
668	+	#filesEventCount = 0
669	+	#AF
670	+	# flag if next while loop should touch new file
671	+	newFile = 1
672	+	# job event counter
673	+	jobSkipEventCount = 0
674	+
675	+	# ---- Iterate over the files in the block until we've met the requested ---- #
676	+	# ---- total # of events or we've gone over all the files in this block  ---- #
677	+	pString=''
678	+	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
679	+	file = files[fileCount]
680	+	if self.useParent:
681	+	parent = self.parentFiles[file]
682	+	for f in parent :
683	+	pString += '\\\"' + f + '\\\"\,'
684	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
685	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
686	+	if newFile :
687	+	try:
688	+	numEventsInFile = self.eventsbyfile[file]
689	+	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
690	+	# increase filesEventCount
691	+	filesEventCount += numEventsInFile
692	+	# Add file to current job
693	+	parString += '\\\"' + file + '\\\"\,'
694	+	newFile = 0
695	+	except KeyError:
696	+	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
697	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
698	+	#common.logger.message("AF filesEventCount %s - jobSkipEventCount %s "%(filesEventCount,jobSkipEventCount))
699	+	# if less events in file remain than eventsPerJobRequested
700	+	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
701	+	#AF
702	+	#AF skip fileboundary part
703	+	#AF
704	+	# go to next file
705	+	newFile = 1
706	+	fileCount += 1
707	+	# if events in file equal to eventsPerJobRequested
708	+	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
709	+	# close job and touch new file
710	+	fullString = parString[:-2]
711	+	if self.useParent:
712	+	fullParentString = pString[:-2]
713	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
714	+	else:
715	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
716	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
717	+	self.jobDestination.append(blockSites[block])
718	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
719	+	jobsOfBlock[block].append(jobCount+1)
720	+	# reset counter
721	+	jobCount = jobCount + 1
722	+	totalEventCount = totalEventCount + eventsPerJobRequested
723	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
724	+	jobSkipEventCount = 0
725	+	# reset file
726	+	pString = ""
727	+	parString = ""
728	+	filesEventCount = 0
729	+	newFile = 1
730	+	fileCount += 1
731	+
732	+	# if more events in file remain than eventsPerJobRequested
733	+	else :
734	+	# close job but don't touch new file
735	+	fullString = parString[:-2]
736	+	if self.useParent:
737	+	fullParentString = pString[:-2]
738	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
739	+	else:
740	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
741	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
742	+	self.jobDestination.append(blockSites[block])
743	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
744	+	jobsOfBlock[block].append(jobCount+1)
745	+	# increase counter
746	+	jobCount = jobCount + 1
747	+	totalEventCount = totalEventCount + eventsPerJobRequested
748	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
749	+	# calculate skip events for last file
750	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
751	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
752	+	# remove all but the last file
753	+	filesEventCount = self.eventsbyfile[file]
754	+	if self.useParent:
755	+	for f in parent : pString += '\\\"' + f + '\\\"\,'
756	+	parString = '\\\"' + file + '\\\"\,'
757	+	pass # END if
758	+	pass # END while (iterate over files in the block)
759	+	pass # END while (iterate over blocks in the dataset)
760	+	self.ncjobs = self.total_number_of_jobs = jobCount
761	+	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
762	+	common.logger.message("eventsRemaining "+str(eventsRemaining))
763	+	common.logger.message("jobCount "+str(jobCount))
764	+	common.logger.message(" totalNumberOfJobs "+str(totalNumberOfJobs))
765	+	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
766	+	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
767	+
768	+	# screen output
769	+	screenOutput = "List of jobs and available destination sites:\n\n"
770	+
771	+	#AF
772	+	#AF   skip check on  block with no sites
773	+	#AF
774	+	self.list_of_args = list_of_lists
775	+
776	+	return
777	+
778	+
779	+
780		def jobSplittingNoInput(self):
781		"""
782		Perform job splitting based on number of event per job
#	Line 602 | Line 825 | class Cmssw(JobType):
825		if (self.firstRun):
826		## pythia first run
827		args.append(str(self.firstRun)+str(i))
605	–	if (self.sourceSeed):
606	–	args.append(str(self.sourceSeed)+str(i))
607	–	if (self.sourceSeedVtx):
608	–	## + vtx random seed
609	–	args.append(str(self.sourceSeedVtx)+str(i))
610	–	if (self.sourceSeedG4):
611	–	## + G4 random seed
612	–	args.append(str(self.sourceSeedG4)+str(i))
613	–	if (self.sourceSeedMix):
614	–	## + Mix random seed
615	–	args.append(str(self.sourceSeedMix)+str(i))
616	–	pass
617	–	pass
828		self.list_of_args.append(args)
619	–	pass
829
830		return
831
832
833	<	def jobSplittingForScript(self):#CarlosDaniele
833	>	def jobSplittingForScript(self):
834		"""
835		Perform job splitting based on number of job
836		"""
#	Line 637 | Line 846 | class Cmssw(JobType):
846		# argument is seed number.$i
847		self.list_of_args = []
848		for i in range(self.total_number_of_jobs):
640	–	## Since there is no input, any site is good
641	–	# self.jobDestination.append(["Any"])
849		self.jobDestination.append([""])
643	–	## no random seed
850		self.list_of_args.append([str(i)])
851		return
852
853	<	def split(self, jobParams):
853	>	def split(self, jobParams,firstJobID):
854
649	–	common.jobDB.load()
650	–	#### Fabio
855		njobs = self.total_number_of_jobs
856		arglist = self.list_of_args
857		# create the empty structure
858		for i in range(njobs):
859		jobParams.append("")
860
861	<	for job in range(njobs):
862	<	jobParams[job] = arglist[job]
863	<	# print str(arglist[job])
864	<	# print jobParams[job]
865	<	common.jobDB.setArguments(job, jobParams[job])
866	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
867	<	common.jobDB.setDestination(job, self.jobDestination[job])
861	>	listID=[]
862	>	listField=[]
863	>	for id in range(njobs):
864	>	job = id + int(firstJobID)
865	>	jobParams[id] = arglist[id]
866	>	listID.append(job+1)
867	>	job_ToSave ={}
868	>	concString = ' '
869	>	argu=''
870	>	if len(jobParams[id]):
871	>	argu +=   concString.join(jobParams[id] )
872	>	job_ToSave['arguments']= str(job+1)+' '+argu
873	>	job_ToSave['dlsDestination']= self.jobDestination[id]
874	>	listField.append(job_ToSave)
875	>	msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
876	>	+"                     Destination: "+str(self.jobDestination[id])
877	>	common.logger.debug(5,msg)
878	>	common._db.updateJob_(listID,listField)
879	>	self.argsList = (len(jobParams[0])+1)
880
665	–	common.jobDB.save()
881		return
882
668	–	def getJobTypeArguments(self, nj, sched):
669	–	result = ''
670	–	for i in common.jobDB.arguments(nj):
671	–	result=result+str(i)+" "
672	–	return result
673	–
883		def numberOfJobs(self):
675	–	# Fabio
884		return self.total_number_of_jobs
885
886		def getTarBall(self, exe):
887		"""
888		Return the TarBall with lib and exe
889		"""
890	<
683	<	# if it exist, just return it
684	<	#
685	<	# Marco. Let's start to use relative path for Boss XML files
686	<	#
687	<	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
890	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
891		if os.path.exists(self.tgzNameWithPath):
892		return self.tgzNameWithPath
893
#	Line 697 | Line 900 | class Cmssw(JobType):
900
901		# First of all declare the user Scram area
902		swArea = self.scram.getSWArea_()
700	–	#print "swArea = ", swArea
701	–	# swVersion = self.scram.getSWVersion()
702	–	# print "swVersion = ", swVersion
903		swReleaseTop = self.scram.getReleaseTop_()
704	–	#print "swReleaseTop = ", swReleaseTop
904
905		## check if working area is release top
906		if swReleaseTop == '' or swArea == swReleaseTop:
907	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
908		return
909
910		import tarfile
#	Line 746 | Line 946 | class Cmssw(JobType):
946		tar.add(module,moduleDir)
947
948		## Now check if any data dir(s) is present
949	<	swAreaLen=len(swArea)
950	<	for root, dirs, files in os.walk(swArea):
951	<	if "data" in dirs:
952	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
953	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
954	<
955	<	### Removed ProdAgent Api dependencies ###
956	<	### Add ProdAgent dir to tar
957	<	#paDir = 'ProdAgentApi'
958	<	#pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
959	<	#if os.path.isdir(pa):
960	<	#    tar.add(pa,paDir)
949	>	self.dataExist = False
950	>	todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
951	>	while len(todo_list):
952	>	entry, name = todo_list.pop()
953	>	if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
954	>	continue
955	>	if os.path.isdir(swArea+"/src/"+entry):
956	>	entryPath = entry + '/'
957	>	todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
958	>	if name == 'data':
959	>	self.dataExist=True
960	>	common.logger.debug(5,"data "+entry+" to be tarred")
961	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
962	>	pass
963	>	pass
964	>
965	>	### CMSSW ParameterSet
966	>	if not self.pset is None:
967	>	cfg_file = common.work_space.jobDir()+self.configFilename()
968	>	tar.add(cfg_file,self.configFilename())
969	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
970	>
971
972		## Add ProdCommon dir to tar
973	<	prodcommonDir = 'ProdCommon'
974	<	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
975	<	if os.path.isdir(prodcommonPath):
976	<	tar.add(prodcommonPath,prodcommonDir)
973	>	prodcommonDir = './'
974	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
975	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
976	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage']
977	>	for file in neededStuff:
978	>	tar.add(prodcommonPath+file,prodcommonDir+file)
979	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
980	>
981	>	##### ML stuff
982	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
983	>	path=os.environ['CRABDIR'] + '/python/'
984	>	for file in ML_file_list:
985	>	tar.add(path+file,file)
986	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
987	>
988	>	##### Utils
989	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
990	>	for file in Utils_file_list:
991	>	tar.add(path+file,file)
992	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
993
994	+	##### AdditionalFiles
995	+	tar.dereference=True
996	+	for file in self.additional_inbox_files:
997	+	tar.add(file,string.split(file,'/')[-1])
998	+	tar.dereference=False
999		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1000	+
1001		tar.close()
1002	<	except :
1003	<	raise CrabException('Could not create tar-ball')
1002	>	except IOError, exc:
1003	>	common.logger.write(str(exc))
1004	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1005	>	except tarfile.TarError, exc:
1006	>	common.logger.write(str(exc))
1007	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1008
1009		## check for tarball size
1010		tarballinfo = os.stat(self.tgzNameWithPath)
1011		if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
1012	<	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
1012	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
1013	>	+'MB input sandbox limit \n'
1014	>	msg += '      and not supported by the direct GRID submission system.\n'
1015	>	msg += '      Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
1016	>	msg += '      For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
1017	>	raise CrabException(msg)
1018
1019		## create tar-ball with ML stuff
779	–	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
780	–	try:
781	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
782	–	path=os.environ['CRABDIR'] + '/python/'
783	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
784	–	tar.add(path+file,file)
785	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
786	–	tar.close()
787	–	except :
788	–	raise CrabException('Could not create ML files tar-ball')
789	–
790	–	return
1020
1021	<	def additionalInputFileTgz(self):
793	<	"""
794	<	Put all additional files into a tar ball and return its name
795	<	"""
796	<	import tarfile
797	<	tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
798	<	tar = tarfile.open(tarName, "w:gz")
799	<	for file in self.additional_inbox_files:
800	<	tar.add(file,string.split(file,'/')[-1])
801	<	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
802	<	tar.close()
803	<	return tarName
804	<
805	<	def wsSetupEnvironment(self, nj):
1021	>	def wsSetupEnvironment(self, nj=0):
1022		"""
1023		Returns part of a job script which prepares
1024		the execution environment for the job 'nj'.
1025		"""
1026	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1027	+	psetName = 'pset.py'
1028	+	else:
1029	+	psetName = 'pset.cfg'
1030		# Prepare JobType-independent part
1031	<	txt = ''
1031	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
1032		txt += 'echo ">>> setup environment"\n'
1033		txt += 'if [ $middleware == LCG ]; then \n'
1034		txt += self.wsSetupCMSLCGEnvironment_()
1035		txt += 'elif [ $middleware == OSG ]; then\n'
1036		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
1037		txt += '    if [ ! $? == 0 ] ;then\n'
1038	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1039	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
1040	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
821	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
822	<	txt += '        exit 1\n'
1038	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1039	>	txt += '        job_exit_code=10016\n'
1040	>	txt += '        func_exit\n'
1041		txt += '    fi\n'
1042		txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
1043		txt += '\n'
#	Line 827 | Line 1045 | class Cmssw(JobType):
1045		txt += '    cd $WORKING_DIR\n'
1046		txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
1047		txt += self.wsSetupCMSOSGEnvironment_()
830	–	#txt += '    echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
831	–	#txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1048		txt += 'fi\n'
1049
1050		# Prepare JobType-specific part
#	Line 839 | Line 1055 | class Cmssw(JobType):
1055		txt += scram+' project CMSSW '+self.version+'\n'
1056		txt += 'status=$?\n'
1057		txt += 'if [ $status != 0 ] ; then\n'
1058	<	txt += '    echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
1059	<	txt += '    echo "JOB_EXIT_STATUS = 10034"\n'
1060	<	txt += '    echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
845	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
846	<	txt += '    if [ $middleware == OSG ]; then \n'
847	<	txt += '        cd $RUNTIME_AREA\n'
848	<	txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
849	<	txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
850	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
851	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
852	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
853	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
854	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
855	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
856	<	txt += '        fi\n'
857	<	txt += '    fi \n'
858	<	txt += '    exit 1 \n'
1058	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
1059	>	txt += '    job_exit_code=10034\n'
1060	>	txt += '    func_exit\n'
1061		txt += 'fi \n'
1062		txt += 'cd '+self.version+'\n'
861	–	########## FEDE FOR DBS2 ######################
1063		txt += 'SOFTWARE_DIR=`pwd`\n'
1064		txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
864	–	###############################################
865	–	### needed grep for bug in scramv1 ###
1065		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
1066	+	txt += 'if [ $? != 0 ] ; then\n'
1067	+	txt += '    echo "ERROR ==> Problem with the command: "\n'
1068	+	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
1069	+	txt += '    job_exit_code=10034\n'
1070	+	txt += '    func_exit\n'
1071	+	txt += 'fi \n'
1072		# Handle the arguments:
1073		txt += "\n"
1074		txt += "## number of arguments (first argument always jobnumber)\n"
1075		txt += "\n"
1076	<	txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
1076	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
1077		txt += "then\n"
1078	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
1079	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
1080	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
876	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
877	<	txt += '    if [ $middleware == OSG ]; then \n'
878	<	txt += '        cd $RUNTIME_AREA\n'
879	<	txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
880	<	txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
881	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
882	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
883	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
884	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
885	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
886	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
887	<	txt += '        fi\n'
888	<	txt += '    fi \n'
889	<	txt += "    exit 1\n"
1078	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
1079	>	txt += '    job_exit_code=50113\n'
1080	>	txt += "    func_exit\n"
1081		txt += "fi\n"
1082		txt += "\n"
1083
1084		# Prepare job-specific part
1085		job = common.job_list[nj]
895	–	### FEDE FOR DBS OUTPUT PUBLICATION
1086		if (self.datasetPath):
1087	+	self.primaryDataset = self.datasetPath.split("/")[1]
1088	+	DataTier = self.datasetPath.split("/")[2]
1089		txt += '\n'
1090		txt += 'DatasetPath='+self.datasetPath+'\n'
1091
1092	<	datasetpath_split = self.datasetPath.split("/")
1093	<
902	<	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
903	<	txt += 'DataTier='+datasetpath_split[2]+'\n'
1092	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
1093	>	txt += 'DataTier='+DataTier+'\n'
1094		txt += 'ApplicationFamily=cmsRun\n'
1095
1096		else:
1097	+	self.primaryDataset = 'null'
1098		txt += 'DatasetPath=MCDataTier\n'
1099		txt += 'PrimaryDataset=null\n'
1100		txt += 'DataTier=null\n'
1101		txt += 'ApplicationFamily=MCDataTier\n'
1102	<	if self.pset != None: #CarlosDaniele
1102	>	if self.pset != None:
1103		pset = os.path.basename(job.configFilename())
1104		txt += '\n'
1105		txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
1106		if (self.datasetPath): # standard job
1107	<	txt += 'InputFiles=${args[1]}\n'
1108	<	txt += 'MaxEvents=${args[2]}\n'
1109	<	txt += 'SkipEvents=${args[3]}\n'
1107	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
1108	>	if (self.useParent):
1109	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
1110	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
1111	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
1112	>	else:
1113	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
1114	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
1115		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1116	<	txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1116	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
1117		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
922	–	txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1118		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
924	–	txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1119		else:  # pythia like job
1120	<	seedIndex=1
1120	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
1121	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
1122	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
1123	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
1124		if (self.firstRun):
1125	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
1125	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
1126		txt += 'echo "FirstRun: <$FirstRun>"\n'
930	–	txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
931	–	seedIndex=seedIndex+1
1127
1128	<	if (self.sourceSeed):
934	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
935	<	txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
936	<	seedIndex=seedIndex+1
937	<	## the following seeds are not always present
938	<	if (self.sourceSeedVtx):
939	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
940	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
941	<	txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
942	<	seedIndex += 1
943	<	if (self.sourceSeedG4):
944	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
945	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
946	<	txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
947	<	seedIndex += 1
948	<	if (self.sourceSeedMix):
949	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
950	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
951	<	txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
952	<	seedIndex += 1
953	<	pass
954	<	pass
955	<	txt += 'mv -f '+pset+' pset.cfg\n'
1128	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
1129
957	–	if len(self.additional_inbox_files) > 0:
958	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
959	–	txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
960	–	txt += 'fi\n'
961	–	pass
1130
1131	<	if self.pset != None: #CarlosDaniele
1131	>	if self.pset != None:
1132	>	# FUTURE: Can simply for 2_1_x and higher
1133		txt += '\n'
1134	<	txt += 'echo "***** cat pset.cfg *********"\n'
1135	<	txt += 'cat pset.cfg\n'
1136	<	txt += 'echo "****** end pset.cfg ********"\n'
1137	<	txt += '\n'
1138	<	### FEDE FOR DBS OUTPUT PUBLICATION
1139	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1134	>	if self.debug_wrapper==True:
1135	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
1136	>	txt += 'cat ' + psetName + '\n'
1137	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
1138	>	txt += '\n'
1139	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1140	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
1141	>	else:
1142	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
1143		txt += 'echo "PSETHASH = $PSETHASH" \n'
972	–	##############
1144		txt += '\n'
1145		return txt
1146
1147	<	def wsBuildExe(self, nj=0):
1147	>	def wsUntarSoftware(self, nj=0):
1148		"""
1149		Put in the script the commands to build an executable
1150		or a library.
1151		"""
1152
1153	<	txt = ""
1153	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
1154
1155		if os.path.isfile(self.tgzNameWithPath):
1156		txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1157		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1158	+	if  self.debug_wrapper:
1159	+	txt += 'ls -Al \n'
1160		txt += 'untar_status=$? \n'
1161		txt += 'if [ $untar_status -ne 0 ]; then \n'
1162	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
1163	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
1164	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
992	<	txt += '   if [ $middleware == OSG ]; then \n'
993	<	txt += '       cd $RUNTIME_AREA\n'
994	<	txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
995	<	txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
996	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
997	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
998	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
999	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1000	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1001	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1002	<	txt += '       fi\n'
1003	<	txt += '   fi \n'
1004	<	txt += '   \n'
1005	<	txt += '   exit 1 \n'
1162	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
1163	>	txt += '   job_exit_code=$untar_status\n'
1164	>	txt += '   func_exit\n'
1165		txt += 'else \n'
1166		txt += '   echo "Successful untar" \n'
1167		txt += 'fi \n'
1168		txt += '\n'
1169	<	#### Removed ProdAgent API dependencies
1011	<	txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
1169	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1170		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1171	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1014	<	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
1171	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1172		txt += 'else\n'
1173	<	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1173	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1174		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1018	–	###################
1175		txt += 'fi\n'
1176		txt += '\n'
1177
#	Line 1023 | Line 1179 | class Cmssw(JobType):
1179
1180		return txt
1181
1182	<	def modifySteeringCards(self, nj):
1182	>	def wsBuildExe(self, nj=0):
1183		"""
1184	<	modify the card provided by the user,
1185	<	writing a new card into share dir
1184	>	Put in the script the commands to build an executable
1185	>	or a library.
1186		"""
1187
1188	+	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1189	+	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1190	+
1191	+	txt += 'rm -r lib/ module/ \n'
1192	+	txt += 'mv $RUNTIME_AREA/lib/ . \n'
1193	+	txt += 'mv $RUNTIME_AREA/module/ . \n'
1194	+	if self.dataExist == True:
1195	+	txt += 'rm -r src/ \n'
1196	+	txt += 'mv $RUNTIME_AREA/src/ . \n'
1197	+	if len(self.additional_inbox_files)>0:
1198	+	for file in self.additional_inbox_files:
1199	+	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1200	+	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1201	+	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1202	+
1203	+	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1204	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1205	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1206	+	txt += 'else\n'
1207	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1208	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1209	+	txt += 'fi\n'
1210	+	txt += '\n'
1211	+
1212	+	return txt
1213	+
1214	+
1215		def executableName(self):
1216	<	if self.scriptExe: #CarlosDaniele
1216	>	if self.scriptExe:
1217		return "sh "
1218		else:
1219		return self.executable
1220
1221		def executableArgs(self):
1222	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1223		if self.scriptExe:#CarlosDaniele
1224		return   self.scriptExe + " $NJob"
1225		else:
1226	<	# if >= CMSSW_1_5_X, add -j crab_fjr.xml
1227	<	version_array = self.scram.getSWVersion().split('_')
1228	<	major = 0
1229	<	minor = 0
1230	<	try:
1231	<	major = int(version_array[1])
1232	<	minor = int(version_array[2])
1233	<	except:
1050	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1051	<	raise CrabException(msg)
1052	<	if major >= 1 and minor >= 5 :
1053	<	return " -j " + self.fjrFileName + " -p pset.cfg"
1226	>	ex_args = ""
1227	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1228	>	# Framework job report
1229	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1230	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1231	>	# Type of config file
1232	>	if self.CMSSW_major >= 2 :
1233	>	ex_args += " -p pset.py"
1234		else:
1235	<	return " -p pset.cfg"
1235	>	ex_args += " -p pset.cfg"
1236	>	return ex_args
1237
1238		def inputSandbox(self, nj):
1239		"""
1240		Returns a list of filenames to be put in JDL input sandbox.
1241		"""
1242		inp_box = []
1062	–	# # dict added to delete duplicate from input sandbox file list
1063	–	# seen = {}
1064	–	## code
1243		if os.path.isfile(self.tgzNameWithPath):
1244		inp_box.append(self.tgzNameWithPath)
1245	<	if os.path.isfile(self.MLtgzfile):
1068	<	inp_box.append(self.MLtgzfile)
1069	<	## config
1070	<	if not self.pset is None:
1071	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1072	<	## additional input files
1073	<	tgz = self.additionalInputFileTgz()
1074	<	inp_box.append(tgz)
1245	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
1246		return inp_box
1247
1248		def outputSandbox(self, nj):
#	Line 1083 | Line 1254 | class Cmssw(JobType):
1254		## User Declared output files
1255		for out in (self.output_file+self.output_file_sandbox):
1256		n_out = nj + 1
1257	<	out_box.append(self.numberFile_(out,str(n_out)))
1257	>	out_box.append(numberFile(out,str(n_out)))
1258		return out_box
1259
1089	–	def prepareSteeringCards(self):
1090	–	"""
1091	–	Make initial modifications of the user's steering card file.
1092	–	"""
1093	–	return
1260
1261		def wsRenameOutput(self, nj):
1262		"""
1263		Returns part of a job script which renames the produced files.
1264		"""
1265
1266	<	txt = '\n'
1266	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1267		txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1268		txt += 'echo ">>> current directory content:"\n'
1269	<	txt += 'ls \n'
1269	>	if self.debug_wrapper:
1270	>	txt += 'ls -Al\n'
1271		txt += '\n'
1272
1106	–	txt += 'output_exit_status=0\n'
1107	–
1108	–	for fileWithSuffix in (self.output_file_sandbox):
1109	–	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1110	–	txt += '\n'
1111	–	txt += '# check output file\n'
1112	–	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1113	–	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1114	–	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1115	–	txt += 'else\n'
1116	–	txt += '    exit_status=60302\n'
1117	–	txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1118	–	if common.scheduler.name() == 'CONDOR_G':
1119	–	txt += '    if [ $middleware == OSG ]; then \n'
1120	–	txt += '        echo "prepare dummy output file"\n'
1121	–	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1122	–	txt += '    fi \n'
1123	–	txt += 'fi\n'
1124	–
1273		for fileWithSuffix in (self.output_file):
1274	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1274	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1275		txt += '\n'
1276		txt += '# check output file\n'
1277		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
#	Line 1134 | Line 1282 | class Cmssw(JobType):
1282		txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1283		txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1284		txt += 'else\n'
1285	<	txt += '    exit_status=60302\n'
1286	<	txt += '    echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1287	<	txt += '    echo "JOB_EXIT_STATUS = $exit_status"\n'
1140	<	txt += '    output_exit_status=$exit_status\n'
1141	<	if common.scheduler.name() == 'CONDOR_G':
1285	>	txt += '    job_exit_code=60302\n'
1286	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1287	>	if common.scheduler.name().upper() == 'CONDOR_G':
1288		txt += '    if [ $middleware == OSG ]; then \n'
1289		txt += '        echo "prepare dummy output file"\n'
1290		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1146 | Line 1292 | class Cmssw(JobType):
1292		txt += 'fi\n'
1293		file_list = []
1294		for fileWithSuffix in (self.output_file):
1295	<	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1295	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
1296
1297	<	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1297	>	txt += 'file_list="'+string.join(file_list,',')+'"\n'
1298		txt += '\n'
1299		txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1300		txt += 'echo ">>> current directory content:"\n'
1301	<	txt += 'ls \n'
1301	>	if self.debug_wrapper:
1302	>	txt += 'ls -Al\n'
1303		txt += '\n'
1304		txt += 'cd $RUNTIME_AREA\n'
1305		txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1306		return txt
1307
1161	–	def numberFile_(self, file, txt):
1162	–	"""
1163	–	append _'txt' before last extension of a file
1164	–	"""
1165	–	p = string.split(file,".")
1166	–	# take away last extension
1167	–	name = p[0]
1168	–	for x in p[1:-1]:
1169	–	name=name+"."+x
1170	–	# add "_txt"
1171	–	if len(p)>1:
1172	–	ext = p[len(p)-1]
1173	–	result = name + '_' + txt + "." + ext
1174	–	else:
1175	–	result = name + '_' + txt
1176	–
1177	–	return result
1178	–
1308		def getRequirements(self, nj=[]):
1309		"""
1310		return job requirements to add to jdl files
#	Line 1185 | Line 1314 | class Cmssw(JobType):
1314		req='Member("VO-cms-' + \
1315		self.version + \
1316		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1317	<	## SL add requirement for OS version only if SL4
1189	<	#reSL4 = re.compile( r'slc4' )
1190	<	if self.executable_arch: # and reSL4.search(self.executable_arch):
1317	>	if self.executable_arch:
1318		req+=' && Member("VO-cms-' + \
1319		self.executable_arch + \
1320		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1321
1322		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1323	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1324	+	req += ' && other.GlueCEStateStatus == "Production" '
1325
1326		return req
1327
1328		def configFilename(self):
1329		""" return the config filename """
1330	<	return self.name()+'.cfg'
1330	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1331	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1332	>	return self.name()+'.py'
1333	>	else:
1334	>	return self.name()+'.cfg'
1335
1336		def wsSetupCMSOSGEnvironment_(self):
1337		"""
1338		Returns part of a job script which is prepares
1339		the execution environment and which is common for all CMS jobs.
1340		"""
1341	<	txt = '    echo ">>> setup CMS OSG environment:"\n'
1341	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1342	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
1343		txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1344		txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1345		txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
#	Line 1213 | Line 1347 | class Cmssw(JobType):
1347		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1348		txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1349		txt += '    else\n'
1350	<	txt += '        echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1351	<	txt += '        echo "JOB_EXIT_STATUS = 10020"\n'
1352	<	txt += '        echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1219	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1220	<	txt += '\n'
1221	<	txt += '        cd $RUNTIME_AREA\n'
1222	<	txt += '        echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1223	<	txt += '        echo ">>> Remove working directory: $WORKING_DIR"\n'
1224	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
1225	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
1226	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1227	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1228	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1229	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1230	<	txt += '        fi\n'
1231	<	txt += '\n'
1232	<	txt += '        exit 1\n'
1350	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1351	>	txt += '        job_exit_code=10020\n'
1352	>	txt += '        func_exit\n'
1353		txt += '    fi\n'
1354		txt += '\n'
1355	<	txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1355	>	txt += '    echo "==> setup cms environment ok"\n'
1356		txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1357
1358		return txt
1359
1240	–	### OLI_DANIELE
1360		def wsSetupCMSLCGEnvironment_(self):
1361		"""
1362		Returns part of a job script which is prepares
1363		the execution environment and which is common for all CMS jobs.
1364		"""
1365	<	txt = '    echo ">>> setup CMS LCG environment:"\n'
1365	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1366	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1367		txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1368		txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1369		txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1370		txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1371	<	txt += '        echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1372	<	txt += '        echo "JOB_EXIT_STATUS = 10031" \n'
1373	<	txt += '        echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1254	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1255	<	txt += '        exit 1\n'
1371	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1372	>	txt += '        job_exit_code=10031\n'
1373	>	txt += '        func_exit\n'
1374		txt += '    else\n'
1375		txt += '        echo "Sourcing environment... "\n'
1376		txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1377	<	txt += '            echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1378	<	txt += '            echo "JOB_EXIT_STATUS = 10020"\n'
1379	<	txt += '            echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1262	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1263	<	txt += '            exit 1\n'
1377	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1378	>	txt += '            job_exit_code=10020\n'
1379	>	txt += '            func_exit\n'
1380		txt += '        fi\n'
1381		txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1382		txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1383		txt += '        result=$?\n'
1384		txt += '        if [ $result -ne 0 ]; then\n'
1385	<	txt += '            echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1386	<	txt += '            echo "JOB_EXIT_STATUS = 10032"\n'
1387	<	txt += '            echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1272	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1273	<	txt += '            exit 1\n'
1385	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1386	>	txt += '            job_exit_code=10032\n'
1387	>	txt += '            func_exit\n'
1388		txt += '        fi\n'
1389		txt += '    fi\n'
1390		txt += '    \n'
1391	<	txt += '    echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1391	>	txt += '    echo "==> setup cms environment ok"\n'
1392		return txt
1393
1394	<	### FEDE FOR DBS OUTPUT PUBLICATION
1281	<	def modifyReport(self, nj):
1394	>	def wsModifyReport(self, nj):
1395		"""
1396		insert the part of the script that modifies the FrameworkJob Report
1397		"""
1398	<
1399	<	txt = ''
1287	<	try:
1288	<	publish_data = int(self.cfg_params['USER.publish_data'])
1289	<	except KeyError:
1290	<	publish_data = 0
1398	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1399	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1400		if (publish_data == 1):
1292	–	txt += 'echo ">>> Modify Job Report:" \n'
1293	–	################ FEDE FOR DBS2 #############################################
1294	–	#txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1295	–	txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1296	–	#############################################################################
1297	–
1298	–	txt += 'if [ -z "$SE" ]; then\n'
1299	–	txt += '    SE="" \n'
1300	–	txt += 'fi \n'
1301	–	txt += 'if [ -z "$SE_PATH" ]; then\n'
1302	–	txt += '    SE_PATH="" \n'
1303	–	txt += 'fi \n'
1304	–	txt += 'echo "SE = $SE"\n'
1305	–	txt += 'echo "SE_PATH = $SE_PATH"\n'
1401
1402		processedDataset = self.cfg_params['USER.publish_data_name']
1403	<	txt += 'ProcessedDataset='+processedDataset+'\n'
1404	<	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1405	<	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1406	<	#### FEDE: added slash in LFN ##############
1403	>
1404	>	txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
1405	>	txt += '    FOR_LFN=$LFNBaseName\n'
1406	>	txt += 'else\n'
1407		txt += '    FOR_LFN=/copy_problems/ \n'
1408	<	txt += 'else \n'
1409	<	txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1410	<	#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1411	<	txt += '    FOR_LFN=/store$tmp \n'
1412	<	txt += 'fi \n'
1408	>	txt += '    SE=""\n'
1409	>	txt += '    SE_PATH=""\n'
1410	>	txt += 'fi\n'
1411	>
1412	>	txt += 'echo ">>> Modify Job Report:" \n'
1413	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1414	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1415	>	#txt += 'ProcessedDataset=$procDataset \n'
1416		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1417	+	txt += 'echo "SE = $SE"\n'
1418	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1419		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1420		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1421	<	txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1422	<	txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1423	<
1421	>	args = '$RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' \
1422	>	'$USER-$ProcessedDataset-$PSETHASH $ApplicationFamily '+ \
1423	>	'  $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH'
1424	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1425	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1426		txt += 'modifyReport_result=$?\n'
1325	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1427		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1428	<	txt += '    exit_status=1\n'
1429	<	txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1428	>	txt += '    modifyReport_result=70500\n'
1429	>	txt += '    job_exit_code=$modifyReport_result\n'
1430	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1431	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1432		txt += 'else\n'
1433	<	txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1433	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1434		txt += 'fi\n'
1332	–	else:
1333	–	txt += 'echo "no data publication required"\n'
1435		return txt
1436
1437	<	def cleanEnv(self):
1438	<	txt = ''
1439	<	txt += 'if [ $middleware == OSG ]; then\n'
1440	<	txt += '    cd $RUNTIME_AREA\n'
1441	<	txt += '    echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1442	<	txt += '    echo ">>> Remove working directory: $WORKING_DIR"\n'
1443	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1444	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1445	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1446	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1447	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1448	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1437	>	def wsParseFJR(self):
1438	>	"""
1439	>	Parse the FrameworkJobReport to obtain useful infos
1440	>	"""
1441	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1442	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1443	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1444	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1445	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1446	>	if self.debug_wrapper :
1447	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1448	>	txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1449	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1450	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1451	>	txt += '            cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1452	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1453	>	txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1454	>	txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1455	>	txt += '        else\n'
1456	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1457	>	txt += '        fi\n'
1458	>	txt += '    else\n'
1459	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1460	>	txt += '    fi\n'
1461	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1462	>	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1463	>	txt += '      echo ">>> Executable succeded  $executable_exit_status"\n'
1464	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1465	>	# VERIFY PROCESSED DATA
1466	>	txt += '      echo ">>> Verify list of processed files:"\n'
1467	>	txt += '      echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n'
1468	>	txt += '      python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1469	>	txt += '      cat input-files.txt  | sort | uniq > tmp.txt\n'
1470	>	txt += '      mv tmp.txt input-files.txt\n'
1471	>	txt += '      echo "cat input-files.txt"\n'
1472	>	txt += '      echo "----------------------"\n'
1473	>	txt += '      cat input-files.txt\n'
1474	>	txt += '      cat processed-files.txt | sort | uniq > tmp.txt\n'
1475	>	txt += '      mv tmp.txt processed-files.txt\n'
1476	>	txt += '      echo "----------------------"\n'
1477	>	txt += '      echo "cat processed-files.txt"\n'
1478	>	txt += '      echo "----------------------"\n'
1479	>	txt += '      cat processed-files.txt\n'
1480	>	txt += '      echo "----------------------"\n'
1481	>	txt += '      diff -q input-files.txt processed-files.txt\n'
1482	>	txt += '      fileverify_status=$?\n'
1483	>	txt += '      if [ $fileverify_status -ne 0 ]; then\n'
1484	>	txt += '         executable_exit_status=30001\n'
1485	>	txt += '         echo "ERROR ==> not all input files processed"\n'
1486	>	txt += '         echo "      ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1487	>	txt += '         echo "      ==> diff input-files.txt processed-files.txt"\n'
1488	>	txt += '      fi\n'
1489	>	txt += '    elif [ $executable_exit_status -ne 0 ] || [ $executable_exit_status -ne 50015 ] || [ $executable_exit_status -ne 50017 ];then\n'
1490	>	txt += '      echo ">>> Executable failed  $executable_exit_status"\n'
1491	>	txt += '      echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1492	>	txt += '      echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1493	>	txt += '      job_exit_code=$executable_exit_status\n'
1494	>	txt += '      func_exit\n'
1495		txt += '    fi\n'
1496	+	txt += '\n'
1497	+	txt += 'else\n'
1498	+	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1499		txt += 'fi\n'
1500		txt += '\n'
1501	+	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1502	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1503	+	txt += 'job_exit_code=$executable_exit_status\n'
1504	+
1505		return txt
1506
1507		def setParam_(self, param, value):
#	Line 1365 | Line 1519 | class Cmssw(JobType):
1519		nd[e]=0
1520		return nd.keys()
1521
1522	<
1369	<	def checkOut(self, limit):
1522	>	def outList(self):
1523		"""
1524		check the dimension of the output files
1525		"""
1526	<	txt = 'echo ">>> Starting output sandbox limit check :"\n'
1527	<	allOutFiles = ""
1526	>	txt = ''
1527	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1528		listOutFiles = []
1529	<	txt += 'stdoutFile=`ls *stdout` \n'
1530	<	txt += 'stderrFile=`ls *stderr` \n'
1529	>	stdout = 'CMSSW_$NJob.stdout'
1530	>	stderr = 'CMSSW_$NJob.stderr'
1531		if (self.return_data == 1):
1532	<	for fileOut in (self.output_file+self.output_file_sandbox):
1533	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile"
1534	<	else:
1535	<	for fileOut in (self.output_file_sandbox):
1536	<	txt += 'echo " '+fileOut+'";\n'
1537	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob') + " $stdoutFile $stderrFile"
1538	<	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1539	<	txt += 'ls -gGhrta;\n'
1540	<	txt += 'sum=0;\n'
1541	<	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1542	<	txt += '    if [ -e $file ]; then\n'
1543	<	txt += '        tt=`ls -gGrta $file | awk \'{ print $3 }\'`\n'
1391	<	txt += '        sum=`expr $sum + $tt`\n'
1392	<	txt += '    else\n'
1393	<	txt += '        echo "WARNING: output file $file not found!"\n'
1394	<	txt += '    fi\n'
1395	<	txt += 'done\n'
1396	<	txt += 'echo "Total Output dimension: $sum";\n'
1397	<	txt += 'limit='+str(limit)+';\n'
1398	<	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1399	<	txt += 'if [ $limit -lt $sum ]; then\n'
1400	<	txt += '    echo "WARNING: output files have to big size - something will be lost;"\n'
1401	<	txt += '    echo "         checking the output file sizes..."\n'
1402	<	txt += '    tot=0;\n'
1403	<	txt += '    for filefile in '+str(allOutFiles)+' ; do\n'
1404	<	txt += '        dimFile=`ls -gGrta $filefile | awk \'{ print $3 }\';`\n'
1405	<	txt += '        tot=`expr $tot + $tt`;\n'
1406	<	txt += '        if [ $limit -lt $dimFile ]; then\n'
1407	<	txt += '            echo "deleting file: $filefile";\n'
1408	<	txt += '            rm -f $filefile\n'
1409	<	txt += '        elif [ $limit -lt $tot ]; then\n'
1410	<	txt += '            echo "deleting file: $filefile";\n'
1411	<	txt += '            rm -f $filefile\n'
1412	<	txt += '        else\n'
1413	<	txt += '            echo "saving file: $filefile"\n'
1414	<	txt += '        fi\n'
1415	<	txt += '    done\n'
1416	<
1417	<	txt += '    ls -agGhrt;\n'
1418	<	txt += '    echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1419	<	txt += '    echo "JOB_EXIT_STATUS = 70000";\n'
1420	<	txt += '    exit_status=70000;\n'
1421	<	txt += 'else'
1422	<	txt += '    echo "Total Output dimension $sum is fine.";\n'
1423	<	txt += 'fi\n'
1424	<	txt += 'echo "Ending output sandbox limit check"\n'
1532	>	for file in (self.output_file+self.output_file_sandbox):
1533	>	listOutFiles.append(numberFile(file, '$NJob'))
1534	>	listOutFiles.append(stdout)
1535	>	listOutFiles.append(stderr)
1536	>	else:
1537	>	for file in (self.output_file_sandbox):
1538	>	listOutFiles.append(numberFile(file, '$NJob'))
1539	>	listOutFiles.append(stdout)
1540	>	listOutFiles.append(stderr)
1541	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1542	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1543	>	txt += 'export filesToCheck\n'
1544		return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines