[ViewVC] Diff of: cvsroot/COMP/CRAB/python/cms

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.137 by slacapra, Fri Nov 16 11:09:31 2007 UTC vs.
Revision 1.267 by spiga, Wed Jan 14 14:10:23 2009 UTC

#	Line 2 \| Line 2 \| from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	<	from BlackWhiteListParser import BlackWhiteListParser
5	>	from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
6		import common
7		import Scram
8
9		import os, string, glob
10
11		class Cmssw(JobType):
12	<	def __init__(self, cfg_params, ncjobs):
12	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15	+	self.skip_blocks = skip_blocks
16	+	self.argsList = []
17
18		self._params = {}
19		self.cfg_params = cfg_params
20
21		# init BlackWhiteListParser
22	<	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23	<
24	<	try:
25	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
26	<	except KeyError:
27	<	self.MaxTarBallSize = 9.5
22	>	seWhiteList = cfg_params.get('EDG.se_white_list',[])
23	>	seBlackList = cfg_params.get('EDG.se_black_list',[])
24	>	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
25	>
26	>	### Temporary patch to automatically skip the ISB size check:
27	>	server=self.cfg_params.get('CRAB.server_name',None)
28	>	size = 9.5
29	>	if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
30	>	### D.S.
31	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
32
33		# number of jobs requested to be created, limit obj splitting
34		self.ncjobs = ncjobs
#	Line 35 \| Line 41 \| class Cmssw(JobType):
41		self.executable = ''
42		self.executable_arch = self.scram.getArch()
43		self.tgz_name = 'default.tgz'
38	–	self.additional_tgz_name = 'additional.tgz'
44		self.scriptName = 'CMSSW.sh'
45	<	self.pset = '' #scrip use case Da
46	<	self.datasetPath = '' #scrip use case Da
45	>	self.pset = ''
46	>	self.datasetPath = ''
47
48		# set FJR file name
49		self.fjrFileName = 'crab_fjr.xml'
50
51		self.version = self.scram.getSWVersion()
52	+	version_array = self.version.split('_')
53	+	self.CMSSW_major = 0
54	+	self.CMSSW_minor = 0
55	+	self.CMSSW_patch = 0
56	+	try:
57	+	self.CMSSW_major = int(version_array[1])
58	+	self.CMSSW_minor = int(version_array[2])
59	+	self.CMSSW_patch = int(version_array[3])
60	+	except:
61	+	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
62	+	raise CrabException(msg)
63
64	<	#
49	<	# Try to block creation in case of arch/version mismatch
50	<	#
51	<
52	<	a = string.split(self.version, "_")
64	>	### collect Data cards
65
66	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
67	<	msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
56	<	common.logger.message(msg)
57	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
66	>	if not cfg_params.has_key('CMSSW.datasetpath'):
67	>	msg = "Error: datasetpath not defined "
68		raise CrabException(msg)
69
70	<	common.taskDB.setDict('codeVersion',self.version)
71	<	self.setParam_('application', self.version)
70	>	### Temporary: added to remove input file control in the case of PU
71	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
72
73	<	### collect Data cards
73	>	tmp = cfg_params['CMSSW.datasetpath']
74	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75
76	<	try:
67	<	tmp = cfg_params['CMSSW.datasetpath']
68	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
69	<	if string.lower(tmp)=='none':
70	<	self.datasetPath = None
71	<	self.selectNoInput = 1
72	<	else:
73	<	self.datasetPath = tmp
74	<	self.selectNoInput = 0
75	<	except KeyError:
76	>	if tmp =='':
77		msg = "Error: datasetpath not defined "
78		raise CrabException(msg)
79	<
80	<	# ML monitoring
81	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
81	<	if not self.datasetPath:
82	<	self.setParam_('dataset', 'None')
83	<	self.setParam_('owner', 'None')
79	>	elif string.lower(tmp)=='none':
80	>	self.datasetPath = None
81	>	self.selectNoInput = 1
82		else:
83	<	try:
84	<	datasetpath_split = self.datasetPath.split("/")
87	<	# standard style
88	<	self.setParam_('datasetFull', self.datasetPath)
89	<	self.setParam_('dataset', datasetpath_split[1])
90	<	self.setParam_('owner', datasetpath_split[2])
91	<	except:
92	<	self.setParam_('dataset', self.datasetPath)
93	<	self.setParam_('owner', self.datasetPath)
94	<
95	<	self.setTaskid_()
96	<	self.setParam_('taskId', self.cfg_params['taskId'])
83	>	self.datasetPath = tmp
84	>	self.selectNoInput = 0
85
86		self.dataTiers = []
87	<
87	>	self.debugWrap = ''
88	>	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
89	>	if self.debug_wrapper: self.debugWrap='--debug'
90		## now the application
91	<	try:
92	<	self.executable = cfg_params['CMSSW.executable']
93	<	self.setParam_('exe', self.executable)
94	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
105	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
106	<	log.debug(3,msg)
107	<	except KeyError:
108	<	self.executable = 'cmsRun'
109	<	self.setParam_('exe', self.executable)
110	<	msg = "User executable not defined. Use cmsRun"
111	<	log.debug(3,msg)
112	<	pass
91	>	self.managedGenerators = ['madgraph','comphep']
92	>	self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
93	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
94	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
95
96	<	try:
115	<	self.pset = cfg_params['CMSSW.pset']
116	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
117	<	if self.pset.lower() != 'none' :
118	<	if (not os.path.exists(self.pset)):
119	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
120	<	else:
121	<	self.pset = None
122	<	except KeyError:
96	>	if not cfg_params.has_key('CMSSW.pset'):
97		raise CrabException("PSet file missing. Cannot run cmsRun ")
98	+	self.pset = cfg_params['CMSSW.pset']
99	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
100	+	if self.pset.lower() != 'none' :
101	+	if (not os.path.exists(self.pset)):
102	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
103	+	else:
104	+	self.pset = None
105
106		# output files
107		## stuff which must be returned always via sandbox
#	Line 130 \| Line 111 \| class Cmssw(JobType):
111		self.output_file_sandbox.append(self.fjrFileName)
112
113		# other output files to be returned via sandbox or copied to SE
114	<	try:
115	<	self.output_file = []
116	<	tmp = cfg_params['CMSSW.output_file']
117	<	if tmp != '':
118	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
119	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
120	<	for tmp in tmpOutFiles:
121	<	tmp=string.strip(tmp)
141	<	self.output_file.append(tmp)
142	<	pass
143	<	else:
144	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
145	<	pass
146	<	pass
147	<	except KeyError:
148	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
149	<	pass
114	>	outfileflag = False
115	>	self.output_file = []
116	>	tmp = cfg_params.get('CMSSW.output_file',None)
117	>	if tmp :
118	>	self.output_file = [x.strip() for x in tmp.split(',')]
119	>	outfileflag = True #output found
120	>	#else:
121	>	# log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
122
123		# script_exe file as additional file in inputSandbox
124	<	try:
125	<	self.scriptExe = cfg_params['USER.script_exe']
126	<	if self.scriptExe != '':
127	<	if not os.path.isfile(self.scriptExe):
128	<	msg ="ERROR. file "+self.scriptExe+" not found"
129	<	raise CrabException(msg)
158	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
159	<	except KeyError:
160	<	self.scriptExe = ''
124	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
125	>	if self.scriptExe :
126	>	if not os.path.isfile(self.scriptExe):
127	>	msg ="ERROR. file "+self.scriptExe+" not found"
128	>	raise CrabException(msg)
129	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
130
162	–	#CarlosDaniele
131		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
132	<	msg ="Error. script_exe not defined"
133	<	raise CrabException(msg)
132	>	msg ="Error. script_exe not defined"
133	>	raise CrabException(msg)
134	>
135	>	# use parent files...
136	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
137
138		## additional input files
139	<	try:
139	>	if cfg_params.has_key('USER.additional_input_files'):
140		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
141		for tmp in tmpAddFiles:
142		tmp = string.strip(tmp)
#	Line 182 \| Line 153 \| class Cmssw(JobType):
153		if not os.path.exists(file):
154		raise CrabException("Additional input file not found: "+file)
155		pass
185	–	# fname = string.split(file, '/')[-1]
186	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
187	–	# shutil.copyfile(file, storedFile)
156		self.additional_inbox_files.append(string.strip(file))
157		pass
158		pass
159		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
160	<	except KeyError:
193	<	pass
194	<
195	<	# files per job
196	<	try:
197	<	if (cfg_params['CMSSW.files_per_jobs']):
198	<	raise CrabException("files_per_jobs no longer supported. Quitting.")
199	<	except KeyError:
200	<	pass
160	>	pass
161
162		## Events per job
163	<	try:
163	>	if cfg_params.has_key('CMSSW.events_per_job'):
164		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
165		self.selectEventsPerJob = 1
166	<	except KeyError:
166	>	else:
167		self.eventsPerJob = -1
168		self.selectEventsPerJob = 0
169
170		## number of jobs
171	<	try:
171	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
172		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
173		self.selectNumberOfJobs = 1
174	<	except KeyError:
174	>	else:
175		self.theNumberOfJobs = 0
176		self.selectNumberOfJobs = 0
177
178	<	try:
178	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
179		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
180		self.selectTotalNumberEvents = 1
181	<	except KeyError:
181	>	if self.selectNumberOfJobs == 1:
182	>	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
183	>	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
184	>	raise CrabException(msg)
185	>	else:
186		self.total_number_of_events = 0
187		self.selectTotalNumberEvents = 0
188
189	<	if self.pset != None: #CarlosDaniele
189	>	if self.pset != None:
190		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
191		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
192		raise CrabException(msg)
#	Line 231 \| Line 195 \| class Cmssw(JobType):
195		msg = 'Must specify number_of_jobs.'
196		raise CrabException(msg)
197
198	<	## source seed for pythia
199	<	try:
200	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
201	<	except KeyError:
202	<	self.sourceSeed = None
203	<	common.logger.debug(5,"No seed given")
204	<
205	<	try:
206	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
207	<	except KeyError:
208	<	self.sourceSeedVtx = None
209	<	common.logger.debug(5,"No vertex seed given")
198	>	## New method of dealing with seeds
199	>	self.incrementSeeds = []
200	>	self.preserveSeeds = []
201	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
202	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
203	>	for tmp in tmpList:
204	>	tmp.strip()
205	>	self.preserveSeeds.append(tmp)
206	>	if cfg_params.has_key('CMSSW.increment_seeds'):
207	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
208	>	for tmp in tmpList:
209	>	tmp.strip()
210	>	self.incrementSeeds.append(tmp)
211	>
212	>	## FUTURE: Can remove in CRAB 2.4.0
213	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
214	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
215	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
216	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
217	>	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
218	>	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
219	>	raise CrabException(msg)
220
221	<	try:
248	<	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
249	<	except KeyError:
250	<	self.sourceSeedG4 = None
251	<	common.logger.debug(5,"No g4 sim hits seed given")
221	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
222
223	<	try:
224	<	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
225	<	except KeyError:
256	<	self.sourceSeedMix = None
257	<	common.logger.debug(5,"No mix seed given")
258	<
259	<	try:
260	<	self.firstRun = int(cfg_params['CMSSW.first_run'])
261	<	except KeyError:
262	<	self.firstRun = None
263	<	common.logger.debug(5,"No first run given")
264	<	if self.pset != None: #CarlosDaniele
265	<	import PsetManipulator as pp
266	<	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
223	>	# Copy/return
224	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
225	>	self.return_data = int(cfg_params.get('USER.return_data',0))
226
227		#DBSDLS-start
228		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
#	Line 276 \| Line 235 \| class Cmssw(JobType):
235		if self.datasetPath:
236		blockSites = self.DataDiscoveryAndLocation(cfg_params)
237		#DBSDLS-end
238	<
239	<	self.tgzNameWithPath = self.getTarBall(self.executable)
281	<
238	>
239	>	noBboundary = int(cfg_params.get('CMSSW.no_block_boundary',0))
240		## Select Splitting
241		if self.selectNoInput:
242	<	if self.pset == None: #CarlosDaniele
242	>	if self.pset == None:
243		self.jobSplittingForScript()
244		else:
245		self.jobSplittingNoInput()
246	+	elif noBboundary == 1:
247	+	self.jobSplittingNoBlockBoundary(blockSites)
248		else:
249		self.jobSplittingByBlocks(blockSites)
250
251	<	# modify Pset
252	<	if self.pset != None: #CarlosDaniele
253	<	try:
254	<	if (self.datasetPath): # standard job
255	<	# allow to processa a fraction of events in a file
256	<	PsetEdit.inputModule("INPUTFILE")
257	<	PsetEdit.maxEvent(0)
258	<	PsetEdit.skipEvent(0)
259	<	else: # pythia like job
251	>	# modify Pset only the first time
252	>	if isNew:
253	>	if self.pset != None:
254	>	import PsetManipulator as pp
255	>	PsetEdit = pp.PsetManipulator(self.pset)
256	>	try:
257	>	# Add FrameworkJobReport to parameter-set, set max events.
258	>	# Reset later for data jobs by writeCFG which does all modifications
259	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
260		PsetEdit.maxEvent(self.eventsPerJob)
261	<	if (self.firstRun):
262	<	PsetEdit.pythiaFirstRun(0) #First Run
263	<	if (self.sourceSeed) :
264	<	PsetEdit.pythiaSeed(0)
265	<	if (self.sourceSeedVtx) :
266	<	PsetEdit.vtxSeed(0)
267	<	if (self.sourceSeedG4) :
268	<	PsetEdit.g4Seed(0)
269	<	if (self.sourceSeedMix) :
270	<	PsetEdit.mixSeed(0)
271	<	# add FrameworkJobReport to parameter-set
272	<	PsetEdit.addCrabFJR(self.fjrFileName)
273	<	PsetEdit.psetWriter(self.configFilename())
274	<	except:
275	<	msg='Error while manipuliating ParameterSet: exiting...'
276	<	raise CrabException(msg)
261	>	PsetEdit.skipEvent(0)
262	>	PsetEdit.psetWriter(self.configFilename())
263	>	## If present, add TFileService to output files
264	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
265	>	tfsOutput = PsetEdit.getTFileService()
266	>	if tfsOutput:
267	>	if tfsOutput in self.output_file:
268	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
269	>	else:
270	>	outfileflag = True #output found
271	>	self.output_file.append(tfsOutput)
272	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
273	>	pass
274	>	pass
275	>	## If present and requested, add PoolOutputModule to output files
276	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
277	>	edmOutput = PsetEdit.getPoolOutputModule()
278	>	if edmOutput:
279	>	if edmOutput in self.output_file:
280	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
281	>	else:
282	>	self.output_file.append(edmOutput)
283	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
284	>	pass
285	>	pass
286	>	except CrabException:
287	>	msg='Error while manipulating ParameterSet: exiting...'
288	>	raise CrabException(msg)
289	>	## Prepare inputSandbox TarBall (only the first time)
290	>	self.tgzNameWithPath = self.getTarBall(self.executable)
291
292		def DataDiscoveryAndLocation(self, cfg_params):
293
#	Line 326 \| Line 300 \| class Cmssw(JobType):
300		## Contact the DBS
301		common.logger.message("Contacting Data Discovery Services ...")
302		try:
303	<
330	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
303	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
304		self.pubdata.fetchDBSInfo()
305
306		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 343 \| Line 316 \| class Cmssw(JobType):
316		self.filesbyblock=self.pubdata.getFiles()
317		self.eventsbyblock=self.pubdata.getEventsPerBlock()
318		self.eventsbyfile=self.pubdata.getEventsPerFile()
319	+	self.parentFiles=self.pubdata.getParent()
320
321		## get max number of events
322	<	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
322	>	self.maxEvents=self.pubdata.getMaxEvents()
323
324		## Contact the DLS and build a list of sites hosting the fileblocks
325		try:
326		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
327		dataloc.fetchDLSInfo()
328	+
329		except DataLocation.DataLocationError , ex:
330		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
331		raise CrabException(msg)
332
333
334		sites = dataloc.getSites()
335	+	if len(sites)==0:
336	+	msg = 'ERROR ***: no location for any of the blocks of this dataset: \n\t %s \n'%datasetPath
337	+	msg += "\tMaybe the dataset is located only at T1's (or at T0), where analysis jobs are not allowed\n"
338	+	msg += "\tPlease check DataDiscovery page https://cmsweb.cern.ch/dbs_discovery/\n"
339	+	raise CrabException(msg)
340	+
341		allSites = []
342		listSites = sites.values()
343		for listSite in listSites:
#	Line 419 \| Line 400 \| class Cmssw(JobType):
400		else :
401		totalNumberOfJobs = self.ncjobs
402
422	–
403		blocks = blockSites.keys()
404		blockCount = 0
405		# Backup variable in case self.maxEvents counted events in a non-included block
#	Line 460 \| Line 440 \| class Cmssw(JobType):
440
441		# ---- Iterate over the files in the block until we've met the requested ---- #
442		# ---- total # of events or we've gone over all the files in this block ---- #
443	+	pString=''
444		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
445		file = files[fileCount]
446	+	if self.useParent:
447	+	parent = self.parentFiles[file]
448	+	for f in parent :
449	+	pString += '\\\"' + f + '\\\"\,'
450	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
451	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
452		if newFile :
453		try:
454		numEventsInFile = self.eventsbyfile[file]
#	Line 474 \| Line 461 \| class Cmssw(JobType):
461		except KeyError:
462		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
463
464	<
464	>	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
465		# if less events in file remain than eventsPerJobRequested
466	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
466	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
467		# if last file in block
468		if ( fileCount == numFilesInBlock-1 ) :
469		# end job using last file, use remaining events in block
470		# close job and touch new file
471		fullString = parString[:-2]
472	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
472	>	if self.useParent:
473	>	fullParentString = pString[:-2]
474	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
475	>	else:
476	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
477		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
478		self.jobDestination.append(blockSites[block])
479		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 494 \| Line 485 \| class Cmssw(JobType):
485		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
486		jobSkipEventCount = 0
487		# reset file
488	+	pString = ""
489		parString = ""
490		filesEventCount = 0
491		newFile = 1
#	Line 506 \| Line 498 \| class Cmssw(JobType):
498		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
499		# close job and touch new file
500		fullString = parString[:-2]
501	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
501	>	if self.useParent:
502	>	fullParentString = pString[:-2]
503	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
504	>	else:
505	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
506		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
507		self.jobDestination.append(blockSites[block])
508		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 517 \| Line 513 \| class Cmssw(JobType):
513		eventsRemaining = eventsRemaining - eventsPerJobRequested
514		jobSkipEventCount = 0
515		# reset file
516	+	pString = ""
517		parString = ""
518		filesEventCount = 0
519		newFile = 1
#	Line 526 \| Line 523 \| class Cmssw(JobType):
523		else :
524		# close job but don't touch new file
525		fullString = parString[:-2]
526	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
526	>	if self.useParent:
527	>	fullParentString = pString[:-2]
528	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
529	>	else:
530	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
531		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
532		self.jobDestination.append(blockSites[block])
533		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 540 \| Line 541 \| class Cmssw(JobType):
541		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
542		# remove all but the last file
543		filesEventCount = self.eventsbyfile[file]
544	<	parString = ""
545	<	parString += '\\\"' + file + '\\\"\,'
544	>	if self.useParent:
545	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
546	>	parString = '\\\"' + file + '\\\"\,'
547		pass # END if
548		pass # END while (iterate over files in the block)
549		pass # END while (iterate over blocks in the dataset)
#	Line 561 \| Line 563 \| class Cmssw(JobType):
563		for block in blocks:
564		if block in jobsOfBlock.keys() :
565		blockCounter += 1
566	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
566	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
567	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
568		if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
569		noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
570		bloskNoSite.append( blockCounter )
#	Line 581 \| Line 584 \| class Cmssw(JobType):
584		for range_jobs in noSiteBlock:
585		msg += str(range_jobs) + virgola
586		msg += '\n will not be submitted and this block of data can not be analyzed!\n'
587	+	if self.cfg_params.has_key('EDG.se_white_list'):
588	+	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
589	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
590	+	msg += 'Please check if the dataset is available at this site!)\n'
591	+	if self.cfg_params.has_key('EDG.ce_white_list'):
592	+	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
593	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
594	+	msg += 'Please check if the dataset is available at this site!)\n'
595	+
596		common.logger.message(msg)
597
598		self.list_of_args = list_of_lists
599		return
600
601	+	def jobSplittingNoBlockBoundary(self,blockSites):
602	+	"""
603	+	"""
604	+	# ---- Handle the possible job splitting configurations ---- #
605	+	if (self.selectTotalNumberEvents):
606	+	totalEventsRequested = self.total_number_of_events
607	+	if (self.selectEventsPerJob):
608	+	eventsPerJobRequested = self.eventsPerJob
609	+	if (self.selectNumberOfJobs):
610	+	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
611	+
612	+	# If user requested all the events in the dataset
613	+	if (totalEventsRequested == -1):
614	+	eventsRemaining=self.maxEvents
615	+	# If user requested more events than are in the dataset
616	+	elif (totalEventsRequested > self.maxEvents):
617	+	eventsRemaining = self.maxEvents
618	+	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
619	+	# If user requested less events than are in the dataset
620	+	else:
621	+	eventsRemaining = totalEventsRequested
622	+
623	+	# If user requested more events per job than are in the dataset
624	+	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
625	+	eventsPerJobRequested = self.maxEvents
626	+
627	+	# For user info at end
628	+	totalEventCount = 0
629	+
630	+	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
631	+	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
632	+
633	+	if (self.selectNumberOfJobs):
634	+	common.logger.message("May not create the exact number_of_jobs requested.")
635	+
636	+	if ( self.ncjobs == 'all' ) :
637	+	totalNumberOfJobs = 999999999
638	+	else :
639	+	totalNumberOfJobs = self.ncjobs
640	+
641	+	blocks = blockSites.keys()
642	+	blockCount = 0
643	+	# Backup variable in case self.maxEvents counted events in a non-included block
644	+	numBlocksInDataset = len(blocks)
645	+
646	+	jobCount = 0
647	+	list_of_lists = []
648	+
649	+	#AF
650	+	#AF do not reset input files and event count on block boundary
651	+	#AF
652	+	parString=""
653	+	filesEventCount = 0
654	+	#AF
655	+
656	+	# list tracking which jobs are in which jobs belong to which block
657	+	jobsOfBlock = {}
658	+	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
659	+	block = blocks[blockCount]
660	+	blockCount += 1
661	+	if block not in jobsOfBlock.keys() :
662	+	jobsOfBlock[block] = []
663	+
664	+	if self.eventsbyblock.has_key(block) :
665	+	numEventsInBlock = self.eventsbyblock[block]
666	+	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
667	+	files = self.filesbyblock[block]
668	+	numFilesInBlock = len(files)
669	+	if (numFilesInBlock <= 0):
670	+	continue
671	+	fileCount = 0
672	+	#AF
673	+	#AF do not reset input files and event count of block boundary
674	+	#AF
675	+	## ---- New block => New job ---- #
676	+	#parString = ""
677	+	# counter for number of events in files currently worked on
678	+	#filesEventCount = 0
679	+	#AF
680	+	# flag if next while loop should touch new file
681	+	newFile = 1
682	+	# job event counter
683	+	jobSkipEventCount = 0
684	+
685	+	# ---- Iterate over the files in the block until we've met the requested ---- #
686	+	# ---- total # of events or we've gone over all the files in this block ---- #
687	+	pString=''
688	+	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
689	+	file = files[fileCount]
690	+	if self.useParent:
691	+	parent = self.parentFiles[file]
692	+	for f in parent :
693	+	pString += '\\\"' + f + '\\\"\,'
694	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
695	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
696	+	if newFile :
697	+	try:
698	+	numEventsInFile = self.eventsbyfile[file]
699	+	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
700	+	# increase filesEventCount
701	+	filesEventCount += numEventsInFile
702	+	# Add file to current job
703	+	parString += '\\\"' + file + '\\\"\,'
704	+	newFile = 0
705	+	except KeyError:
706	+	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
707	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
708	+	#common.logger.message("AF filesEventCount %s - jobSkipEventCount %s "%(filesEventCount,jobSkipEventCount))
709	+	# if less events in file remain than eventsPerJobRequested
710	+	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
711	+	#AF
712	+	#AF skip fileboundary part
713	+	#AF
714	+	# go to next file
715	+	newFile = 1
716	+	fileCount += 1
717	+	# if events in file equal to eventsPerJobRequested
718	+	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
719	+	# close job and touch new file
720	+	fullString = parString[:-2]
721	+	if self.useParent:
722	+	fullParentString = pString[:-2]
723	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
724	+	else:
725	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
726	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
727	+	self.jobDestination.append(blockSites[block])
728	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
729	+	jobsOfBlock[block].append(jobCount+1)
730	+	# reset counter
731	+	jobCount = jobCount + 1
732	+	totalEventCount = totalEventCount + eventsPerJobRequested
733	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
734	+	jobSkipEventCount = 0
735	+	# reset file
736	+	pString = ""
737	+	parString = ""
738	+	filesEventCount = 0
739	+	newFile = 1
740	+	fileCount += 1
741	+
742	+	# if more events in file remain than eventsPerJobRequested
743	+	else :
744	+	# close job but don't touch new file
745	+	fullString = parString[:-2]
746	+	if self.useParent:
747	+	fullParentString = pString[:-2]
748	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
749	+	else:
750	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
751	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
752	+	self.jobDestination.append(blockSites[block])
753	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
754	+	jobsOfBlock[block].append(jobCount+1)
755	+	# increase counter
756	+	jobCount = jobCount + 1
757	+	totalEventCount = totalEventCount + eventsPerJobRequested
758	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
759	+	# calculate skip events for last file
760	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
761	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
762	+	# remove all but the last file
763	+	filesEventCount = self.eventsbyfile[file]
764	+	if self.useParent:
765	+	for f in parent : pString += '\\\"' + f + '\\\"\,'
766	+	parString = '\\\"' + file + '\\\"\,'
767	+	pass # END if
768	+	pass # END while (iterate over files in the block)
769	+	pass # END while (iterate over blocks in the dataset)
770	+	self.ncjobs = self.total_number_of_jobs = jobCount
771	+	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
772	+	common.logger.message("eventsRemaining "+str(eventsRemaining))
773	+	common.logger.message("jobCount "+str(jobCount))
774	+	common.logger.message(" totalNumberOfJobs "+str(totalNumberOfJobs))
775	+	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
776	+	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
777	+
778	+	# screen output
779	+	screenOutput = "List of jobs and available destination sites:\n\n"
780	+
781	+	#AF
782	+	#AF skip check on block with no sites
783	+	#AF
784	+	self.list_of_args = list_of_lists
785	+
786	+	return
787	+
788	+
789	+
790		def jobSplittingNoInput(self):
791		"""
792		Perform job splitting based on number of event per job
#	Line 629 \| Line 830 \| class Cmssw(JobType):
830		self.list_of_args = []
831		for i in range(self.total_number_of_jobs):
832		## Since there is no input, any site is good
632	–	# self.jobDestination.append(["Any"])
833		self.jobDestination.append([""]) #must be empty to write correctly the xml
834		args=[]
835		if (self.firstRun):
836	<	## pythia first run
637	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
836	>	## pythia first run
837		args.append(str(self.firstRun)+str(i))
838	<	else:
839	<	## no first run
840	<	#self.list_of_args.append([str(i)])
841	<	args.append(str(i))
842	<	if (self.sourceSeed):
843	<	args.append(str(self.sourceSeed)+str(i))
645	<	if (self.sourceSeedVtx):
646	<	## + vtx random seed
647	<	args.append(str(self.sourceSeedVtx)+str(i))
648	<	if (self.sourceSeedG4):
649	<	## + G4 random seed
650	<	args.append(str(self.sourceSeedG4)+str(i))
651	<	if (self.sourceSeedMix):
652	<	## + Mix random seed
653	<	args.append(str(self.sourceSeedMix)+str(i))
654	<	pass
655	<	pass
838	>	if (self.generator in self.managedGenerators):
839	>	if (self.generator == 'comphep' and i == 0):
840	>	# COMPHEP is brain-dead and wants event #'s like 1,100,200,300
841	>	args.append('1')
842	>	else:
843	>	args.append(str(i*self.eventsPerJob))
844		self.list_of_args.append(args)
657	–	pass
658	–
659	–	# print self.list_of_args
660	–
845		return
846
847
848	<	def jobSplittingForScript(self):#CarlosDaniele
848	>	def jobSplittingForScript(self):
849		"""
850		Perform job splitting based on number of job
851		"""
#	Line 677 \| Line 861 \| class Cmssw(JobType):
861		# argument is seed number.$i
862		self.list_of_args = []
863		for i in range(self.total_number_of_jobs):
680	–	## Since there is no input, any site is good
681	–	# self.jobDestination.append(["Any"])
864		self.jobDestination.append([""])
683	–	## no random seed
865		self.list_of_args.append([str(i)])
866		return
867
868	<	def split(self, jobParams):
868	>	def split(self, jobParams,firstJobID):
869
689	–	common.jobDB.load()
690	–	#### Fabio
870		njobs = self.total_number_of_jobs
871		arglist = self.list_of_args
872	+	if njobs==0:
873	+	raise CrabException("Ask to split "+str(njobs)+" jobs: aborting")
874	+
875		# create the empty structure
876		for i in range(njobs):
877		jobParams.append("")
878
879	<	for job in range(njobs):
880	<	jobParams[job] = arglist[job]
881	<	# print str(arglist[job])
882	<	# print jobParams[job]
883	<	common.jobDB.setArguments(job, jobParams[job])
884	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
885	<	common.jobDB.setDestination(job, self.jobDestination[job])
879	>	listID=[]
880	>	listField=[]
881	>	for id in range(njobs):
882	>	job = id + int(firstJobID)
883	>	jobParams[id] = arglist[id]
884	>	listID.append(job+1)
885	>	job_ToSave ={}
886	>	concString = ' '
887	>	argu=''
888	>	if len(jobParams[id]):
889	>	argu += concString.join(jobParams[id] )
890	>	job_ToSave['arguments']= str(job+1)+' '+argu
891	>	job_ToSave['dlsDestination']= self.jobDestination[id]
892	>	listField.append(job_ToSave)
893	>	msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \
894	>	+" Destination: "+str(self.jobDestination[id])
895	>	common.logger.debug(5,msg)
896	>	common._db.updateJob_(listID,listField)
897	>	self.argsList = (len(jobParams[0])+1)
898
705	–	common.jobDB.save()
899		return
900
708	–	def getJobTypeArguments(self, nj, sched):
709	–	result = ''
710	–	for i in common.jobDB.arguments(nj):
711	–	result=result+str(i)+" "
712	–	return result
713	–
901		def numberOfJobs(self):
715	–	# Fabio
902		return self.total_number_of_jobs
903
904		def getTarBall(self, exe):
905		"""
906		Return the TarBall with lib and exe
907		"""
908	<
723	<	# if it exist, just return it
724	<	#
725	<	# Marco. Let's start to use relative path for Boss XML files
726	<	#
727	<	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
908	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
909		if os.path.exists(self.tgzNameWithPath):
910		return self.tgzNameWithPath
911
#	Line 737 \| Line 918 \| class Cmssw(JobType):
918
919		# First of all declare the user Scram area
920		swArea = self.scram.getSWArea_()
740	–	#print "swArea = ", swArea
741	–	# swVersion = self.scram.getSWVersion()
742	–	# print "swVersion = ", swVersion
921		swReleaseTop = self.scram.getReleaseTop_()
744	–	#print "swReleaseTop = ", swReleaseTop
922
923		## check if working area is release top
924		if swReleaseTop == '' or swArea == swReleaseTop:
925	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
926		return
927
928		import tarfile
#	Line 773 \| Line 951 \| class Cmssw(JobType):
951		pass
952
953		## Now get the libraries: only those in local working area
954	+	tar.dereference=True
955		libDir = 'lib'
956		lib = swArea+'/' +libDir
957		common.logger.debug(5,"lib "+lib+" to be tarred")
#	Line 784 \| Line 963 \| class Cmssw(JobType):
963		module = swArea + '/' + moduleDir
964		if os.path.isdir(module):
965		tar.add(module,moduleDir)
966	+	tar.dereference=False
967
968		## Now check if any data dir(s) is present
969	<	swAreaLen=len(swArea)
970	<	for root, dirs, files in os.walk(swArea):
971	<	if "data" in dirs:
972	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
973	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
974	<
975	<	## Add ProdAgent dir to tar
976	<	paDir = 'ProdAgentApi'
977	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
978	<	if os.path.isdir(pa):
979	<	tar.add(pa,paDir)
980	<
981	<	### FEDE FOR DBS PUBLICATION
982	<	## Add PRODCOMMON dir to tar
983	<	prodcommonDir = 'ProdCommon'
984	<	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
985	<	if os.path.isdir(prodcommonPath):
986	<	tar.add(prodcommonPath,prodcommonDir)
987	<	#############################
969	>	self.dataExist = False
970	>	todo_list = [(i, i) for i in os.listdir(swArea+"/src")]
971	>	while len(todo_list):
972	>	entry, name = todo_list.pop()
973	>	if name.startswith('crab_0_') or name.startswith('.') or name == 'CVS':
974	>	continue
975	>	if os.path.isdir(swArea+"/src/"+entry):
976	>	entryPath = entry + '/'
977	>	todo_list += [(entryPath + i, i) for i in os.listdir(swArea+"/src/"+entry)]
978	>	if name == 'data':
979	>	self.dataExist=True
980	>	common.logger.debug(5,"data "+entry+" to be tarred")
981	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
982	>	pass
983	>	pass
984	>
985	>	### CMSSW ParameterSet
986	>	if not self.pset is None:
987	>	cfg_file = common.work_space.jobDir()+self.configFilename()
988	>	tar.add(cfg_file,self.configFilename())
989	>
990	>
991	>	## Add ProdCommon dir to tar
992	>	prodcommonDir = './'
993	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
994	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
995	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage']
996	>	for file in neededStuff:
997	>	tar.add(prodcommonPath+file,prodcommonDir+file)
998	>
999	>	##### ML stuff
1000	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
1001	>	path=os.environ['CRABDIR'] + '/python/'
1002	>	for file in ML_file_list:
1003	>	tar.add(path+file,file)
1004	>
1005	>	##### Utils
1006	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
1007	>	for file in Utils_file_list:
1008	>	tar.add(path+file,file)
1009	>
1010	>	##### AdditionalFiles
1011	>	tar.dereference=True
1012	>	for file in self.additional_inbox_files:
1013	>	tar.add(file,string.split(file,'/')[-1])
1014	>	tar.dereference=False
1015	>	common.logger.debug(5,"Files in "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1016
809	–	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1017		tar.close()
1018	<	except :
1019	<	raise CrabException('Could not create tar-ball')
1018	>	except IOError, exc:
1019	>	common.logger.write(str(exc))
1020	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1021	>	except tarfile.TarError, exc:
1022	>	common.logger.write(str(exc))
1023	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1024
1025		## check for tarball size
1026		tarballinfo = os.stat(self.tgzNameWithPath)
1027		if ( tarballinfo.st_size > self.MaxTarBallSize10241024 ) :
1028	<	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
1028	>	msg = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
1029	>	+'MB input sandbox limit \n'
1030	>	msg += ' and not supported by the direct GRID submission system.\n'
1031	>	msg += ' Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
1032	>	msg += ' For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
1033	>	raise CrabException(msg)
1034
1035		## create tar-ball with ML stuff
820	–	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
821	–	try:
822	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
823	–	path=os.environ['CRABDIR'] + '/python/'
824	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
825	–	tar.add(path+file,file)
826	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
827	–	tar.close()
828	–	except :
829	–	raise CrabException('Could not create ML files tar-ball')
830	–
831	–	return
832	–
833	–	def additionalInputFileTgz(self):
834	–	"""
835	–	Put all additional files into a tar ball and return its name
836	–	"""
837	–	import tarfile
838	–	tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
839	–	tar = tarfile.open(tarName, "w:gz")
840	–	for file in self.additional_inbox_files:
841	–	tar.add(file,string.split(file,'/')[-1])
842	–	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
843	–	tar.close()
844	–	return tarName
1036
1037	<	def wsSetupEnvironment(self, nj):
1037	>	def wsSetupEnvironment(self, nj=0):
1038		"""
1039		Returns part of a job script which prepares
1040		the execution environment for the job 'nj'.
1041		"""
1042	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1043	+	psetName = 'pset.py'
1044	+	else:
1045	+	psetName = 'pset.cfg'
1046		# Prepare JobType-independent part
1047	<	txt = ''
1047	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
1048		txt += 'echo ">>> setup environment"\n'
1049		txt += 'if [ $middleware == LCG ]; then \n'
1050		txt += self.wsSetupCMSLCGEnvironment_()
1051		txt += 'elif [ $middleware == OSG ]; then\n'
1052		txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
1053		txt += ' if [ ! $? == 0 ] ;then\n'
1054	<	txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1055	<	txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
1056	<	txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
862	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
863	<	txt += ' exit 1\n'
1054	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1055	>	txt += ' job_exit_code=10016\n'
1056	>	txt += ' func_exit\n'
1057		txt += ' fi\n'
1058		txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
1059		txt += '\n'
#	Line 868 \| Line 1061 \| class Cmssw(JobType):
1061		txt += ' cd $WORKING_DIR\n'
1062		txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
1063		txt += self.wsSetupCMSOSGEnvironment_()
871	–	#txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
872	–	#txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1064		txt += 'fi\n'
1065
1066		# Prepare JobType-specific part
#	Line 880 \| Line 1071 \| class Cmssw(JobType):
1071		txt += scram+' project CMSSW '+self.version+'\n'
1072		txt += 'status=$?\n'
1073		txt += 'if [ $status != 0 ] ; then\n'
1074	<	txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
1075	<	txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
1076	<	txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
886	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
887	<	txt += ' if [ $middleware == OSG ]; then \n'
888	<	txt += ' cd $RUNTIME_AREA\n'
889	<	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
890	<	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
891	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
892	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
893	<	txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
894	<	txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
895	<	txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
896	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
897	<	txt += ' fi\n'
898	<	txt += ' fi \n'
899	<	txt += ' exit 1 \n'
1074	>	txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
1075	>	txt += ' job_exit_code=10034\n'
1076	>	txt += ' func_exit\n'
1077		txt += 'fi \n'
1078		txt += 'cd '+self.version+'\n'
902	–	########## FEDE FOR DBS2 ######################
1079		txt += 'SOFTWARE_DIR=`pwd`\n'
1080		txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
905	–	###############################################
906	–	### needed grep for bug in scramv1 ###
907	–	txt += scram+' runtime -sh\n'
1081		txt += 'eval `'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME`\n'
1082	<	txt += 'echo $PATH\n'
1083	<
1082	>	txt += 'if [ $? != 0 ] ; then\n'
1083	>	txt += ' echo "ERROR ==> Problem with the command: "\n'
1084	>	txt += ' echo "eval \`'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
1085	>	txt += ' job_exit_code=10034\n'
1086	>	txt += ' func_exit\n'
1087	>	txt += 'fi \n'
1088		# Handle the arguments:
1089		txt += "\n"
1090		txt += "## number of arguments (first argument always jobnumber)\n"
1091		txt += "\n"
1092	<	txt += "if [ $nargs -lt 2 ]\n"
1092	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
1093		txt += "then\n"
1094	<	txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
1095	<	txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
1096	<	txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
920	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
921	<	txt += ' if [ $middleware == OSG ]; then \n'
922	<	txt += ' cd $RUNTIME_AREA\n'
923	<	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
924	<	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
925	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
926	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
927	<	txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
928	<	txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
929	<	txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
930	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
931	<	txt += ' fi\n'
932	<	txt += ' fi \n'
933	<	txt += " exit 1\n"
1094	>	txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
1095	>	txt += ' job_exit_code=50113\n'
1096	>	txt += " func_exit\n"
1097		txt += "fi\n"
1098		txt += "\n"
1099
1100		# Prepare job-specific part
1101		job = common.job_list[nj]
939	–	### FEDE FOR DBS OUTPUT PUBLICATION
1102		if (self.datasetPath):
1103	+	self.primaryDataset = self.datasetPath.split("/")[1]
1104	+	DataTier = self.datasetPath.split("/")[2]
1105		txt += '\n'
1106		txt += 'DatasetPath='+self.datasetPath+'\n'
1107
1108	<	datasetpath_split = self.datasetPath.split("/")
1109	<
946	<	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
947	<	txt += 'DataTier='+datasetpath_split[2]+'\n'
1108	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
1109	>	txt += 'DataTier='+DataTier+'\n'
1110		txt += 'ApplicationFamily=cmsRun\n'
1111
1112		else:
1113	+	self.primaryDataset = 'null'
1114		txt += 'DatasetPath=MCDataTier\n'
1115		txt += 'PrimaryDataset=null\n'
1116		txt += 'DataTier=null\n'
1117		txt += 'ApplicationFamily=MCDataTier\n'
1118	<	if self.pset != None: #CarlosDaniele
1118	>	if self.pset != None:
1119		pset = os.path.basename(job.configFilename())
1120		txt += '\n'
1121		txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
1122		if (self.datasetPath): # standard job
1123	<	txt += 'InputFiles=${args[1]}\n'
1124	<	txt += 'MaxEvents=${args[2]}\n'
1125	<	txt += 'SkipEvents=${args[3]}\n'
1123	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
1124	>	if (self.useParent):
1125	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
1126	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
1127	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
1128	>	else:
1129	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
1130	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
1131		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1132	<	txt += 'sed "s#\'INPUTFILE\'#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1132	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
1133		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
966	–	txt += 'sed "s#int32 input = 0#int32 input = $MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1134		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
968	–	txt += 'sed "s#uint32 skipEvents = 0#uint32 skipEvents = $SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1135		else: # pythia like job
1136	<	seedIndex=1
1136	>	argNum = 1
1137	>	txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
1138	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
1139	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
1140	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
1141		if (self.firstRun):
1142	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
1142	>	txt += 'export FirstRun=${args[%s]}\n' % argNum
1143		txt += 'echo "FirstRun: <$FirstRun>"\n'
1144	<	txt += 'sed "s#uint32 firstRun = 0#uint32 firstRun = $FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1145	<	seedIndex=seedIndex+1
1144	>	argNum += 1
1145	>	if (self.generator == 'madgraph'):
1146	>	txt += 'export FirstEvent=${args[%s]}\n' % argNum
1147	>	txt += 'echo "FirstEvent:<$FirstEvent>"\n'
1148	>	argNum += 1
1149	>	elif (self.generator == 'comphep'):
1150	>	txt += 'export CompHEPFirstEvent=${args[%s]}\n' % argNum
1151	>	txt += 'echo "CompHEPFirstEvent:<$CompHEPFirstEvent>"\n'
1152	>	argNum += 1
1153
1154	<	if (self.sourceSeed):
978	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
979	<	txt += 'sed "s#uint32 sourceSeed = 0#uint32 sourceSeed = $Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
980	<	seedIndex=seedIndex+1
981	<	## the following seeds are not always present
982	<	if (self.sourceSeedVtx):
983	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
984	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
985	<	txt += 'sed "s#uint32 VtxSmeared = 0#uint32 VtxSmeared = $VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
986	<	seedIndex += 1
987	<	if (self.sourceSeedG4):
988	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
989	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
990	<	txt += 'sed "s#uint32 g4SimHits = 0#uint32 g4SimHits = $G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
991	<	seedIndex += 1
992	<	if (self.sourceSeedMix):
993	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
994	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
995	<	txt += 'sed "s#uint32 mix = 0#uint32 mix = $mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
996	<	seedIndex += 1
997	<	pass
998	<	pass
999	<	txt += 'mv -f '+pset+' pset.cfg\n'
1154	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
1155
1001	–	if len(self.additional_inbox_files) > 0:
1002	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1003	–	txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1004	–	txt += 'fi\n'
1005	–	pass
1156
1157	<	if self.pset != None: #CarlosDaniele
1158	<	txt += '\n'
1009	<	txt += 'echo "*** cat pset.cfg *******"\n'
1010	<	txt += 'cat pset.cfg\n'
1011	<	txt += 'echo "**** end pset.cfg ******"\n'
1157	>	if self.pset != None:
1158	>	# FUTURE: Can simply for 2_1_x and higher
1159		txt += '\n'
1160	<	### FEDE FOR DBS OUTPUT PUBLICATION
1161	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1160	>	if self.debug_wrapper==True:
1161	>	txt += 'echo "*** cat ' + psetName + ' *******"\n'
1162	>	txt += 'cat ' + psetName + '\n'
1163	>	txt += 'echo "**** end ' + psetName + ' ******"\n'
1164	>	txt += '\n'
1165	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1166	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
1167	>	else:
1168	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
1169		txt += 'echo "PSETHASH = $PSETHASH" \n'
1016	–	##############
1170		txt += '\n'
1171		return txt
1172
1173	<	def wsBuildExe(self, nj=0):
1173	>	def wsUntarSoftware(self, nj=0):
1174		"""
1175		Put in the script the commands to build an executable
1176		or a library.
1177		"""
1178
1179	<	txt = ""
1179	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
1180
1181		if os.path.isfile(self.tgzNameWithPath):
1182		txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1183	<	txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1183	>	txt += 'tar xzf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1184	>	if self.debug_wrapper:
1185	>	txt += 'tar tzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1186	>	txt += 'ls -Al \n'
1187		txt += 'untar_status=$? \n'
1188		txt += 'if [ $untar_status -ne 0 ]; then \n'
1189	<	txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
1190	<	txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n'
1191	<	txt += ' echo "JobExitCode=$untar_status" \| tee -a $RUNTIME_AREA/$repo\n'
1036	<	txt += ' if [ $middleware == OSG ]; then \n'
1037	<	txt += ' cd $RUNTIME_AREA\n'
1038	<	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1039	<	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
1040	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1041	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1042	<	txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1043	<	txt += ' echo "JOB_EXIT_STATUS = 50999"\n'
1044	<	txt += ' echo "JobExitCode=50999" \| tee -a $RUNTIME_AREA/$repo\n'
1045	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1046	<	txt += ' fi\n'
1047	<	txt += ' fi \n'
1048	<	txt += ' \n'
1049	<	txt += ' exit 1 \n'
1189	>	txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
1190	>	txt += ' job_exit_code=$untar_status\n'
1191	>	txt += ' func_exit\n'
1192		txt += 'else \n'
1193		txt += ' echo "Successful untar" \n'
1194		txt += 'fi \n'
1195		txt += '\n'
1196	<	txt += 'echo ">>> Include ProdAgentApi and PRODCOMMON in PYTHONPATH:"\n'
1196	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1197		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1198	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1057	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1198	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
1199		txt += 'else\n'
1200	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1200	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1201		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1061	–	###################
1202		txt += 'fi\n'
1203		txt += '\n'
1204
#	Line 1066 \| Line 1206 \| class Cmssw(JobType):
1206
1207		return txt
1208
1209	<	def modifySteeringCards(self, nj):
1209	>	def wsBuildExe(self, nj=0):
1210		"""
1211	<	modify the card provided by the user,
1212	<	writing a new card into share dir
1211	>	Put in the script the commands to build an executable
1212	>	or a library.
1213		"""
1214
1215	+	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1216	+	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1217	+
1218	+	txt += 'rm -r lib/ module/ \n'
1219	+	txt += 'mv $RUNTIME_AREA/lib/ . \n'
1220	+	txt += 'mv $RUNTIME_AREA/module/ . \n'
1221	+	if self.dataExist == True:
1222	+	txt += 'rm -r src/ \n'
1223	+	txt += 'mv $RUNTIME_AREA/src/ . \n'
1224	+	if len(self.additional_inbox_files)>0:
1225	+	for file in self.additional_inbox_files:
1226	+	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1227	+	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1228	+	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1229	+
1230	+	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1231	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1232	+	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
1233	+	txt += 'else\n'
1234	+	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1235	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1236	+	txt += 'fi\n'
1237	+	txt += '\n'
1238	+
1239	+	return txt
1240	+
1241	+
1242		def executableName(self):
1243	<	if self.scriptExe: #CarlosDaniele
1243	>	if self.scriptExe:
1244		return "sh "
1245		else:
1246		return self.executable
1247
1248		def executableArgs(self):
1249	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1250		if self.scriptExe:#CarlosDaniele
1251		return self.scriptExe + " $NJob"
1252		else:
1253	<	# if >= CMSSW_1_5_X, add -e
1254	<	version_array = self.scram.getSWVersion().split('_')
1255	<	major = 0
1256	<	minor = 0
1257	<	try:
1258	<	major = int(version_array[1])
1259	<	minor = int(version_array[2])
1260	<	except:
1093	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1094	<	raise CrabException(msg)
1095	<	if major >= 1 and minor >= 5 :
1096	<	return " -e -p pset.cfg"
1253	>	ex_args = ""
1254	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1255	>	# Framework job report
1256	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1257	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1258	>	# Type of config file
1259	>	if self.CMSSW_major >= 2 :
1260	>	ex_args += " -p pset.py"
1261		else:
1262	<	return " -p pset.cfg"
1262	>	ex_args += " -p pset.cfg"
1263	>	return ex_args
1264
1265		def inputSandbox(self, nj):
1266		"""
1267		Returns a list of filenames to be put in JDL input sandbox.
1268		"""
1269		inp_box = []
1105	–	# # dict added to delete duplicate from input sandbox file list
1106	–	# seen = {}
1107	–	## code
1270		if os.path.isfile(self.tgzNameWithPath):
1271		inp_box.append(self.tgzNameWithPath)
1272	<	if os.path.isfile(self.MLtgzfile):
1111	<	inp_box.append(self.MLtgzfile)
1112	<	## config
1113	<	if not self.pset is None:
1114	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1115	<	## additional input files
1116	<	tgz = self.additionalInputFileTgz()
1117	<	inp_box.append(tgz)
1272	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
1273		return inp_box
1274
1275		def outputSandbox(self, nj):
#	Line 1126 \| Line 1281 \| class Cmssw(JobType):
1281		## User Declared output files
1282		for out in (self.output_file+self.output_file_sandbox):
1283		n_out = nj + 1
1284	<	out_box.append(self.numberFile_(out,str(n_out)))
1284	>	out_box.append(numberFile(out,str(n_out)))
1285		return out_box
1286
1132	–	def prepareSteeringCards(self):
1133	–	"""
1134	–	Make initial modifications of the user's steering card file.
1135	–	"""
1136	–	return
1287
1288		def wsRenameOutput(self, nj):
1289		"""
1290		Returns part of a job script which renames the produced files.
1291		"""
1292
1293	<	txt = '\n'
1294	<	txt += 'echo" >>> directory content:"\n'
1295	<	txt += 'ls \n'
1296	<	txt = '\n'
1297	<
1298	<	txt += 'output_exit_status=0\n'
1149	<
1150	<	for fileWithSuffix in (self.output_file_sandbox):
1151	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1152	<	txt += '\n'
1153	<	txt += '# check output file\n'
1154	<	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1155	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1156	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1157	<	txt += 'else\n'
1158	<	txt += ' exit_status=60302\n'
1159	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1160	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1161	<	txt += ' if [ $middleware == OSG ]; then \n'
1162	<	txt += ' echo "prepare dummy output file"\n'
1163	<	txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1164	<	txt += ' fi \n'
1165	<	txt += 'fi\n'
1293	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1294	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1295	>	txt += 'echo ">>> current directory content:"\n'
1296	>	if self.debug_wrapper:
1297	>	txt += 'ls -Al\n'
1298	>	txt += '\n'
1299
1300		for fileWithSuffix in (self.output_file):
1301	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1301	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1302		txt += '\n'
1303		txt += '# check output file\n'
1304		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1305	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1306	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1305	>	if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1306	>	txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
1307	>	txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1308	>	else:
1309	>	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1310	>	txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1311		txt += 'else\n'
1312	<	txt += ' exit_status=60302\n'
1313	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1314	<	txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n'
1178	<	txt += ' output_exit_status=$exit_status\n'
1179	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1312	>	txt += ' job_exit_code=60302\n'
1313	>	txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1314	>	if common.scheduler.name().upper() == 'CONDOR_G':
1315		txt += ' if [ $middleware == OSG ]; then \n'
1316		txt += ' echo "prepare dummy output file"\n'
1317		txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1184 \| Line 1319 \| class Cmssw(JobType):
1319		txt += 'fi\n'
1320		file_list = []
1321		for fileWithSuffix in (self.output_file):
1322	<	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1322	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
1323
1324	<	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1324	>	txt += 'file_list="'+string.join(file_list,',')+'"\n'
1325	>	txt += '\n'
1326	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1327	>	txt += 'echo ">>> current directory content:"\n'
1328	>	if self.debug_wrapper:
1329	>	txt += 'ls -Al\n'
1330	>	txt += '\n'
1331		txt += 'cd $RUNTIME_AREA\n'
1332		txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1333		return txt
1334
1194	–	def numberFile_(self, file, txt):
1195	–	"""
1196	–	append _'txt' before last extension of a file
1197	–	"""
1198	–	p = string.split(file,".")
1199	–	# take away last extension
1200	–	name = p[0]
1201	–	for x in p[1:-1]:
1202	–	name=name+"."+x
1203	–	# add "_txt"
1204	–	if len(p)>1:
1205	–	ext = p[len(p)-1]
1206	–	result = name + '_' + txt + "." + ext
1207	–	else:
1208	–	result = name + '_' + txt
1209	–
1210	–	return result
1211	–
1335		def getRequirements(self, nj=[]):
1336		"""
1337		return job requirements to add to jdl files
#	Line 1218 \| Line 1341 \| class Cmssw(JobType):
1341		req='Member("VO-cms-' + \
1342		self.version + \
1343		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1344	<	## SL add requirement for OS version only if SL4
1222	<	#reSL4 = re.compile( r'slc4' )
1223	<	if self.executable_arch: # and reSL4.search(self.executable_arch):
1344	>	if self.executable_arch:
1345		req+=' && Member("VO-cms-' + \
1346		self.executable_arch + \
1347		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1348
1349		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1350	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1351	+	req += ' && other.GlueCEStateStatus == "Production" '
1352
1353		return req
1354
1355		def configFilename(self):
1356		""" return the config filename """
1357	<	return self.name()+'.cfg'
1357	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1358	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1359	>	return self.name()+'.py'
1360	>	else:
1361	>	return self.name()+'.cfg'
1362
1363		def wsSetupCMSOSGEnvironment_(self):
1364		"""
1365		Returns part of a job script which is prepares
1366		the execution environment and which is common for all CMS jobs.
1367		"""
1368	<	txt = ' echo ">>> setup CMS OSG environment:"\n'
1368	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1369	>	txt += ' echo ">>> setup CMS OSG environment:"\n'
1370		txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1371		txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1372		txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
#	Line 1246 \| Line 1374 \| class Cmssw(JobType):
1374		txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1375		txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1376		txt += ' else\n'
1377	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1378	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1379	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1252	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1253	<	txt += '\n'
1254	<	txt += ' cd $RUNTIME_AREA\n'
1255	<	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1256	<	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
1257	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1258	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1259	<	txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1260	<	txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1261	<	txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1262	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1263	<	txt += ' fi\n'
1264	<	txt += '\n'
1265	<	txt += ' exit 1\n'
1377	>	txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1378	>	txt += ' job_exit_code=10020\n'
1379	>	txt += ' func_exit\n'
1380		txt += ' fi\n'
1381		txt += '\n'
1382	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1382	>	txt += ' echo "==> setup cms environment ok"\n'
1383		txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1384
1385		return txt
1386
1273	–	### OLI_DANIELE
1387		def wsSetupCMSLCGEnvironment_(self):
1388		"""
1389		Returns part of a job script which is prepares
1390		the execution environment and which is common for all CMS jobs.
1391		"""
1392	<	txt = ' echo ">>> setup CMS LCG environment:"\n'
1392	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1393	>	txt += ' echo ">>> setup CMS LCG environment:"\n'
1394		txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1395		txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1396		txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
1397		txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1398	<	txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1399	<	txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1400	<	txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1287	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1288	<	txt += ' exit 1\n'
1398	>	txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1399	>	txt += ' job_exit_code=10031\n'
1400	>	txt += ' func_exit\n'
1401		txt += ' else\n'
1402		txt += ' echo "Sourcing environment... "\n'
1403		txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1404	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1405	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1406	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1295	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1296	<	txt += ' exit 1\n'
1404	>	txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1405	>	txt += ' job_exit_code=10020\n'
1406	>	txt += ' func_exit\n'
1407		txt += ' fi\n'
1408		txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1409		txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1410		txt += ' result=$?\n'
1411		txt += ' if [ $result -ne 0 ]; then\n'
1412	<	txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1413	<	txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1414	<	txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1305	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1306	<	txt += ' exit 1\n'
1412	>	txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1413	>	txt += ' job_exit_code=10032\n'
1414	>	txt += ' func_exit\n'
1415		txt += ' fi\n'
1416		txt += ' fi\n'
1417		txt += ' \n'
1418	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1418	>	txt += ' echo "==> setup cms environment ok"\n'
1419		return txt
1420
1421	<	### FEDE FOR DBS OUTPUT PUBLICATION
1314	<	def modifyReport(self, nj):
1421	>	def wsModifyReport(self, nj):
1422		"""
1423		insert the part of the script that modifies the FrameworkJob Report
1424		"""
1425	<
1426	<	txt = ''
1320	<	try:
1321	<	publish_data = int(self.cfg_params['USER.publish_data'])
1322	<	except KeyError:
1323	<	publish_data = 0
1425	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1426	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1427		if (publish_data == 1):
1325	–	txt += 'echo ">>> Modify Job Report:" \n'
1326	–	################ FEDE FOR DBS2 #############################################
1327	–	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1328	–	#############################################################################
1329	–
1330	–	txt += 'if [ -z "$SE" ]; then\n'
1331	–	txt += ' SE="" \n'
1332	–	txt += 'fi \n'
1333	–	txt += 'if [ -z "$SE_PATH" ]; then\n'
1334	–	txt += ' SE_PATH="" \n'
1335	–	txt += 'fi \n'
1336	–	txt += 'echo "SE = $SE"\n'
1337	–	txt += 'echo "SE_PATH = $SE_PATH"\n'
1428
1429		processedDataset = self.cfg_params['USER.publish_data_name']
1430	<	txt += 'ProcessedDataset='+processedDataset+'\n'
1431	<	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1432	<	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1433	<	#### FEDE: added slash in LFN ##############
1430	>
1431	>	txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
1432	>	txt += ' FOR_LFN=$LFNBaseName\n'
1433	>	txt += 'else\n'
1434		txt += ' FOR_LFN=/copy_problems/ \n'
1435	<	txt += 'else \n'
1436	<	txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1437	<	##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1438	<	txt += ' FOR_LFN=/store$tmp \n'
1439	<	txt += 'fi \n'
1435	>	txt += ' SE=""\n'
1436	>	txt += ' SE_PATH=""\n'
1437	>	txt += 'fi\n'
1438	>
1439	>	txt += 'echo ">>> Modify Job Report:" \n'
1440	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1441	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1442	>	#txt += 'ProcessedDataset=$procDataset \n'
1443		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1444	+	txt += 'echo "SE = $SE"\n'
1445	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1446		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1447		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1448	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1449	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1450	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1451	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1452	<
1448	>	args = '$RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' \
1449	>	'$USER-$ProcessedDataset-$PSETHASH $ApplicationFamily '+ \
1450	>	' $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH'
1451	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1452	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1453		txt += 'modifyReport_result=$?\n'
1359	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1454		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1455	<	txt += ' exit_status=1\n'
1456	<	txt += ' echo "ERROR: Problem with ModifyJobReport"\n'
1455	>	txt += ' modifyReport_result=70500\n'
1456	>	txt += ' job_exit_code=$modifyReport_result\n'
1457	>	txt += ' echo "ModifyReportResult=$modifyReport_result" \| tee -a $RUNTIME_AREA/$repo\n'
1458	>	txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
1459		txt += 'else\n'
1460	<	txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1460	>	txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1461		txt += 'fi\n'
1366	–	else:
1367	–	txt += 'echo "no data publication required"\n'
1462		return txt
1463
1464	<	def cleanEnv(self):
1465	<	txt = ''
1466	<	txt += 'if [ $middleware == OSG ]; then\n'
1467	<	txt += ' cd $RUNTIME_AREA\n'
1468	<	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1469	<	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
1470	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1471	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1472	<	txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1473	<	txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1474	<	txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1475	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1464	>	def wsParseFJR(self):
1465	>	"""
1466	>	Parse the FrameworkJobReport to obtain useful infos
1467	>	"""
1468	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1469	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1470	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1471	>	txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1472	>	txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1473	>	if self.debug_wrapper :
1474	>	txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1475	>	txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1476	>	txt += ' if [ $executable_exit_status -eq 50115 ];then\n'
1477	>	txt += ' echo ">>> crab_fjr.xml contents: "\n'
1478	>	txt += ' cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1479	>	txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1480	>	txt += ' elif [ $executable_exit_status -eq -999 ];then\n'
1481	>	txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1482	>	txt += ' else\n'
1483	>	txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1484	>	txt += ' fi\n'
1485	>	txt += ' else\n'
1486	>	txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1487	>	txt += ' fi\n'
1488	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1489	>	txt += ' if [ $executable_exit_status -eq 0 ];then\n'
1490	>	txt += ' echo ">>> Executable succeded $executable_exit_status"\n'
1491	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1492	>	# VERIFY PROCESSED DATA
1493	>	txt += ' echo ">>> Verify list of processed files:"\n'
1494	>	txt += ' echo $InputFiles \|tr -d \'\\\\\' \|tr \',\' \'\\n\'\|tr -d \'"\' > input-files.txt\n'
1495	>	txt += ' python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1496	>	txt += ' cat input-files.txt \| sort \| uniq > tmp.txt\n'
1497	>	txt += ' mv tmp.txt input-files.txt\n'
1498	>	txt += ' echo "cat input-files.txt"\n'
1499	>	txt += ' echo "----------------------"\n'
1500	>	txt += ' cat input-files.txt\n'
1501	>	txt += ' cat processed-files.txt \| sort \| uniq > tmp.txt\n'
1502	>	txt += ' mv tmp.txt processed-files.txt\n'
1503	>	txt += ' echo "----------------------"\n'
1504	>	txt += ' echo "cat processed-files.txt"\n'
1505	>	txt += ' echo "----------------------"\n'
1506	>	txt += ' cat processed-files.txt\n'
1507	>	txt += ' echo "----------------------"\n'
1508	>	txt += ' diff -q input-files.txt processed-files.txt\n'
1509	>	txt += ' fileverify_status=$?\n'
1510	>	txt += ' if [ $fileverify_status -ne 0 ]; then\n'
1511	>	txt += ' executable_exit_status=30001\n'
1512	>	txt += ' echo "ERROR ==> not all input files processed"\n'
1513	>	txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1514	>	txt += ' echo " ==> diff input-files.txt processed-files.txt"\n'
1515	>	txt += ' fi\n'
1516	>	txt += ' elif [ $executable_exit_status -ne 0 ] \|\| [ $executable_exit_status -ne 50015 ] \|\| [ $executable_exit_status -ne 50017 ];then\n'
1517	>	txt += ' echo ">>> Executable failed $executable_exit_status"\n'
1518	>	txt += ' echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1519	>	txt += ' echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1520	>	txt += ' job_exit_code=$executable_exit_status\n'
1521	>	txt += ' func_exit\n'
1522		txt += ' fi\n'
1523	+	txt += '\n'
1524	+	txt += 'else\n'
1525	+	txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1526		txt += 'fi\n'
1527		txt += '\n'
1528	+	txt += 'echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1529	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1530	+	txt += 'job_exit_code=$executable_exit_status\n'
1531	+
1532		return txt
1533
1534		def setParam_(self, param, value):
#	Line 1390 \| Line 1537 \| class Cmssw(JobType):
1537		def getParams(self):
1538		return self._params
1539
1393	–	def setTaskid_(self):
1394	–	self._taskId = self.cfg_params['taskId']
1395	–
1396	–	def getTaskid(self):
1397	–	return self._taskId
1398	–
1540		def uniquelist(self, old):
1541		"""
1542		remove duplicates from a list
#	Line 1405 \| Line 1546 \| class Cmssw(JobType):
1546		nd[e]=0
1547		return nd.keys()
1548
1549	<
1409	<	def checkOut(self, limit):
1549	>	def outList(self,list=False):
1550		"""
1551		check the dimension of the output files
1552		"""
1553	<	txt += 'echo ">>> Starting output sandbox limit check :"\n'
1554	<	allOutFiles = ""
1553	>	txt = ''
1554	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1555		listOutFiles = []
1556	<	for fileOut in (self.output_file+self.output_file_sandbox):
1557	<	if fileOut.find('crab_fjr') == -1:
1558	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1559	<	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1560	<	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1561	<	txt += 'ls -gGhrta;\n'
1562	<	txt += 'sum=0;\n'
1563	<	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1564	<	txt += ' if [ -e $file ]; then\n'
1565	<	txt += ' tt=`ls -gGrta $file \| awk \'{ print $3 }\'`\n'
1566	<	txt += ' sum=`expr $sum + $tt`\n'
1567	<	txt += ' else\n'
1568	<	txt += ' echo "WARNING: output file $file not found!"\n'
1569	<	txt += ' fi\n'
1570	<	txt += 'done\n'
1571	<	txt += 'echo "Total Output dimension: $sum";\n'
1432	<	txt += 'limit='+str(limit)+';\n'
1433	<	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1434	<	txt += 'if [ $limit -lt $sum ]; then\n'
1435	<	txt += ' echo "WARNING: output files have to big size - something will be lost;"\n'
1436	<	txt += ' echo " checking the output file sizes..."\n'
1437	<	"""
1438	<	txt += ' dim=0;\n'
1439	<	txt += ' exclude=0;\n'
1440	<	txt += ' for files in '+str(allOutFiles)+' ; do\n'
1441	<	txt += ' sumTemp=0;\n'
1442	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1443	<	txt += ' if [ $file != $file2 ]; then\n'
1444	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1445	<	txt += ' sumTemp=`expr $sumTemp + $tt`;\n'
1446	<	txt += ' fi\n'
1447	<	txt += ' done\n'
1448	<	txt += ' if [ $sumTemp -lt $limit ]; then\n'
1449	<	txt += ' if [ $dim -lt $sumTemp ]; then\n'
1450	<	txt += ' dim=$sumTemp;\n'
1451	<	txt += ' exclude=$file;\n'
1452	<	txt += ' fi\n'
1453	<	txt += ' fi\n'
1454	<	txt += ' done\n'
1455	<	txt += ' echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1456	<	"""
1457	<	txt += ' tot=0;\n'
1458	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1459	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1460	<	txt += ' tot=`expr $tot + $tt`;\n'
1461	<	txt += ' if [ $limit -lt $tot ]; then\n'
1462	<	txt += ' tot=`expr $tot - $tt`;\n'
1463	<	txt += ' fileLast=$file;\n'
1464	<	txt += ' break;\n'
1465	<	txt += ' fi\n'
1466	<	txt += ' done\n'
1467	<	txt += ' echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1468	<	txt += ' flag=0;\n'
1469	<	txt += ' for filess in '+str(allOutFiles)+' ; do\n'
1470	<	txt += ' if [ $fileLast = $filess ]; then\n'
1471	<	txt += ' flag=1;\n'
1472	<	txt += ' fi\n'
1473	<	txt += ' if [ $flag -eq 1 ]; then\n'
1474	<	txt += ' rm -f $filess;\n'
1475	<	txt += ' fi\n'
1476	<	txt += ' done\n'
1477	<	txt += ' ls -agGhrt;\n'
1478	<	txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1479	<	txt += ' echo "JOB_EXIT_STATUS = 70000";\n'
1480	<	txt += ' exit_status=70000;\n'
1481	<	txt += 'else'
1482	<	txt += ' echo "Total Output dimension $sum is fine.";\n'
1483	<	txt += 'fi\n'
1484	<	txt += 'echo "Ending output sandbox limit check"\n'
1556	>	stdout = 'CMSSW_$NJob.stdout'
1557	>	stderr = 'CMSSW_$NJob.stderr'
1558	>	if (self.return_data == 1):
1559	>	for file in (self.output_file+self.output_file_sandbox):
1560	>	listOutFiles.append(numberFile(file, '$NJob'))
1561	>	listOutFiles.append(stdout)
1562	>	listOutFiles.append(stderr)
1563	>	else:
1564	>	for file in (self.output_file_sandbox):
1565	>	listOutFiles.append(numberFile(file, '$NJob'))
1566	>	listOutFiles.append(stdout)
1567	>	listOutFiles.append(stderr)
1568	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1569	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1570	>	txt += 'export filesToCheck\n'
1571	>	if list : return self.output_file
1572		return txt

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing COMP/CRAB/python/cms_cmssw.py (file contents): Revision 1.137 by slacapra, Fri Nov 16 11:09:31 2007 UTC vs. Revision 1.267 by spiga, Wed Jan 14 14:10:23 2009 UTC

Diff Legend

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.137 by slacapra, Fri Nov 16 11:09:31 2007 UTC vs.
Revision 1.267 by spiga, Wed Jan 14 14:10:23 2009 UTC