[ViewVC] Diff of: cvsroot/COMP/CRAB/python/cms

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.116.2.8 by fanzago, Wed Oct 17 13:25:00 2007 UTC vs.
Revision 1.169 by spiga, Tue Apr 1 14:53:36 2008 UTC

#	Line 13 \| Line 13 \| class Cmssw(JobType):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15
16	+	self.argsList = []
17	+
18		self._params = {}
19		self.cfg_params = cfg_params
18	–
20		# init BlackWhiteListParser
21		self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
22
23	<	try:
23	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24	<	except KeyError:
25	<	self.MaxTarBallSize = 9.5
23	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
24
25		# number of jobs requested to be created, limit obj splitting
26		self.ncjobs = ncjobs
27
28		log = common.logger
29	<
29	>
30		self.scram = Scram.Scram(cfg_params)
31		self.additional_inbox_files = []
32		self.scriptExe = ''
#	Line 37 \| Line 35 \| class Cmssw(JobType):
35		self.tgz_name = 'default.tgz'
36		self.additional_tgz_name = 'additional.tgz'
37		self.scriptName = 'CMSSW.sh'
38	<	self.pset = '' #scrip use case Da
38	>	self.pset = '' #scrip use case Da
39		self.datasetPath = '' #scrip use case Da
40
41		# set FJR file name
42		self.fjrFileName = 'crab_fjr.xml'
43
44		self.version = self.scram.getSWVersion()
45	<
45	>
46		#
47		# Try to block creation in case of arch/version mismatch
48		#
49
50	<	a = string.split(self.version, "_")
51	<
52	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
54	<	raise CrabException(msg)
55	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57	<	raise CrabException(msg)
58	<
59	<	common.taskDB.setDict('codeVersion',self.version)
50	>	# a = string.split(self.version, "_")
51	>	#
52	>	# if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53	>	# msg = "Warning: You are using %s version of CMSSW with %s architecture. \n--> Did you compile your libraries with SLC3? Otherwise you can find some problems running on SLC4 Grid nodes.\n"%(self.version, self.executable_arch)
54	>	# common.logger.message(msg)
55	>	# if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
56	>	# msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
57	>	# raise CrabException(msg)
58	>	#
59	>
60		self.setParam_('application', self.version)
61
62		### collect Data cards
63
64	<	## get DBS mode
65	<	try:
68	<	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69	<	except KeyError:
70	<	self.use_dbs_1 = 0
71	<
72	<	try:
73	<	tmp = cfg_params['CMSSW.datasetpath']
74	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75	<	if string.lower(tmp)=='none':
76	<	self.datasetPath = None
77	<	self.selectNoInput = 1
78	<	else:
79	<	self.datasetPath = tmp
80	<	self.selectNoInput = 0
81	<	except KeyError:
82	<	msg = "Error: datasetpath not defined "
64	>	if not cfg_params.has_key('CMSSW.datasetpath'):
65	>	msg = "Error: datasetpath not defined "
66		raise CrabException(msg)
67	+	tmp = cfg_params['CMSSW.datasetpath']
68	+	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
69	+	if string.lower(tmp)=='none':
70	+	self.datasetPath = None
71	+	self.selectNoInput = 1
72	+	else:
73	+	self.datasetPath = tmp
74	+	self.selectNoInput = 0
75
76		# ML monitoring
77		# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
#	Line 88 \| Line 79 \| class Cmssw(JobType):
79		self.setParam_('dataset', 'None')
80		self.setParam_('owner', 'None')
81		else:
82	+	## SL what is supposed to fail here?
83		try:
84		datasetpath_split = self.datasetPath.split("/")
85		# standard style
86		self.setParam_('datasetFull', self.datasetPath)
87	<	if self.use_dbs_1 == 1 :
88	<	self.setParam_('dataset', datasetpath_split[1])
97	<	self.setParam_('owner', datasetpath_split[-1])
98	<	else:
99	<	self.setParam_('dataset', datasetpath_split[1])
100	<	self.setParam_('owner', datasetpath_split[2])
87	>	self.setParam_('dataset', datasetpath_split[1])
88	>	self.setParam_('owner', datasetpath_split[2])
89		except:
90		self.setParam_('dataset', self.datasetPath)
91		self.setParam_('owner', self.datasetPath)
92	<
93	<	self.setTaskid_()
106	<	self.setParam_('taskId', self.cfg_params['taskId'])
92	>
93	>	self.setParam_('taskId', common._db.queryTask('name')) ## new BL--DS
94
95		self.dataTiers = []
96
97		## now the application
98	<	try:
99	<	self.executable = cfg_params['CMSSW.executable']
100	<	self.setParam_('exe', self.executable)
114	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
115	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
116	<	log.debug(3,msg)
117	<	except KeyError:
118	<	self.executable = 'cmsRun'
119	<	self.setParam_('exe', self.executable)
120	<	msg = "User executable not defined. Use cmsRun"
121	<	log.debug(3,msg)
122	<	pass
98	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
99	>	self.setParam_('exe', self.executable)
100	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
101
102	<	try:
125	<	self.pset = cfg_params['CMSSW.pset']
126	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
127	<	if self.pset.lower() != 'none' :
128	<	if (not os.path.exists(self.pset)):
129	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
130	<	else:
131	<	self.pset = None
132	<	except KeyError:
102	>	if not cfg_params.has_key('CMSSW.pset'):
103		raise CrabException("PSet file missing. Cannot run cmsRun ")
104	+	self.pset = cfg_params['CMSSW.pset']
105	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
106	+	if self.pset.lower() != 'none' :
107	+	if (not os.path.exists(self.pset)):
108	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
109	+	else:
110	+	self.pset = None
111
112		# output files
113		## stuff which must be returned always via sandbox
#	Line 140 \| Line 117 \| class Cmssw(JobType):
117		self.output_file_sandbox.append(self.fjrFileName)
118
119		# other output files to be returned via sandbox or copied to SE
120	<	try:
121	<	self.output_file = []
122	<	tmp = cfg_params['CMSSW.output_file']
123	<	if tmp != '':
124	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
125	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
126	<	for tmp in tmpOutFiles:
127	<	tmp=string.strip(tmp)
151	<	self.output_file.append(tmp)
152	<	pass
153	<	else:
154	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
120	>	self.output_file = []
121	>	tmp = cfg_params.get('CMSSW.output_file',None)
122	>	if tmp :
123	>	tmpOutFiles = string.split(tmp,',')
124	>	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
125	>	for tmp in tmpOutFiles:
126	>	tmp=string.strip(tmp)
127	>	self.output_file.append(tmp)
128		pass
129	<	pass
157	<	except KeyError:
129	>	else:
130		log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
131	<	pass
131	>	pass
132
133		# script_exe file as additional file in inputSandbox
134	<	try:
135	<	self.scriptExe = cfg_params['USER.script_exe']
136	<	if self.scriptExe != '':
137	<	if not os.path.isfile(self.scriptExe):
138	<	msg ="ERROR. file "+self.scriptExe+" not found"
139	<	raise CrabException(msg)
168	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
169	<	except KeyError:
170	<	self.scriptExe = ''
134	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
135	>	if self.scriptExe :
136	>	if not os.path.isfile(self.scriptExe):
137	>	msg ="ERROR. file "+self.scriptExe+" not found"
138	>	raise CrabException(msg)
139	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
140
141		#CarlosDaniele
142		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
#	Line 175 \| Line 144 \| class Cmssw(JobType):
144		raise CrabException(msg)
145
146		## additional input files
147	<	try:
147	>	if cfg_params.has_key('USER.additional_input_files'):
148		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
149		for tmp in tmpAddFiles:
150		tmp = string.strip(tmp)
#	Line 199 \| Line 168 \| class Cmssw(JobType):
168		pass
169		pass
170		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
171	<	except KeyError:
203	<	pass
204	<
205	<	# files per job
206	<	try:
207	<	if (cfg_params['CMSSW.files_per_jobs']):
208	<	raise CrabException("files_per_jobs no longer supported. Quitting.")
209	<	except KeyError:
210	<	pass
171	>	pass
172
173		## Events per job
174	<	try:
174	>	if cfg_params.has_key('CMSSW.events_per_job'):
175		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
176		self.selectEventsPerJob = 1
177	<	except KeyError:
177	>	else:
178		self.eventsPerJob = -1
179		self.selectEventsPerJob = 0
180	<
180	>
181		## number of jobs
182	<	try:
182	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
183		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
184		self.selectNumberOfJobs = 1
185	<	except KeyError:
185	>	else:
186		self.theNumberOfJobs = 0
187		self.selectNumberOfJobs = 0
188
189	<	try:
189	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
190		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
191		self.selectTotalNumberEvents = 1
192	<	except KeyError:
192	>	else:
193		self.total_number_of_events = 0
194		self.selectTotalNumberEvents = 0
195
196	<	if self.pset != None: #CarlosDaniele
196	>	if self.pset != None: #CarlosDaniele
197		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
198		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
199		raise CrabException(msg)
#	Line 241 \| Line 202 \| class Cmssw(JobType):
202		msg = 'Must specify number_of_jobs.'
203		raise CrabException(msg)
204
205	<	## source seed for pythia
206	<	try:
207	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
208	<	except KeyError:
209	<	self.sourceSeed = None
210	<	common.logger.debug(5,"No seed given")
211	<
212	<	try:
213	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
214	<	except KeyError:
215	<	self.sourceSeedVtx = None
216	<	common.logger.debug(5,"No vertex seed given")
217	<
218	<	try:
219	<	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
220	<	except KeyError:
221	<	self.sourceSeedG4 = None
222	<	common.logger.debug(5,"No g4 sim hits seed given")
205	>	## New method of dealing with seeds
206	>	self.incrementSeeds = []
207	>	self.preserveSeeds = []
208	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
209	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
210	>	for tmp in tmpList:
211	>	tmp.strip()
212	>	self.preserveSeeds.append(tmp)
213	>	if cfg_params.has_key('CMSSW.increment_seeds'):
214	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
215	>	for tmp in tmpList:
216	>	tmp.strip()
217	>	self.incrementSeeds.append(tmp)
218	>
219	>	## Old method of dealing with seeds
220	>	## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
221	>	## remove
222	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
223	>	if self.sourceSeed:
224	>	print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
225	>	self.incrementSeeds.append('sourceSeed')
226	>
227	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
228	>	if self.sourceSeedVtx:
229	>	print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
230	>	self.incrementSeeds.append('VtxSmeared')
231	>
232	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
233	>	if self.sourceSeedG4:
234	>	print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
235	>	self.incrementSeeds.append('g4SimHits')
236	>
237	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
238	>	if self.sourceSeedMix:
239	>	print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
240	>	self.incrementSeeds.append('mix')
241
242	<	try:
264	<	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
265	<	except KeyError:
266	<	self.sourceSeedMix = None
267	<	common.logger.debug(5,"No mix seed given")
242	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
243
269	–	try:
270	–	self.firstRun = int(cfg_params['CMSSW.first_run'])
271	–	except KeyError:
272	–	self.firstRun = None
273	–	common.logger.debug(5,"No first run given")
244		if self.pset != None: #CarlosDaniele
245	<	ver = string.split(self.version,"_")
276	<	if (int(ver[1])>=1 and int(ver[2])>=5):
277	<	import PsetManipulator150 as pp
278	<	else:
279	<	import PsetManipulator as pp
245	>	import PsetManipulator as pp
246		PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
247
248	+	# Copy/return
249	+
250	+	self.copy_data = int(cfg_params.get('USER.copy_data',0))
251	+	self.return_data = int(cfg_params.get('USER.return_data',0))
252	+
253		#DBSDLS-start
254	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
254	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
255		self.maxEvents=0 # max events available ( --> check the requested nb. of evts in Creator.py)
256		self.DBSPaths={} # all dbs paths requested ( --> input to the site local discovery script)
257		self.jobDestination=[] # Site destination(s) for each job (list of lists)
#	Line 289 \| Line 260 \| class Cmssw(JobType):
260		blockSites = {}
261		if self.datasetPath:
262		blockSites = self.DataDiscoveryAndLocation(cfg_params)
263	<	#DBSDLS-end
263	>	#DBSDLS-end
264
265		self.tgzNameWithPath = self.getTarBall(self.executable)
266	<
266	>
267		## Select Splitting
268	<	if self.selectNoInput:
268	>	if self.selectNoInput:
269		if self.pset == None: #CarlosDaniele
270		self.jobSplittingForScript()
271		else:
#	Line 305 \| Line 276 \| class Cmssw(JobType):
276		# modify Pset
277		if self.pset != None: #CarlosDaniele
278		try:
279	<	if (self.datasetPath): # standard job
280	<	# allow to processa a fraction of events in a file
310	<	PsetEdit.inputModule("INPUT")
311	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
312	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
313	<	else: # pythia like job
314	<	PsetEdit.maxEvent(self.eventsPerJob)
315	<	if (self.firstRun):
316	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN") #First Run
317	<	if (self.sourceSeed) :
318	<	PsetEdit.pythiaSeed("INPUT")
319	<	if (self.sourceSeedVtx) :
320	<	PsetEdit.vtxSeed("INPUTVTX")
321	<	if (self.sourceSeedG4) :
322	<	PsetEdit.g4Seed("INPUTG4")
323	<	if (self.sourceSeedMix) :
324	<	PsetEdit.mixSeed("INPUTMIX")
325	<	# add FrameworkJobReport to parameter-set
279	>	# Add FrameworkJobReport to parameter-set, set max events.
280	>	# Reset later for data jobs by writeCFG which does all modifications
281		PsetEdit.addCrabFJR(self.fjrFileName)
282	+	PsetEdit.maxEvent(self.eventsPerJob)
283		PsetEdit.psetWriter(self.configFilename())
284		except:
285		msg='Error while manipuliating ParameterSet: exiting...'
#	Line 332 \| Line 288 \| class Cmssw(JobType):
288		def DataDiscoveryAndLocation(self, cfg_params):
289
290		import DataDiscovery
335	–	import DataDiscovery_DBS2
291		import DataLocation
292		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
293
#	Line 341 \| Line 296 \| class Cmssw(JobType):
296		## Contact the DBS
297		common.logger.message("Contacting Data Discovery Services ...")
298		try:
299	<
345	<	if self.use_dbs_1 == 1 :
346	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
347	<	else :
348	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
299	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
300		self.pubdata.fetchDBSInfo()
301
302		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 357 \| Line 308 \| class Cmssw(JobType):
308		except DataDiscovery.DataDiscoveryError, ex:
309		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
310		raise CrabException(msg)
360	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
361	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
362	–	raise CrabException(msg)
363	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
364	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
365	–	raise CrabException(msg)
366	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
367	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
368	–	raise CrabException(msg)
311
312		self.filesbyblock=self.pubdata.getFiles()
313		self.eventsbyblock=self.pubdata.getEventsPerBlock()
314		self.eventsbyfile=self.pubdata.getEventsPerFile()
315
316		## get max number of events
317	<	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
317	>	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
318
319		## Contact the DLS and build a list of sites hosting the fileblocks
320		try:
#	Line 381 \| Line 323 \| class Cmssw(JobType):
323		except DataLocation.DataLocationError , ex:
324		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
325		raise CrabException(msg)
326	<
326	>
327
328		sites = dataloc.getSites()
329		allSites = []
#	Line 395 \| Line 337 \| class Cmssw(JobType):
337		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
338
339		return sites
340	<
340	>
341	>	# to Be Removed DS -- BL
342	>	# def setArgsList(self, argsList):
343	>	# self.argsList = argsList
344	>
345		def jobSplittingByBlocks(self, blockSites):
346		"""
347		Perform job splitting. Jobs run over an integer number of files
#	Line 445 \| Line 391 \| class Cmssw(JobType):
391		totalNumberOfJobs = 999999999
392		else :
393		totalNumberOfJobs = self.ncjobs
394	<
394	>
395
396		blocks = blockSites.keys()
397		blockCount = 0
#	Line 465 \| Line 411 \| class Cmssw(JobType):
411		blockCount += 1
412		if block not in jobsOfBlock.keys() :
413		jobsOfBlock[block] = []
414	<
414	>
415		if self.eventsbyblock.has_key(block) :
416		numEventsInBlock = self.eventsbyblock[block]
417		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
418	<
418	>
419		files = self.filesbyblock[block]
420		numFilesInBlock = len(files)
421		if (numFilesInBlock <= 0):
#	Line 477 \| Line 423 \| class Cmssw(JobType):
423		fileCount = 0
424
425		# ---- New block => New job ---- #
426	<	parString = "\\{"
426	>	parString = ""
427		# counter for number of events in files currently worked on
428		filesEventCount = 0
429		# flag if next while loop should touch new file
430		newFile = 1
431		# job event counter
432		jobSkipEventCount = 0
433	<
433	>
434		# ---- Iterate over the files in the block until we've met the requested ---- #
435		# ---- total # of events or we've gone over all the files in this block ---- #
436		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
#	Line 500 \| Line 446 \| class Cmssw(JobType):
446		newFile = 0
447		except KeyError:
448		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
449	<
449	>
450
451		# if less events in file remain than eventsPerJobRequested
452		if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
#	Line 509 \| Line 455 \| class Cmssw(JobType):
455		# end job using last file, use remaining events in block
456		# close job and touch new file
457		fullString = parString[:-2]
512	–	fullString += '\\}'
458		list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
459		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
460		self.jobDestination.append(blockSites[block])
#	Line 522 \| Line 467 \| class Cmssw(JobType):
467		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
468		jobSkipEventCount = 0
469		# reset file
470	<	parString = "\\{"
470	>	parString = ""
471		filesEventCount = 0
472		newFile = 1
473		fileCount += 1
#	Line 534 \| Line 479 \| class Cmssw(JobType):
479		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
480		# close job and touch new file
481		fullString = parString[:-2]
537	–	fullString += '\\}'
482		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
483		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
484		self.jobDestination.append(blockSites[block])
#	Line 546 \| Line 490 \| class Cmssw(JobType):
490		eventsRemaining = eventsRemaining - eventsPerJobRequested
491		jobSkipEventCount = 0
492		# reset file
493	<	parString = "\\{"
493	>	parString = ""
494		filesEventCount = 0
495		newFile = 1
496		fileCount += 1
497	<
497	>
498		# if more events in file remain than eventsPerJobRequested
499		else :
500		# close job but don't touch new file
501		fullString = parString[:-2]
558	–	fullString += '\\}'
502		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
503		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
504		self.jobDestination.append(blockSites[block])
#	Line 570 \| Line 513 \| class Cmssw(JobType):
513		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
514		# remove all but the last file
515		filesEventCount = self.eventsbyfile[file]
516	<	parString = "\\{"
574	<	parString += '\\\"' + file + '\\\"\,'
516	>	parString = '\\\"' + file + '\\\"\,'
517		pass # END if
518		pass # END while (iterate over files in the block)
519		pass # END while (iterate over blocks in the dataset)
#	Line 579 \| Line 521 \| class Cmssw(JobType):
521		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
522		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
523		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
524	<
524	>
525		# screen output
526		screenOutput = "List of jobs and available destination sites:\n\n"
527
#	Line 593 \| Line 535 \| class Cmssw(JobType):
535		blockCounter += 1
536		screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
537		if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
538	<	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
538	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
539		bloskNoSite.append( blockCounter )
540	<
540	>
541		common.logger.message(screenOutput)
542		if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
543		msg = 'WARNING: No sites are hosting any part of data for block:\n '
#	Line 611 \| Line 553 \| class Cmssw(JobType):
553		for range_jobs in noSiteBlock:
554		msg += str(range_jobs) + virgola
555		msg += '\n will not be submitted and this block of data can not be analyzed!\n'
556	+	if self.cfg_params.has_key('EDG.se_white_list'):
557	+	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
558	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
559	+	msg += 'Please check if the dataset is available at this site!)\n'
560	+	if self.cfg_params.has_key('EDG.ce_white_list'):
561	+	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
562	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
563	+	msg += 'Please check if the dataset is available at this site!)\n'
564	+
565		common.logger.message(msg)
566
567		self.list_of_args = list_of_lists
#	Line 621 \| Line 572 \| class Cmssw(JobType):
572		Perform job splitting based on number of event per job
573		"""
574		common.logger.debug(5,'Splitting per events')
575	<
576	<	if (self.selectEventsPerJob):
575	>
576	>	if (self.selectEventsPerJob):
577		common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
578		if (self.selectNumberOfJobs):
579		common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
#	Line 636 \| Line 587 \| class Cmssw(JobType):
587		if (self.selectEventsPerJob):
588		if (self.selectTotalNumberEvents):
589		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
590	<	elif(self.selectNumberOfJobs) :
590	>	elif(self.selectNumberOfJobs) :
591		self.total_number_of_jobs =self.theNumberOfJobs
592	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
592	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
593
594		elif (self.selectNumberOfJobs) :
595		self.total_number_of_jobs = self.theNumberOfJobs
596		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
597	<
597	>
598		common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs))
599
600		# is there any remainder?
#	Line 659 \| Line 610 \| class Cmssw(JobType):
610		self.list_of_args = []
611		for i in range(self.total_number_of_jobs):
612		## Since there is no input, any site is good
613	<	# self.jobDestination.append(["Any"])
663	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
613	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
614		args=[]
615		if (self.firstRun):
616	<	## pythia first run
667	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
616	>	## pythia first run
617		args.append(str(self.firstRun)+str(i))
669	–	else:
670	–	## no first run
671	–	#self.list_of_args.append([str(i)])
672	–	args.append(str(i))
673	–	if (self.sourceSeed):
674	–	args.append(str(self.sourceSeed)+str(i))
675	–	if (self.sourceSeedVtx):
676	–	## + vtx random seed
677	–	args.append(str(self.sourceSeedVtx)+str(i))
678	–	if (self.sourceSeedG4):
679	–	## + G4 random seed
680	–	args.append(str(self.sourceSeedG4)+str(i))
681	–	if (self.sourceSeedMix):
682	–	## + Mix random seed
683	–	args.append(str(self.sourceSeedMix)+str(i))
684	–	pass
685	–	pass
618		self.list_of_args.append(args)
687	–	pass
688	–
689	–	# print self.list_of_args
619
620		return
621
#	Line 715 \| Line 644 \| class Cmssw(JobType):
644		return
645
646		def split(self, jobParams):
647	<
719	<	common.jobDB.load()
647	>
648		#### Fabio
649		njobs = self.total_number_of_jobs
650		arglist = self.list_of_args
651		# create the empty structure
652		for i in range(njobs):
653		jobParams.append("")
654	<
654	>
655	>	listID=[]
656	>	listField=[]
657		for job in range(njobs):
658		jobParams[job] = arglist[job]
659	<	# print str(arglist[job])
660	<	# print jobParams[job]
661	<	common.jobDB.setArguments(job, jobParams[job])
662	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
663	<	common.jobDB.setDestination(job, self.jobDestination[job])
659	>	listID.append(job+1)
660	>	job_ToSave ={}
661	>	concString = ' '
662	>	argu=''
663	>	if len(jobParams[job]):
664	>	argu += concString.join(jobParams[job] )
665	>	job_ToSave['arguments']= str(job+1)+' '+argu## new BL--DS
666	>	job_ToSave['dlsDestination']= self.jobDestination[job]## new BL--DS
667	>	#common._db.updateJob_(job,job_ToSave)## new BL--DS
668	>	listField.append(job_ToSave)
669	>	msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \
670	>	+" Destination: "+str(self.jobDestination[job])
671	>	common.logger.debug(5,msg)
672	>	#common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
673	>	common._db.updateJob_(listID,listField)## new BL--DS
674	>	## Pay Attention Here....DS--BL
675	>	self.argsList = (len(jobParams[1])+1)
676
735	–	common.jobDB.save()
677		return
678	<
679	<	def getJobTypeArguments(self, nj, sched):
680	<	result = ''
681	<	for i in common.jobDB.arguments(nj):
682	<	result=result+str(i)+" "
683	<	return result
684	<
678	>	#
679	>	# def getJobTypeArguments(self, nj, sched):
680	>	# result = ''
681	>	# jobs=[]
682	>	# jobs.append(nj)
683	>	# for i in common._db.queryJob('arguments',jobs):## BL--DS
684	>	# result=result+str(i)+" "
685	>	# return result
686	>
687		def numberOfJobs(self):
688		# Fabio
689		return self.total_number_of_jobs
#	Line 749 \| Line 692 \| class Cmssw(JobType):
692		"""
693		Return the TarBall with lib and exe
694		"""
695	<
695	>
696		# if it exist, just return it
697		#
698		# Marco. Let's start to use relative path for Boss XML files
#	Line 772 \| Line 715 \| class Cmssw(JobType):
715		# print "swVersion = ", swVersion
716		swReleaseTop = self.scram.getReleaseTop_()
717		#print "swReleaseTop = ", swReleaseTop
718	<
718	>
719		## check if working area is release top
720		if swReleaseTop == '' or swArea == swReleaseTop:
721		return
#	Line 785 \| Line 728 \| class Cmssw(JobType):
728		exeWithPath = self.scram.findFile_(executable)
729		if ( not exeWithPath ):
730		raise CrabException('User executable '+executable+' not found')
731	<
731	>
732		## then check if it's private or not
733		if exeWithPath.find(swReleaseTop) == -1:
734		# the exe is private, so we must ship
#	Line 801 \| Line 744 \| class Cmssw(JobType):
744		else:
745		# the exe is from release, we'll find it on WN
746		pass
747	<
747	>
748		## Now get the libraries: only those in local working area
749		libDir = 'lib'
750		lib = swArea+'/' +libDir
751		common.logger.debug(5,"lib "+lib+" to be tarred")
752		if os.path.exists(lib):
753		tar.add(lib,libDir)
754	<
754	>
755		## Now check if module dir is present
756		moduleDir = 'module'
757		module = swArea + '/' + moduleDir
#	Line 822 \| Line 765 \| class Cmssw(JobType):
765		common.logger.debug(5,"data "+root+"/data"+" to be tarred")
766		tar.add(root+"/data",root[swAreaLen:]+"/data")
767
825	–	## Add ProdAgent dir to tar
826	–	paDir = 'ProdAgentApi'
827	–	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
828	–	if os.path.isdir(pa):
829	–	tar.add(pa,paDir)
768
769	<	### FEDE FOR DBS PUBLICATION
832	<	## Add PRODCOMMON dir to tar
769	>	## Add ProdCommon dir to tar
770		prodcommonDir = 'ProdCommon'
771		prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
772		if os.path.isdir(prodcommonPath):
773		tar.add(prodcommonPath,prodcommonDir)
774	<	#############################
838	<
774	>
775		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
776		tar.close()
777		except :
#	Line 847 \| Line 783 \| class Cmssw(JobType):
783		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
784
785		## create tar-ball with ML stuff
786	<	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
786	>	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
787		try:
788		tar = tarfile.open(self.MLtgzfile, "w:gz")
789		path=os.environ['CRABDIR'] + '/python/'
790	<	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
790	>	#for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py']:
791	>	### FEDE ####
792	>	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']:
793	>	###############
794		tar.add(path+file,file)
795		common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
796		tar.close()
797		except :
798		raise CrabException('Could not create ML files tar-ball')
799	<
799	>
800		return
801	<
801	>
802		def additionalInputFileTgz(self):
803		"""
804		Put all additional files into a tar ball and return its name
#	Line 873 \| Line 812 \| class Cmssw(JobType):
812		tar.close()
813		return tarName
814
815	<	def wsSetupEnvironment(self, nj):
815	>	def wsSetupEnvironment(self, nj=0):
816		"""
817		Returns part of a job script which prepares
818		the execution environment for the job 'nj'.
819		"""
820		# Prepare JobType-independent part
821	<	txt = ''
822	<
823	<	## OLI_Daniele at this level middleware already known
885	<
886	<	txt += 'if [ $middleware == LCG ]; then \n'
887	<	txt += ' echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
888	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
889	<	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
821	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
822	>	txt += 'echo ">>> setup environment"\n'
823	>	txt += 'if [ $middleware == LCG ]; then \n'
824		txt += self.wsSetupCMSLCGEnvironment_()
825		txt += 'elif [ $middleware == OSG ]; then\n'
826		txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
827	<	txt += ' echo "Created working directory: $WORKING_DIR"\n'
828	<	txt += ' if [ ! -d $WORKING_DIR ] ;then\n'
829	<	txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
830	<	txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
831	<	txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
832	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
833	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
834	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
835	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
902	<	txt += ' exit 1\n'
827	>	txt += ' if [ ! $? == 0 ] ;then\n'
828	>	#txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
829	>	#txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
830	>	#txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
831	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
832	>	#txt += ' exit 1\n'
833	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
834	>	txt += ' job_exit_code=10016\n'
835	>	txt += ' func_exit\n'
836		txt += ' fi\n'
837	+	txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
838		txt += '\n'
839		txt += ' echo "Change to working directory: $WORKING_DIR"\n'
840		txt += ' cd $WORKING_DIR\n'
841	<	txt += self.wsSetupCMSOSGEnvironment_()
842	<	txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
843	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
841	>	txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
842	>	txt += self.wsSetupCMSOSGEnvironment_()
843	>	#txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
844	>	#txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
845		txt += 'fi\n'
846
847		# Prepare JobType-specific part
848		scram = self.scram.commandName()
849		txt += '\n\n'
850	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
850	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
851	>	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
852		txt += scram+' project CMSSW '+self.version+'\n'
853		txt += 'status=$?\n'
854		txt += 'if [ $status != 0 ] ; then\n'
855	<	txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856	<	txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
857	<	txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
858	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
859	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
860	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
861	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
862	<	## OLI_Daniele
863	<	txt += ' if [ $middleware == OSG ]; then \n'
864	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
865	<	txt += ' cd $RUNTIME_AREA\n'
866	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
867	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
868	<	txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
869	<	txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
870	<	txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
871	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
872	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
873	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
874	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
875	<	txt += ' fi\n'
876	<	txt += ' fi \n'
941	<	txt += ' exit 1 \n'
855	>	#txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856	>	#txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
857	>	#txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
858	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
859	>	txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
860	>	txt += ' job_exit_code=10034\n'
861	>	#txt += ' if [ $middleware == OSG ]; then \n'
862	>	#txt += ' cd $RUNTIME_AREA\n'
863	>	#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
864	>	#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
865	>	#txt += ' /bin/rm -rf $WORKING_DIR\n'
866	>	#txt += ' if [ -d $WORKING_DIR ] ;then\n'
867	>	#txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
868	>	#txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
869	>	#txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
870	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
871	>	#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
872	>	#txt += ' job_exit_code=10017\n'
873	>	#txt += ' fi\n'
874	>	#txt += ' fi \n'
875	>	#txt += ' exit 1 \n'
876	>	txt += ' func_exit\n'
877		txt += 'fi \n'
943	–	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
878		txt += 'cd '+self.version+'\n'
879		########## FEDE FOR DBS2 ######################
880		txt += 'SOFTWARE_DIR=`pwd`\n'
881	<	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
881	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
882		###############################################
883		### needed grep for bug in scramv1 ###
950	–	txt += scram+' runtime -sh\n'
884		txt += 'eval `'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME`\n'
952	–	txt += 'echo $PATH\n'
953	–
885		# Handle the arguments:
886		txt += "\n"
887		txt += "## number of arguments (first argument always jobnumber)\n"
888		txt += "\n"
889	<	# txt += "narg=$#\n"
890	<	txt += "if [ $nargs -lt 2 ]\n"
889	>	# txt += "if [ $nargs -lt "+str(len(self.argsList[nj].split()))+" ]\n"
890	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
891		txt += "then\n"
892	<	txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893	<	txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
894	<	txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
895	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
896	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
897	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
898	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
899	<	## OLI_Daniele
900	<	txt += ' if [ $middleware == OSG ]; then \n'
901	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
902	<	txt += ' cd $RUNTIME_AREA\n'
903	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
904	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
905	<	txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
906	<	txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
907	<	txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
908	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
909	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
910	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
911	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
912	<	txt += ' fi\n'
913	<	txt += ' fi \n'
983	<	txt += " exit 1\n"
892	>	#txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
893	>	#txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
894	>	#txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
895	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
896	>	txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
897	>	txt += ' job_exit_code=50113\n'
898	>	#txt += ' if [ $middleware == OSG ]; then \n'
899	>	#txt += ' cd $RUNTIME_AREA\n'
900	>	#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
901	>	#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
902	>	#txt += ' /bin/rm -rf $WORKING_DIR\n'
903	>	#txt += ' if [ -d $WORKING_DIR ] ;then\n'
904	>	#txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
905	>	#txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
906	>	#txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
907	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
908	>	#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
909	>	#txt += ' job_exit_code=10017\n'
910	>	#txt += ' fi\n'
911	>	#txt += ' fi\n'
912	>	#txt += " exit 1\n"
913	>	txt += " func_exit\n"
914		txt += "fi\n"
915		txt += "\n"
916
917		# Prepare job-specific part
918		job = common.job_list[nj]
919		### FEDE FOR DBS OUTPUT PUBLICATION
920	<	if (self.datasetPath):
920	>	if (self.datasetPath):
921		txt += '\n'
922		txt += 'DatasetPath='+self.datasetPath+'\n'
923
924		datasetpath_split = self.datasetPath.split("/")
925	<
925	>
926		txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
927		txt += 'DataTier='+datasetpath_split[2]+'\n'
998	–	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
928		txt += 'ApplicationFamily=cmsRun\n'
929
930		else:
931		txt += 'DatasetPath=MCDataTier\n'
932		txt += 'PrimaryDataset=null\n'
933		txt += 'DataTier=null\n'
1005	–	#txt += 'ProcessedDataset=null\n'
934		txt += 'ApplicationFamily=MCDataTier\n'
935	<	if self.pset != None: #CarlosDaniele
935	>	if self.pset != None:
936		pset = os.path.basename(job.configFilename())
937		txt += '\n'
938		txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
939		if (self.datasetPath): # standard job
940	<	#txt += 'InputFiles=$2\n'
941	<	txt += 'InputFiles=${args[1]}\n'
942	<	txt += 'MaxEvents=${args[2]}\n'
1015	<	txt += 'SkipEvents=${args[3]}\n'
940	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
941	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
942	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
943		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1017	–	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
944		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1019	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
945		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1021	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
946		else: # pythia like job
947	<	seedIndex=1
947	>	txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
948	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
949	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
950	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
951		if (self.firstRun):
952	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
952	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
953		txt += 'echo "FirstRun: <$FirstRun>"\n'
1027	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1028	–	seedIndex=seedIndex+1
954
1030	–	if (self.sourceSeed):
1031	–	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1032	–	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1033	–	seedIndex=seedIndex+1
1034	–	## the following seeds are not always present
1035	–	if (self.sourceSeedVtx):
1036	–	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1037	–	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1038	–	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1039	–	seedIndex += 1
1040	–	if (self.sourceSeedG4):
1041	–	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1042	–	txt += 'echo "G4Seed: <$G4Seed>"\n'
1043	–	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1044	–	seedIndex += 1
1045	–	if (self.sourceSeedMix):
1046	–	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1047	–	txt += 'echo "MixSeed: <$mixSeed>"\n'
1048	–	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1049	–	seedIndex += 1
1050	–	pass
1051	–	pass
955		txt += 'mv -f '+pset+' pset.cfg\n'
956
957		if len(self.additional_inbox_files) > 0:
958		txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
959		txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
960		txt += 'fi\n'
961	<	pass
961	>	pass
962
963	<	if self.pset != None: #CarlosDaniele
1061	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1062	<
963	>	if self.pset != None:
964		txt += '\n'
965		txt += 'echo "*** cat pset.cfg *******"\n'
966		txt += 'cat pset.cfg\n'
967		txt += 'echo "**** end pset.cfg ******"\n'
968		txt += '\n'
1068	–	### FEDE FOR DBS OUTPUT PUBLICATION
969		txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
970		txt += 'echo "PSETHASH = $PSETHASH" \n'
1071	–	##############
971		txt += '\n'
1073	–	# txt += 'echo "*** cat pset1.cfg *******"\n'
1074	–	# txt += 'cat pset1.cfg\n'
1075	–	# txt += 'echo "**** end pset1.cfg ******"\n'
972		return txt
973	<
974	<	def wsBuildExe(self, nj=0):
973	>	#### FEDE #####
974	>	def wsUntarSoftware(self, nj=0):
975		"""
976		Put in the script the commands to build an executable
977		or a library.
978		"""
979
980	<	txt = ""
980	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
981
982		if os.path.isfile(self.tgzNameWithPath):
983	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
983	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
984		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
985		txt += 'untar_status=$? \n'
986		txt += 'if [ $untar_status -ne 0 ]; then \n'
987	<	txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
988	<	txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n'
989	<	txt += ' echo "JobExitCode=$untar_status" \| tee -a $RUNTIME_AREA/$repo\n'
1094	<	txt += ' if [ $middleware == OSG ]; then \n'
1095	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1096	<	txt += ' cd $RUNTIME_AREA\n'
1097	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1098	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1099	<	txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1100	<	txt += ' echo "JOB_EXIT_STATUS = 50999"\n'
1101	<	txt += ' echo "JobExitCode=50999" \| tee -a $RUNTIME_AREA/$repo\n'
1102	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1103	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1104	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1105	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1106	<	txt += ' fi\n'
1107	<	txt += ' fi \n'
1108	<	txt += ' \n'
1109	<	txt += ' exit 1 \n'
987	>	txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
988	>	txt += ' job_exit_code=$untar_status\n'
989	>	txt += ' func_exit\n'
990		txt += 'else \n'
991		txt += ' echo "Successful untar" \n'
992		txt += 'fi \n'
993		txt += '\n'
994	<	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
994	>	txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
995		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
996	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1117	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1118	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1119	<	#txt += ' export PYTHONPATH=ProdAgentApi\n'
996	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
997		txt += 'else\n'
998	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1122	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1123	<	#txt += ' export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
998	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
999		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1125	–	###################
1000		txt += 'fi\n'
1001		txt += '\n'
1002
1003		pass
1004	+
1005	+	return txt
1006
1007	+	def wsBuildExe(self, nj=0):
1008	+	"""
1009	+	Put in the script the commands to build an executable
1010	+	or a library.
1011	+	"""
1012	+
1013	+	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1014	+	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1015	+
1016	+	txt += 'mv $RUNTIME_AREA/lib . \n'
1017	+	txt += 'mv $RUNTIME_AREA/module . \n'
1018	+	txt += 'mv $RUNTIME_AREA/ProdCommon . \n'
1019	+
1020	+
1021	+	#if os.path.isfile(self.tgzNameWithPath):
1022	+	# txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1023	+	# txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1024	+	# txt += 'untar_status=$? \n'
1025	+	# txt += 'if [ $untar_status -ne 0 ]; then \n'
1026	+	# txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
1027	+	# txt += ' job_exit_code=$untar_status\n'
1028	+	# txt += ' func_exit\n'
1029	+	# txt += 'else \n'
1030	+	# txt += ' echo "Successful untar" \n'
1031	+	# txt += 'fi \n'
1032	+	# txt += '\n'
1033	+	# txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
1034	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1035	+	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
1036	+	txt += 'else\n'
1037	+	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1038	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1039	+	txt += 'fi\n'
1040	+	txt += '\n'
1041	+
1042		return txt
1043	+	############################################################################
1044
1045		def modifySteeringCards(self, nj):
1046		"""
1047	<	modify the card provided by the user,
1047	>	modify the card provided by the user,
1048		writing a new card into share dir
1049		"""
1050	<
1050	>
1051		def executableName(self):
1052		if self.scriptExe: #CarlosDaniele
1053		return "sh "
#	Line 1143 \| Line 1055 \| class Cmssw(JobType):
1055		return self.executable
1056
1057		def executableArgs(self):
1058	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1059		if self.scriptExe:#CarlosDaniele
1060		return self.scriptExe + " $NJob"
1061		else:
1149	–	# if >= CMSSW_1_5_X, add -e
1062		version_array = self.scram.getSWVersion().split('_')
1063		major = 0
1064		minor = 0
#	Line 1154 \| Line 1066 \| class Cmssw(JobType):
1066		major = int(version_array[1])
1067		minor = int(version_array[2])
1068		except:
1069	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1069	>	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1070		raise CrabException(msg)
1071	+
1072	+	ex_args = ""
1073	+
1074	+	# Framework job report
1075		if major >= 1 and minor >= 5 :
1076	<	return " -e -p pset.cfg"
1076	>	#ex_args += " -j " + self.fjrFileName
1077	>	### FEDE it could be improved!!! ####
1078	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1079	>	#######################################
1080	>	# Type of cfg file
1081	>	if major >= 2 :
1082	>	ex_args += " -p pset.pycfg"
1083		else:
1084	<	return " -p pset.cfg"
1084	>	ex_args += " -p pset.cfg"
1085	>	return ex_args
1086
1087		def inputSandbox(self, nj):
1088		"""
#	Line 1179 \| Line 1102 \| class Cmssw(JobType):
1102		## additional input files
1103		tgz = self.additionalInputFileTgz()
1104		inp_box.append(tgz)
1105	+	## executable
1106	+	wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1107	+	inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1108		return inp_box
1109
1110		def outputSandbox(self, nj):
#	Line 1189 \| Line 1115 \| class Cmssw(JobType):
1115
1116		## User Declared output files
1117		for out in (self.output_file+self.output_file_sandbox):
1118	<	n_out = nj + 1
1118	>	n_out = nj + 1
1119		out_box.append(self.numberFile_(out,str(n_out)))
1120		return out_box
1121
#	Line 1204 \| Line 1130 \| class Cmssw(JobType):
1130		Returns part of a job script which renames the produced files.
1131		"""
1132
1133	<	txt = '\n'
1134	<	txt += '# directory content\n'
1133	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1134	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1135	>	txt += 'echo ">>> current directory content:"\n'
1136		txt += 'ls \n'
1137	+	txt += '\n'
1138	+
1139	+	#txt += 'output_exit_status=0\n'
1140	+
1141	+	### FEDE #######
1142	+	#for fileWithSuffix in (self.output_file_sandbox):
1143	+	# output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1144	+	# txt += '\n'
1145	+	# txt += '# check output file\n'
1146	+	# txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1147	+	# txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1148	+	# txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1149	+	# txt += 'else\n'
1150	+	# txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1151	+	# txt += ' job_exit_code=60302\n'
1152	+	# if common.scheduler.name().upper() == 'CONDOR_G':
1153	+	# txt += ' if [ $middleware == OSG ]; then \n'
1154	+	# txt += ' echo "prepare dummy output file"\n'
1155	+	# txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1156	+	# txt += ' fi \n'
1157	+	# txt += 'fi\n'
1158
1211	–	txt += 'output_exit_status=0\n'
1212	–
1213	–	for fileWithSuffix in (self.output_file_sandbox):
1214	–	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1215	–	txt += '\n'
1216	–	txt += '# check output file\n'
1217	–	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1218	–	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1219	–	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1220	–	txt += 'else\n'
1221	–	txt += ' exit_status=60302\n'
1222	–	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1223	–	if common.scheduler.boss_scheduler_name == 'condor_g':
1224	–	txt += ' if [ $middleware == OSG ]; then \n'
1225	–	txt += ' echo "prepare dummy output file"\n'
1226	–	txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1227	–	txt += ' fi \n'
1228	–	txt += 'fi\n'
1229	–
1159		for fileWithSuffix in (self.output_file):
1160		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1161		txt += '\n'
1162		txt += '# check output file\n'
1163		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1164	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1165	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1164	>	if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1165	>	txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
1166	>	txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1167	>	else:
1168	>	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1169	>	txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1170		txt += 'else\n'
1171	<	txt += ' exit_status=60302\n'
1172	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1173	<	txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n'
1174	<	txt += ' output_exit_status=$exit_status\n'
1175	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1171	>	#txt += ' exit_status=60302\n'
1172	>	#txt += ' echo "ERROR: Output file '+fileWithSuffix+' not found"\n'
1173	>	#txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n'
1174	>	#txt += ' output_exit_status=$exit_status\n'
1175	>	txt += ' job_exit_code=60302\n'
1176	>	txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1177	>	if common.scheduler.name().upper() == 'CONDOR_G':
1178		txt += ' if [ $middleware == OSG ]; then \n'
1179		txt += ' echo "prepare dummy output file"\n'
1180		txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1248 \| Line 1183 \| class Cmssw(JobType):
1183		file_list = []
1184		for fileWithSuffix in (self.output_file):
1185		file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1186	<
1186	>
1187		txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1188	+	txt += '\n'
1189	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1190	+	txt += 'echo ">>> current directory content:"\n'
1191	+	txt += 'ls \n'
1192	+	txt += '\n'
1193		txt += 'cd $RUNTIME_AREA\n'
1194	+	txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1195		return txt
1196
1197		def numberFile_(self, file, txt):
#	Line 1268 \| Line 1209 \| class Cmssw(JobType):
1209		result = name + '_' + txt + "." + ext
1210		else:
1211		result = name + '_' + txt
1212	<
1212	>
1213		return result
1214
1215		def getRequirements(self, nj=[]):
1216		"""
1217	<	return job requirements to add to jdl files
1217	>	return job requirements to add to jdl files
1218		"""
1219		req = ''
1220		if self.version:
#	Line 1288 \| Line 1229 \| class Cmssw(JobType):
1229		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1230
1231		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1232	+	if common.scheduler.name() == "glitecoll":
1233	+	req += ' && other.GlueCEStateStatus == "Production" '
1234
1235		return req
1236
#	Line 1295 \| Line 1238 \| class Cmssw(JobType):
1238		""" return the config filename """
1239		return self.name()+'.cfg'
1240
1298	–	### OLI_DANIELE
1241		def wsSetupCMSOSGEnvironment_(self):
1242		"""
1243		Returns part of a job script which is prepares
1244		the execution environment and which is common for all CMS jobs.
1245		"""
1246	<	txt = '\n'
1247	<	txt += ' echo "### SETUP CMS OSG ENVIRONMENT ###"\n'
1248	<	txt += ' if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1249	<	txt += ' # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1250	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1251	<	txt += ' source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1310	<	txt += ' elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1246	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1247	>	txt += ' echo ">>> setup CMS OSG environment:"\n'
1248	>	txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1249	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1250	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1251	>	txt += ' if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1252		txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1253	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1254	<	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1255	<	txt += ' else\n'
1256	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1257	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1258	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1259	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1260	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1261	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1262	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1263	<	txt += ' exit 1\n'
1264	<	txt += '\n'
1265	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1266	<	txt += ' cd $RUNTIME_AREA\n'
1267	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1268	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1269	<	txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1270	<	txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1271	<	txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1272	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1332	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1333	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1334	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1335	<	txt += ' fi\n'
1253	>	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1254	>	txt += ' else\n'
1255	>	#txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1256	>	#txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1257	>	#txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1258	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1259	>	txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1260	>	txt += ' job_exit_code=10020\n'
1261	>	#txt += ' cd $RUNTIME_AREA\n'
1262	>	#txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1263	>	#txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
1264	>	#txt += ' /bin/rm -rf $WORKING_DIR\n'
1265	>	#txt += ' if [ -d $WORKING_DIR ] ;then\n'
1266	>	#txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1267	>	#txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1268	>	#txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1269	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1270	>	#txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1271	>	#txt += ' job_exit_code=10017\n'
1272	>	#txt += ' fi\n'
1273		txt += '\n'
1274	<	txt += ' exit 1\n'
1275	<	txt += ' fi\n'
1274	>	#txt += ' exit 1\n'
1275	>	txt += ' func_exit\n'
1276	>	txt += ' fi\n'
1277		txt += '\n'
1278	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1279	<	txt += ' echo " END SETUP CMS OSG ENVIRONMENT "\n'
1278	>	txt += ' echo "==> setup cms environment ok"\n'
1279	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1280
1281		return txt
1282	<
1282	>
1283		### OLI_DANIELE
1284		def wsSetupCMSLCGEnvironment_(self):
1285		"""
1286		Returns part of a job script which is prepares
1287		the execution environment and which is common for all CMS jobs.
1288		"""
1289	<	txt = ' \n'
1290	<	txt += ' echo " ### SETUP CMS LCG ENVIRONMENT ### "\n'
1291	<	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1292	<	txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1293	<	txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1294	<	txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1295	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1296	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1297	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1298	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1299	<	txt += ' exit 1\n'
1300	<	txt += ' else\n'
1301	<	txt += ' echo "Sourcing environment... "\n'
1302	<	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1303	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1304	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1305	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1306	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1307	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1308	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1309	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1310	<	txt += ' exit 1\n'
1311	<	txt += ' fi\n'
1312	<	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1313	<	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1314	<	txt += ' result=$?\n'
1315	<	txt += ' if [ $result -ne 0 ]; then\n'
1316	<	txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1317	<	txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1318	<	txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1319	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1320	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1321	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1322	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1323	<	txt += ' exit 1\n'
1324	<	txt += ' fi\n'
1325	<	txt += ' fi\n'
1326	<	txt += ' \n'
1327	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1328	<	txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1289	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1290	>	txt += ' echo ">>> setup CMS LCG environment:"\n'
1291	>	txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1292	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1293	>	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
1294	>	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1295	>	#txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1296	>	#txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1297	>	#txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1298	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1299	>	#txt += ' exit 1\n'
1300	>	txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1301	>	txt += ' job_exit_code=10031\n'
1302	>	txt += ' func_exit\n'
1303	>	txt += ' else\n'
1304	>	txt += ' echo "Sourcing environment... "\n'
1305	>	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1306	>	#txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1307	>	#txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1308	>	#txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1309	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1310	>	#txt += ' exit 1\n'
1311	>	txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1312	>	txt += ' job_exit_code=10020\n'
1313	>	txt += ' func_exit\n'
1314	>	txt += ' fi\n'
1315	>	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1316	>	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1317	>	txt += ' result=$?\n'
1318	>	txt += ' if [ $result -ne 0 ]; then\n'
1319	>	#txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1320	>	#txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1321	>	#txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1322	>	#txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1323	>	#txt += ' exit 1\n'
1324	>	txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1325	>	txt += ' job_exit_code=10032\n'
1326	>	txt += ' func_exit\n'
1327	>	txt += ' fi\n'
1328	>	txt += ' fi\n'
1329	>	txt += ' \n'
1330	>	txt += ' echo "==> setup cms environment ok"\n'
1331		return txt
1332
1333	<	### FEDE FOR DBS OUTPUT PUBLICATION
1333	>	### FEDE FOR DBS OUTPUT PUBLICATION
1334		def modifyReport(self, nj):
1335		"""
1336	<	insert the part of the script that modifies the FrameworkJob Report
1336	>	insert the part of the script that modifies the FrameworkJob Report
1337		"""
1338
1339	<	txt = ''
1339	>	txt = '\n#Written by cms_cmssw::modifyReport\n'
1340		try:
1341	<	publish_data = int(self.cfg_params['USER.publish_data'])
1341	>	publish_data = int(self.cfg_params['USER.publish_data'])
1342		except KeyError:
1343		publish_data = 0
1344	<	if (publish_data == 1):
1345	<	txt += 'echo "Modify Job Report" \n'
1346	<	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1347	<	################ FEDE FOR DBS2 #############################################
1348	<	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1349	<	#############################################################################
1350	<	#try:
1351	<	# publish_data = int(self.cfg_params['USER.publish_data'])
1352	<	#except KeyError:
1353	<	# publish_data = 0
1354	<
1355	<	txt += 'if [ -z "$SE" ]; then\n'
1356	<	txt += ' SE="" \n'
1417	<	txt += 'fi \n'
1418	<	txt += 'if [ -z "$SE_PATH" ]; then\n'
1419	<	txt += ' SE_PATH="" \n'
1420	<	txt += 'fi \n'
1421	<	txt += 'echo "SE = $SE"\n'
1422	<	txt += 'echo "SE_PATH = $SE_PATH"\n'
1344	>	if (publish_data == 1):
1345	>
1346	>	txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1347	>	txt += ' echo ">>> Modify Job Report:" \n'
1348	>	txt += ' chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1349	>	#txt += ' if [ -z "$SE" ]; then\n'
1350	>	#txt += ' SE="" \n'
1351	>	#txt += ' fi \n'
1352	>	#txt += ' if [ -z "$SE_PATH" ]; then\n'
1353	>	#txt += ' SE_PATH="" \n'
1354	>	#txt += ' fi \n'
1355	>	txt += ' echo "SE = $SE"\n'
1356	>	txt += ' echo "SE_PATH = $SE_PATH"\n'
1357
1424	–	#if (publish_data == 1):
1425	–	#processedDataset = self.cfg_params['USER.processed_datasetname']
1358		processedDataset = self.cfg_params['USER.publish_data_name']
1359	<	txt += 'ProcessedDataset='+processedDataset+'\n'
1360	<	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1361	<	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1362	<	#### FEDE: added slash in LFN ##############
1363	<	txt += ' FOR_LFN=/copy_problems/ \n'
1364	<	txt += 'else \n'
1359	>	txt += ' ProcessedDataset='+processedDataset+'\n'
1360	>	#txt += ' if [ "$SE_PATH" == "" ]; then\n'
1361	>	#txt += ' FOR_LFN=/copy_problems/ \n'
1362	>	#txt += ' else \n'
1363	>	#txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1364	>	#txt += ' FOR_LFN=/store$tmp \n'
1365	>	#txt += ' fi \n'
1366		txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1434	–	##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1367		txt += ' FOR_LFN=/store$tmp \n'
1368	<	txt += 'fi \n'
1369	<	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1370	<	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1371	<	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1372	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1373	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1374	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1375	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1376	<
1377	<	txt += 'modifyReport_result=$?\n'
1378	<	txt += 'echo modifyReport_result = $modifyReport_result\n'
1379	<	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1380	<	txt += ' exit_status=1\n'
1381	<	txt += ' echo "ERROR: Problem with ModifyJobReport"\n'
1382	<	txt += 'else\n'
1383	<	txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1368	>	txt += ' echo "ProcessedDataset = $ProcessedDataset"\n'
1369	>	txt += ' echo "FOR_LFN = $FOR_LFN" \n'
1370	>	txt += ' echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1371	>	#txt += ' echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1372	>	#txt += ' $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1373	>	### FEDE ####
1374	>	txt += ' echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1375	>	txt += ' $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1376	>	####################################
1377	>	txt += ' modifyReport_result=$?\n'
1378	>	txt += ' if [ $modifyReport_result -ne 0 ]; then\n'
1379	>	txt += ' modifyReport_result=70500\n'
1380	>	txt += ' job_exit_code=$modifyReport_result\n'
1381	>	txt += ' echo "ModifyReportResult=$modifyReport_result" \| tee -a $RUNTIME_AREA/$repo\n'
1382	>	txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
1383	>	txt += ' else\n'
1384	>	### FEDE #####
1385	>	#txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1386	>	#######################
1387	>	txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1388	>	txt += ' fi\n'
1389		txt += 'fi\n'
1453	–	else:
1454	–	txt += 'echo "no data publication required"\n'
1455	–	#txt += 'ProcessedDataset=no_data_to_publish \n'
1456	–	#### FEDE: added slash in LFN ##############
1457	–	#txt += 'FOR_LFN=/local/ \n'
1458	–	#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1459	–	#txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1390		return txt
1391
1392		def cleanEnv(self):
1393	<	### OLI_DANIELE
1394	<	txt = ''
1465	<	txt += 'if [ $middleware == OSG ]; then\n'
1393	>	txt = '\n#Written by cms_cmssw::cleanEnv\n'
1394	>	txt += 'if [ $middleware == OSG ]; then\n'
1395		txt += ' cd $RUNTIME_AREA\n'
1396	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1396	>	txt += ' echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1397	>	txt += ' echo ">>> Remove working directory: $WORKING_DIR"\n'
1398		txt += ' /bin/rm -rf $WORKING_DIR\n'
1399		txt += ' if [ -d $WORKING_DIR ] ;then\n'
1400	<	txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401	<	txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1402	<	txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1473	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1474	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1475	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1476	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1400	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1401	>	txt += ' job_exit_code=10017\n'
1402	>	txt += ' func_exit\n'
1403		txt += ' fi\n'
1404		txt += 'fi\n'
1405		txt += '\n'
#	Line 1485 \| Line 1411 \| class Cmssw(JobType):
1411		def getParams(self):
1412		return self._params
1413
1488	–	def setTaskid_(self):
1489	–	self._taskId = self.cfg_params['taskId']
1490	–
1491	–	def getTaskid(self):
1492	–	return self._taskId
1493	–
1414		def uniquelist(self, old):
1415		"""
1416		remove duplicates from a list
#	Line 1500 \| Line 1420 \| class Cmssw(JobType):
1420		nd[e]=0
1421		return nd.keys()
1422
1423	<
1504	<	def checkOut(self, limit):
1423	>	def outList(self):
1424		"""
1425		check the dimension of the output files
1426		"""
1427	<	txt = 'echo "*****************************************"\n'
1428	<	txt += 'echo " Starting output sandbox limit check "\n'
1510	<	txt += 'echo "*****************************************"\n'
1511	<	allOutFiles = ""
1427	>	txt = ''
1428	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1429		listOutFiles = []
1430	<	for fileOut in (self.output_file+self.output_file_sandbox):
1431	<	if fileOut.find('crab_fjr') == -1:
1432	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1433	<	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1434	<	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1435	<	txt += 'ls -gGhrta;\n'
1436	<	txt += 'sum=0;\n'
1437	<	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1438	<	txt += ' if [ -e $file ]; then\n'
1439	<	txt += ' tt=`ls -gGrta $file \| awk \'{ print $3 }\'`\n'
1440	<	txt += ' sum=`expr $sum + $tt`\n'
1441	<	txt += ' else\n'
1442	<	txt += ' echo "WARNING: output file $file not found!"\n'
1443	<	txt += ' fi\n'
1444	<	txt += 'done\n'
1445	<	txt += 'echo "Total Output dimension: $sum";\n'
1529	<	txt += 'limit='+str(limit)+';\n'
1530	<	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1531	<	txt += 'if [ $limit -lt $sum ]; then\n'
1532	<	txt += ' echo "WARNING: output files have to big size - something will be lost;"\n'
1533	<	txt += ' echo " checking the output file sizes..."\n'
1534	<	"""
1535	<	txt += ' dim=0;\n'
1536	<	txt += ' exclude=0;\n'
1537	<	txt += ' for files in '+str(allOutFiles)+' ; do\n'
1538	<	txt += ' sumTemp=0;\n'
1539	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1540	<	txt += ' if [ $file != $file2 ]; then\n'
1541	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1542	<	txt += ' sumTemp=`expr $sumTemp + $tt`;\n'
1543	<	txt += ' fi\n'
1544	<	txt += ' done\n'
1545	<	txt += ' if [ $sumTemp -lt $limit ]; then\n'
1546	<	txt += ' if [ $dim -lt $sumTemp ]; then\n'
1547	<	txt += ' dim=$sumTemp;\n'
1548	<	txt += ' exclude=$file;\n'
1549	<	txt += ' fi\n'
1550	<	txt += ' fi\n'
1551	<	txt += ' done\n'
1552	<	txt += ' echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1553	<	"""
1554	<	txt += ' tot=0;\n'
1555	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1556	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1557	<	txt += ' tot=`expr $tot + $tt`;\n'
1558	<	txt += ' if [ $limit -lt $tot ]; then\n'
1559	<	txt += ' tot=`expr $tot - $tt`;\n'
1560	<	txt += ' fileLast=$file;\n'
1561	<	txt += ' break;\n'
1562	<	txt += ' fi\n'
1563	<	txt += ' done\n'
1564	<	txt += ' echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1565	<	txt += ' flag=0;\n'
1566	<	txt += ' for filess in '+str(allOutFiles)+' ; do\n'
1567	<	txt += ' if [ $fileLast = $filess ]; then\n'
1568	<	txt += ' flag=1;\n'
1569	<	txt += ' fi\n'
1570	<	txt += ' if [ $flag -eq 1 ]; then\n'
1571	<	txt += ' rm -f $filess;\n'
1572	<	txt += ' fi\n'
1573	<	txt += ' done\n'
1574	<	txt += ' ls -agGhrt;\n'
1575	<	txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1576	<	txt += ' echo "JOB_EXIT_STATUS = 70000";\n'
1577	<	txt += ' exit_status=70000;\n'
1578	<	txt += 'else'
1579	<	txt += ' echo "Total Output dimension $sum is fine.";\n'
1580	<	txt += 'fi\n'
1581	<	txt += 'echo "*****************************************"\n'
1582	<	txt += 'echo "* Ending output sandbox limit check *"\n'
1583	<	txt += 'echo "*****************************************"\n'
1584	<	return txt
1430	>	stdout = 'CMSSW_$NJob.stdout'
1431	>	stderr = 'CMSSW_$NJob.stderr'
1432	>	if (self.return_data == 1):
1433	>	for file in (self.output_file+self.output_file_sandbox):
1434	>	listOutFiles.append(self.numberFile_(file, '$NJob'))
1435	>	listOutFiles.append(stdout)
1436	>	listOutFiles.append(stderr)
1437	>	else:
1438	>	for file in (self.output_file_sandbox):
1439	>	listOutFiles.append(self.numberFile_(file, '$NJob'))
1440	>	listOutFiles.append(stdout)
1441	>	listOutFiles.append(stderr)
1442	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1443	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1444	>	txt += 'export filesToCheck\n'
1445	>	return txt

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing COMP/CRAB/python/cms_cmssw.py (file contents): Revision 1.116.2.8 by fanzago, Wed Oct 17 13:25:00 2007 UTC vs. Revision 1.169 by spiga, Tue Apr 1 14:53:36 2008 UTC

Diff Legend

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.116.2.8 by fanzago, Wed Oct 17 13:25:00 2007 UTC vs.
Revision 1.169 by spiga, Tue Apr 1 14:53:36 2008 UTC