ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.35 by gutsche, Thu Aug 3 22:44:40 2006 UTC vs.
Revision 1.106 by slacapra, Mon Jul 23 15:56:29 2007 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	–	import math
5		import common
7	–	import PsetManipulator
8	–
9	–	import DBSInfo_EDM
10	–	import DataDiscovery_EDM
11	–	import DataLocation_EDM
6		import Scram
7
8	<	import os, string, re
8	>	import os, string, glob
9
10		class Cmssw(JobType):
11	<	def __init__(self, cfg_params):
11	>	def __init__(self, cfg_params, ncjobs):
12		JobType.__init__(self, 'CMSSW')
13		common.logger.debug(3,'CMSSW::__init__')
14
21	–	self.analisys_common_info = {}
22	–	# Marco.
15		self._params = {}
16		self.cfg_params = cfg_params
17	+
18	+	try:
19	+	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
20	+	except KeyError:
21	+	self.MaxTarBallSize = 9.5
22	+
23	+	# number of jobs requested to be created, limit obj splitting
24	+	self.ncjobs = ncjobs
25	+
26		log = common.logger
27
28		self.scram = Scram.Scram(cfg_params)
28	–	scramArea = ''
29		self.additional_inbox_files = []
30		self.scriptExe = ''
31		self.executable = ''
32	+	self.executable_arch = self.scram.getArch()
33		self.tgz_name = 'default.tgz'
34	+	self.additional_tgz_name = 'additional.tgz'
35	+	self.scriptName = 'CMSSW.sh'
36	+	self.pset = ''      #scrip use case Da
37	+	self.datasetPath = '' #scrip use case Da
38
39	+	# set FJR file name
40	+	self.fjrFileName = 'crab_fjr.xml'
41
42		self.version = self.scram.getSWVersion()
43	+	common.taskDB.setDict('codeVersion',self.version)
44		self.setParam_('application', self.version)
37	–	common.analisys_common_info['sw_version'] = self.version
38	–	### FEDE
39	–	common.analisys_common_info['copy_input_data'] = 0
40	–	common.analisys_common_info['events_management'] = 1
45
46		### collect Data cards
47	+
48	+	## get DBS mode
49	+	try:
50	+	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
51	+	except KeyError:
52	+	self.use_dbs_1 = 0
53	+
54		try:
55		tmp =  cfg_params['CMSSW.datasetpath']
56		log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
#	Line 59 | Line 70 | class Cmssw(JobType):
70		self.setParam_('dataset', 'None')
71		self.setParam_('owner', 'None')
72		else:
73	<	datasetpath_split = self.datasetPath.split("/")
74	<	self.setParam_('dataset', datasetpath_split[1])
75	<	self.setParam_('owner', datasetpath_split[-1])
76	<
73	>	try:
74	>	datasetpath_split = self.datasetPath.split("/")
75	>	# standard style
76	>	if self.use_dbs_1 == 1 :
77	>	self.setParam_('dataset', datasetpath_split[1])
78	>	self.setParam_('owner', datasetpath_split[-1])
79	>	else:
80	>	self.setParam_('dataset', datasetpath_split[1])
81	>	self.setParam_('owner', datasetpath_split[2])
82	>	except:
83	>	self.setParam_('dataset', self.datasetPath)
84	>	self.setParam_('owner', self.datasetPath)
85	>
86		self.setTaskid_()
87		self.setParam_('taskId', self.cfg_params['taskId'])
88
#	Line 85 | Line 105 | class Cmssw(JobType):
105		try:
106		self.pset = cfg_params['CMSSW.pset']
107		log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
108	<	if (not os.path.exists(self.pset)):
109	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
108	>	if self.pset.lower() != 'none' :
109	>	if (not os.path.exists(self.pset)):
110	>	raise CrabException("User defined PSet file "+self.pset+" does not exist")
111	>	else:
112	>	self.pset = None
113		except KeyError:
114		raise CrabException("PSet file missing. Cannot run cmsRun ")
115
116		# output files
117	+	## stuff which must be returned always via sandbox
118	+	self.output_file_sandbox = []
119	+
120	+	# add fjr report by default via sandbox
121	+	self.output_file_sandbox.append(self.fjrFileName)
122	+
123	+	# other output files to be returned via sandbox or copied to SE
124		try:
125		self.output_file = []
96	–
126		tmp = cfg_params['CMSSW.output_file']
127		if tmp != '':
128		tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
#	Line 103 | Line 132 | class Cmssw(JobType):
132		self.output_file.append(tmp)
133		pass
134		else:
135	<	log.message("No output file defined: only stdout/err will be available")
135	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
136		pass
137		pass
138		except KeyError:
139	<	log.message("No output file defined: only stdout/err will be available")
139	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
140		pass
141
142		# script_exe file as additional file in inputSandbox
143		try:
144		self.scriptExe = cfg_params['USER.script_exe']
116	–	self.additional_inbox_files.append(self.scriptExe)
145		if self.scriptExe != '':
146		if not os.path.isfile(self.scriptExe):
147	<	msg ="WARNING. file "+self.scriptExe+" not found"
147	>	msg ="ERROR. file "+self.scriptExe+" not found"
148		raise CrabException(msg)
149	+	self.additional_inbox_files.append(string.strip(self.scriptExe))
150		except KeyError:
151	<	pass
152	<
151	>	self.scriptExe = ''
152	>
153	>	#CarlosDaniele
154	>	if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
155	>	msg ="Error. script_exe  not defined"
156	>	raise CrabException(msg)
157	>
158		## additional input files
159		try:
160		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
161		for tmp in tmpAddFiles:
162	<	if not os.path.exists(tmp):
163	<	raise CrabException("Additional input file not found: "+tmp)
164	<	self.additional_inbox_files.append(string.strip(tmp))
162	>	tmp = string.strip(tmp)
163	>	dirname = ''
164	>	if not tmp[0]=="/": dirname = "."
165	>	files = []
166	>	if string.find(tmp,"*")>-1:
167	>	files = glob.glob(os.path.join(dirname, tmp))
168	>	if len(files)==0:
169	>	raise CrabException("No additional input file found with this pattern: "+tmp)
170	>	else:
171	>	files.append(tmp)
172	>	for file in files:
173	>	if not os.path.exists(file):
174	>	raise CrabException("Additional input file not found: "+file)
175	>	pass
176	>	# fname = string.split(file, '/')[-1]
177	>	# storedFile = common.work_space.pathForTgz()+'share/'+fname
178	>	# shutil.copyfile(file, storedFile)
179	>	self.additional_inbox_files.append(string.strip(file))
180		pass
181		pass
182	+	common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
183		except KeyError:
184		pass
185
#	Line 163 | Line 213 | class Cmssw(JobType):
213		self.total_number_of_events = 0
214		self.selectTotalNumberEvents = 0
215
216	<	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
217	<	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
218	<	raise CrabException(msg)
216	>	if self.pset != None: #CarlosDaniele
217	>	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
218	>	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
219	>	raise CrabException(msg)
220	>	else:
221	>	if (self.selectNumberOfJobs == 0):
222	>	msg = 'Must specify  number_of_jobs.'
223	>	raise CrabException(msg)
224
225		## source seed for pythia
226		try:
#	Line 180 | Line 235 | class Cmssw(JobType):
235		self.sourceSeedVtx = None
236		common.logger.debug(5,"No vertex seed given")
237
238	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
238	>	try:
239	>	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
240	>	except KeyError:
241	>	self.sourceSeedG4 = None
242	>	common.logger.debug(5,"No g4 sim hits seed given")
243	>
244	>	try:
245	>	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
246	>	except KeyError:
247	>	self.sourceSeedMix = None
248	>	common.logger.debug(5,"No mix seed given")
249	>
250	>	try:
251	>	self.firstRun = int(cfg_params['CMSSW.first_run'])
252	>	except KeyError:
253	>	self.firstRun = None
254	>	common.logger.debug(5,"No first run given")
255	>	if self.pset != None: #CarlosDaniele
256	>	ver = string.split(self.version,"_")
257	>	if (int(ver[1])>=1 and int(ver[2])>=5):
258	>	import PsetManipulator150 as pp
259	>	else:
260	>	import PsetManipulator as pp
261	>	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
262
263		#DBSDLS-start
264		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
#	Line 197 | Line 275 | class Cmssw(JobType):
275		self.tgzNameWithPath = self.getTarBall(self.executable)
276
277		## Select Splitting
278	<	if self.selectNoInput: self.jobSplittingNoInput()
279	<	else: self.jobSplittingByBlocks(blockSites)
278	>	if self.selectNoInput:
279	>	if self.pset == None: #CarlosDaniele
280	>	self.jobSplittingForScript()
281	>	else:
282	>	self.jobSplittingNoInput()
283	>	else:
284	>	self.jobSplittingByBlocks(blockSites)
285
286		# modify Pset
287	<	try:
288	<	if (self.datasetPath): # standard job
289	<	# always process all events in a file
290	<	self.PsetEdit.maxEvent("-1")
291	<	self.PsetEdit.inputModule("INPUT")
292	<
293	<	else:  # pythia like job
294	<	self.PsetEdit.maxEvent(self.eventsPerJob)
295	<	if (self.sourceSeed) :
296	<	self.PsetEdit.pythiaSeed("INPUT")
297	<	if (self.sourceSeedVtx) :
298	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
299	<	self.PsetEdit.psetWriter(self.configFilename())
300	<	except:
301	<	msg='Error while manipuliating ParameterSet: exiting...'
302	<	raise CrabException(msg)
287	>	if self.pset != None: #CarlosDaniele
288	>	try:
289	>	if (self.datasetPath): # standard job
290	>	# allow to processa a fraction of events in a file
291	>	PsetEdit.inputModule("INPUT")
292	>	PsetEdit.maxEvent("INPUTMAXEVENTS")
293	>	PsetEdit.skipEvent("INPUTSKIPEVENTS")
294	>	else:  # pythia like job
295	>	PsetEdit.maxEvent(self.eventsPerJob)
296	>	if (self.firstRun):
297	>	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
298	>	if (self.sourceSeed) :
299	>	PsetEdit.pythiaSeed("INPUT")
300	>	if (self.sourceSeedVtx) :
301	>	PsetEdit.vtxSeed("INPUTVTX")
302	>	if (self.sourceSeedG4) :
303	>	self.PsetEdit.g4Seed("INPUTG4")
304	>	if (self.sourceSeedMix) :
305	>	self.PsetEdit.mixSeed("INPUTMIX")
306	>	# add FrameworkJobReport to parameter-set
307	>	PsetEdit.addCrabFJR(self.fjrFileName)
308	>	PsetEdit.psetWriter(self.configFilename())
309	>	except:
310	>	msg='Error while manipuliating ParameterSet: exiting...'
311	>	raise CrabException(msg)
312
313		def DataDiscoveryAndLocation(self, cfg_params):
314
315	+	import DataDiscovery
316	+	import DataDiscovery_DBS2
317	+	import DataLocation
318		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
319
320		datasetPath=self.datasetPath
321
227	–	## TODO
228	–	dataTiersList = ""
229	–	dataTiers = dataTiersList.split(',')
230	–
322		## Contact the DBS
323	+	common.logger.message("Contacting Data Discovery Services ...")
324		try:
325	<	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
325	>
326	>	if self.use_dbs_1 == 1 :
327	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
328	>	else :
329	>	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
330		self.pubdata.fetchDBSInfo()
331
332	<	except DataDiscovery_EDM.NotExistingDatasetError, ex :
332	>	except DataDiscovery.NotExistingDatasetError, ex :
333		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
334		raise CrabException(msg)
335	<
240	<	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
335	>	except DataDiscovery.NoDataTierinProvenanceError, ex :
336		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
337		raise CrabException(msg)
338	<	except DataDiscovery_EDM.DataDiscoveryError, ex:
339	<	msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
338	>	except DataDiscovery.DataDiscoveryError, ex:
339	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
340	>	raise CrabException(msg)
341	>	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
342	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
343	>	raise CrabException(msg)
344	>	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
345	>	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
346	>	raise CrabException(msg)
347	>	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
348	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
349		raise CrabException(msg)
246	–
247	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
248	–	## self.DBSPaths=self.pubdata.getDBSPaths()
249	–	common.logger.message("Required data are :"+self.datasetPath)
350
351		self.filesbyblock=self.pubdata.getFiles()
352	<	self.eventsbyblock=self.pubdata.getEVC()
353	<	## SL we probably don't need this
254	<	self.files = self.filesbyblock.values()
352	>	self.eventsbyblock=self.pubdata.getEventsPerBlock()
353	>	self.eventsbyfile=self.pubdata.getEventsPerFile()
354
355		## get max number of events
356		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
258	–	common.logger.message("\nThe number of available events is %s"%self.maxEvents)
357
358		## Contact the DLS and build a list of sites hosting the fileblocks
359		try:
360	<	dataloc=DataLocation_EDM.DataLocation_EDM(self.filesbyblock.keys(),cfg_params)
360	>	dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
361		dataloc.fetchDLSInfo()
362	<	except DataLocation_EDM.DataLocationError , ex:
362	>	except DataLocation.DataLocationError , ex:
363		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
364		raise CrabException(msg)
365
#	Line 269 | Line 367 | class Cmssw(JobType):
367		sites = dataloc.getSites()
368		allSites = []
369		listSites = sites.values()
370	<	for list in listSites:
371	<	for oneSite in list:
370	>	for listSite in listSites:
371	>	for oneSite in listSite:
372		allSites.append(oneSite)
373		allSites = self.uniquelist(allSites)
374
375	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
376	<	common.logger.debug(6, "List of Sites: "+str(allSites))
377	<	self.setParam_('TargetCE', ','.join(sites))
375	>	# screen output
376	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
377	>
378		return sites
379
380		def jobSplittingByBlocks(self, blockSites):
#	Line 311 | Line 409 | class Cmssw(JobType):
409		else:
410		eventsRemaining = totalEventsRequested
411
412	+	# If user requested more events per job than are in the dataset
413	+	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
414	+	eventsPerJobRequested = self.maxEvents
415	+
416		# For user info at end
417		totalEventCount = 0
418
#	Line 320 | Line 422 | class Cmssw(JobType):
422		if (self.selectNumberOfJobs):
423		common.logger.message("May not create the exact number_of_jobs requested.")
424
425	+	if ( self.ncjobs == 'all' ) :
426	+	totalNumberOfJobs = 999999999
427	+	else :
428	+	totalNumberOfJobs = self.ncjobs
429	+
430	+
431		blocks = blockSites.keys()
432		blockCount = 0
433		# Backup variable in case self.maxEvents counted events in a non-included block
#	Line 328 | Line 436 | class Cmssw(JobType):
436		jobCount = 0
437		list_of_lists = []
438
439	+	# list tracking which jobs are in which jobs belong to which block
440	+	jobsOfBlock = {}
441	+
442		# ---- Iterate over the blocks in the dataset until ---- #
443		# ---- we've met the requested total # of events    ---- #
444	<	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) ):
444	>	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
445		block = blocks[blockCount]
446	<
447	<
448	<	evInBlock = self.eventsbyblock[block]
449	<	common.logger.debug(5,'Events in Block File '+str(evInBlock))
450	<
451	<	#Correct - switch to this when DBS up
452	<	#numEventsInBlock = self.eventsbyblock[block]
342	<	numEventsInBlock = evInBlock
446	>	blockCount += 1
447	>	if block not in jobsOfBlock.keys() :
448	>	jobsOfBlock[block] = []
449	>
450	>	if self.eventsbyblock.has_key(block) :
451	>	numEventsInBlock = self.eventsbyblock[block]
452	>	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
453
454	<	files = self.filesbyblock[block]
455	<	numFilesInBlock = len(files)
456	<	if (numFilesInBlock <= 0):
457	<	continue
458	<	fileCount = 0
459	<
460	<	# ---- New block => New job ---- #
461	<	parString = "\\{"
462	<	jobEventCount = 0
454	>	files = self.filesbyblock[block]
455	>	numFilesInBlock = len(files)
456	>	if (numFilesInBlock <= 0):
457	>	continue
458	>	fileCount = 0
459	>
460	>	# ---- New block => New job ---- #
461	>	parString = "\\{"
462	>	# counter for number of events in files currently worked on
463	>	filesEventCount = 0
464	>	# flag if next while loop should touch new file
465	>	newFile = 1
466	>	# job event counter
467	>	jobSkipEventCount = 0
468
469	<	# ---- Iterate over the files in the block until we've met the requested ---- #
470	<	# ---- total # of events or we've gone over all the files in this block  ---- #
471	<	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ):
472	<	file = files[fileCount]
473	<	fileCount = fileCount + 1
474	<	#numEventsInFile = numberEventsInFile(file)
475	<	# !!!!!!!!!!!!!!!!!           Need to get the # of events in each file.         !!!!!!!!!!!!!!!!!!!!!!!!!
476	<	# For now, I'm assuming that all files in a block
477	<	# have the same number of events
478	<	numEventsInFile = numEventsInBlock/numFilesInBlock
479	<	common.logger.debug(5,"Estimated # of events in the file: "+str(numEventsInFile))
480	<	numEventsInFile = int(numEventsInFile)
481	<	common.logger.debug(5,"After rounding down: "+str(numEventsInFile))
482	<	# Add file to current job
483	<	parString += '\\\"' + file + '\\\"\,'
484	<	jobEventCount = jobEventCount + numEventsInFile
485	<	totalEventCount = totalEventCount + numEventsInFile
486	<	eventsRemaining = eventsRemaining - numEventsInFile
487	<	if (jobEventCount >= eventsPerJobRequested):
488	<	# ---- This job has at least CMSSW.events_per_job => End of job ---- #
489	<	# Don't need the last \,
490	<	fullString = parString[:-2]
491	<	fullString += '\\}'
492	<	list_of_lists.append([fullString])
493	<	common.logger.message("Job "+str(jobCount+1)+" can run over approximately "+str(jobEventCount)+" events.")
494	<
495	<	#self.jobDestination[jobCount] = blockSites[block]
496	<	self.jobDestination.append(blockSites[block])
497	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
498	<	if ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) ):
499	<	# ---- Still need CMSSW.total_number_of_events ---- #
500	<	# ---- and not about to jump into a new block  ---- #
501	<	# ---- => New job                              ---- #
469	>	# ---- Iterate over the files in the block until we've met the requested ---- #
470	>	# ---- total # of events or we've gone over all the files in this block  ---- #
471	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
472	>	file = files[fileCount]
473	>	if newFile :
474	>	try:
475	>	numEventsInFile = self.eventsbyfile[file]
476	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
477	>	# increase filesEventCount
478	>	filesEventCount += numEventsInFile
479	>	# Add file to current job
480	>	parString += '\\\"' + file + '\\\"\,'
481	>	newFile = 0
482	>	except KeyError:
483	>	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
484	>
485	>
486	>	# if less events in file remain than eventsPerJobRequested
487	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
488	>	# if last file in block
489	>	if ( fileCount == numFilesInBlock-1 ) :
490	>	# end job using last file, use remaining events in block
491	>	# close job and touch new file
492	>	fullString = parString[:-2]
493	>	fullString += '\\}'
494	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
495	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
496	>	self.jobDestination.append(blockSites[block])
497	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
498	>	# fill jobs of block dictionary
499	>	jobsOfBlock[block].append(jobCount+1)
500	>	# reset counter
501	>	jobCount = jobCount + 1
502	>	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
503	>	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
504	>	jobSkipEventCount = 0
505	>	# reset file
506	>	parString = "\\{"
507	>	filesEventCount = 0
508	>	newFile = 1
509	>	fileCount += 1
510	>	else :
511	>	# go to next file
512	>	newFile = 1
513	>	fileCount += 1
514	>	# if events in file equal to eventsPerJobRequested
515	>	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
516	>	# close job and touch new file
517	>	fullString = parString[:-2]
518	>	fullString += '\\}'
519	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
520	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
521	>	self.jobDestination.append(blockSites[block])
522	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
523	>	jobsOfBlock[block].append(jobCount+1)
524	>	# reset counter
525	>	jobCount = jobCount + 1
526	>	totalEventCount = totalEventCount + eventsPerJobRequested
527	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
528	>	jobSkipEventCount = 0
529	>	# reset file
530		parString = "\\{"
531	<	jobEventCount = 0
531	>	filesEventCount = 0
532	>	newFile = 1
533	>	fileCount += 1
534	>
535	>	# if more events in file remain than eventsPerJobRequested
536	>	else :
537	>	# close job but don't touch new file
538	>	fullString = parString[:-2]
539	>	fullString += '\\}'
540	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
541	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
542	>	self.jobDestination.append(blockSites[block])
543	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
544	>	jobsOfBlock[block].append(jobCount+1)
545	>	# increase counter
546		jobCount = jobCount + 1
547	+	totalEventCount = totalEventCount + eventsPerJobRequested
548	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
549	+	# calculate skip events for last file
550	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
551	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
552	+	# remove all but the last file
553	+	filesEventCount = self.eventsbyfile[file]
554	+	parString = "\\{"
555	+	parString += '\\\"' + file + '\\\"\,'
556		pass # END if
557	<	pass # END if
392	<	pass # END while (iterate over files in the block)
393	<	if (jobEventCount < eventsPerJobRequested):
394	<	# ---- Job ending prematurely due to end of block => End of job ---- #
395	<	# Don't need the last \,
396	<	fullString = parString[:-2]
397	<	fullString += '\\}'
398	<	list_of_lists.append([fullString])
399	<	common.logger.message("Job "+str(jobCount+1)+" can run over approximately "+str(jobEventCount)+" events.")
400	<	#self.jobDestination[jobCount] = blockSites[block]
401	<	self.jobDestination.append(blockSites[block])
402	<	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
403	<	pass # END if
404	<	blockCount = blockCount + 1
405	<	jobCount = jobCount + 1
557	>	pass # END while (iterate over files in the block)
558		pass # END while (iterate over blocks in the dataset)
559	<	self.total_number_of_jobs = jobCount
560	<	if (eventsRemaining > 0):
559	>	self.ncjobs = self.total_number_of_jobs = jobCount
560	>	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
561		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
562	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on approximately "+str(totalEventCount)+" events.\n")
562	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
563
564	+	# screen output
565	+	screenOutput = "List of jobs and available destination sites:\n\n"
566	+
567	+	blockCounter = 0
568	+	for block in blocks:
569	+	if block in jobsOfBlock.keys() :
570	+	blockCounter += 1
571	+	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
572	+
573	+	common.logger.message(screenOutput)
574	+
575		self.list_of_args = list_of_lists
576		return
577
#	Line 426 | Line 589 | class Cmssw(JobType):
589		raise CrabException(msg)
590
591		if (self.selectEventsPerJob):
592	<	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
592	>	if (self.selectTotalNumberEvents):
593	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
594	>	elif(self.selectNumberOfJobs) :
595	>	self.total_number_of_jobs =self.theNumberOfJobs
596	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
597	>
598		elif (self.selectNumberOfJobs) :
599		self.total_number_of_jobs = self.theNumberOfJobs
600		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
601	<
601	>
602		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
603
604		# is there any remainder?
#	Line 446 | Line 614 | class Cmssw(JobType):
614		self.list_of_args = []
615		for i in range(self.total_number_of_jobs):
616		## Since there is no input, any site is good
617	<	self.jobDestination.append(["Any"])
617	>	# self.jobDestination.append(["Any"])
618	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
619	>	args=[]
620	>	if (self.firstRun):
621	>	## pythia first run
622	>	#self.list_of_args.append([(str(self.firstRun)+str(i))])
623	>	args.append(str(self.firstRun)+str(i))
624	>	else:
625	>	## no first run
626	>	#self.list_of_args.append([str(i)])
627	>	args.append(str(i))
628		if (self.sourceSeed):
629	+	args.append(str(self.sourceSeed)+str(i))
630		if (self.sourceSeedVtx):
631	<	## pythia + vtx random seed
632	<	self.list_of_args.append([
633	<	str(self.sourceSeed)+str(i),
634	<	str(self.sourceSeedVtx)+str(i)
635	<	])
636	<	else:
637	<	## only pythia random seed
638	<	self.list_of_args.append([(str(self.sourceSeed)+str(i))])
639	<	else:
640	<	## no random seed
641	<	self.list_of_args.append([str(i)])
642	<	#print self.list_of_args
631	>	## + vtx random seed
632	>	args.append(str(self.sourceSeedVtx)+str(i))
633	>	if (self.sourceSeedG4):
634	>	## + G4 random seed
635	>	args.append(str(self.sourceSeedG4)+str(i))
636	>	if (self.sourceSeedMix):
637	>	## + Mix random seed
638	>	args.append(str(self.sourceSeedMix)+str(i))
639	>	pass
640	>	pass
641	>	self.list_of_args.append(args)
642	>	pass
643	>
644	>	# print self.list_of_args
645
646		return
647
648	+
649	+	def jobSplittingForScript(self):#CarlosDaniele
650	+	"""
651	+	Perform job splitting based on number of job
652	+	"""
653	+	common.logger.debug(5,'Splitting per job')
654	+	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
655	+
656	+	self.total_number_of_jobs = self.theNumberOfJobs
657	+
658	+	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
659	+
660	+	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created')
661	+
662	+	# argument is seed number.$i
663	+	self.list_of_args = []
664	+	for i in range(self.total_number_of_jobs):
665	+	## Since there is no input, any site is good
666	+	# self.jobDestination.append(["Any"])
667	+	self.jobDestination.append([""])
668	+	## no random seed
669	+	self.list_of_args.append([str(i)])
670	+	return
671	+
672		def split(self, jobParams):
673
674		common.jobDB.load()
#	Line 501 | Line 706 | class Cmssw(JobType):
706		"""
707
708		# if it exist, just return it
709	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
709	>	#
710	>	# Marco. Let's start to use relative path for Boss XML files
711	>	#
712	>	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
713		if os.path.exists(self.tgzNameWithPath):
714		return self.tgzNameWithPath
715
#	Line 515 | Line 723 | class Cmssw(JobType):
723		# First of all declare the user Scram area
724		swArea = self.scram.getSWArea_()
725		#print "swArea = ", swArea
726	<	swVersion = self.scram.getSWVersion()
727	<	#print "swVersion = ", swVersion
726	>	# swVersion = self.scram.getSWVersion()
727	>	# print "swVersion = ", swVersion
728		swReleaseTop = self.scram.getReleaseTop_()
729		#print "swReleaseTop = ", swReleaseTop
730
#	Line 524 | Line 732 | class Cmssw(JobType):
732		if swReleaseTop == '' or swArea == swReleaseTop:
733		return
734
735	<	filesToBeTarred = []
736	<	## First find the executable
737	<	if (self.executable != ''):
738	<	exeWithPath = self.scram.findFile_(executable)
739	<	#           print exeWithPath
740	<	if ( not exeWithPath ):
741	<	raise CrabException('User executable '+executable+' not found')
742	<
743	<	## then check if it's private or not
744	<	if exeWithPath.find(swReleaseTop) == -1:
745	<	# the exe is private, so we must ship
746	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
747	<	path = swArea+'/'
748	<	exe = string.replace(exeWithPath, path,'')
749	<	filesToBeTarred.append(exe)
750	<	pass
751	<	else:
752	<	# the exe is from release, we'll find it on WN
753	<	pass
754	<
755	<	## Now get the libraries: only those in local working area
756	<	libDir = 'lib'
757	<	lib = swArea+'/' +libDir
758	<	common.logger.debug(5,"lib "+lib+" to be tarred")
759	<	if os.path.exists(lib):
760	<	filesToBeTarred.append(libDir)
761	<
762	<	## Now check if module dir is present
763	<	moduleDir = 'module'
764	<	if os.path.isdir(swArea+'/'+moduleDir):
765	<	filesToBeTarred.append(moduleDir)
766	<
767	<	## Now check if the Data dir is present
768	<	dataDir = 'src/Data/'
769	<	if os.path.isdir(swArea+'/'+dataDir):
770	<	filesToBeTarred.append(dataDir)
771	<
772	<	## Create the tar-ball
773	<	if len(filesToBeTarred)>0:
774	<	cwd = os.getcwd()
775	<	os.chdir(swArea)
776	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
777	<	for line in filesToBeTarred:
778	<	tarcmd = tarcmd + line + ' '
779	<	cout = runCommand(tarcmd)
780	<	if not cout:
781	<	raise CrabException('Could not create tar-ball')
782	<	os.chdir(cwd)
783	<	else:
784	<	common.logger.debug(5,"No files to be to be tarred")
735	>	import tarfile
736	>	try: # create tar ball
737	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
738	>	## First find the executable
739	>	if (self.executable != ''):
740	>	exeWithPath = self.scram.findFile_(executable)
741	>	if ( not exeWithPath ):
742	>	raise CrabException('User executable '+executable+' not found')
743	>
744	>	## then check if it's private or not
745	>	if exeWithPath.find(swReleaseTop) == -1:
746	>	# the exe is private, so we must ship
747	>	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
748	>	path = swArea+'/'
749	>	# distinguish case when script is in user project area or given by full path somewhere else
750	>	if exeWithPath.find(path) >= 0 :
751	>	exe = string.replace(exeWithPath, path,'')
752	>	tar.add(path+exe,os.path.basename(executable))
753	>	else :
754	>	tar.add(exeWithPath,os.path.basename(executable))
755	>	pass
756	>	else:
757	>	# the exe is from release, we'll find it on WN
758	>	pass
759	>
760	>	## Now get the libraries: only those in local working area
761	>	libDir = 'lib'
762	>	lib = swArea+'/' +libDir
763	>	common.logger.debug(5,"lib "+lib+" to be tarred")
764	>	if os.path.exists(lib):
765	>	tar.add(lib,libDir)
766	>
767	>	## Now check if module dir is present
768	>	moduleDir = 'module'
769	>	module = swArea + '/' + moduleDir
770	>	if os.path.isdir(module):
771	>	tar.add(module,moduleDir)
772	>
773	>	## Now check if any data dir(s) is present
774	>	swAreaLen=len(swArea)
775	>	for root, dirs, files in os.walk(swArea):
776	>	if "data" in dirs:
777	>	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
778	>	tar.add(root+"/data",root[swAreaLen:]+"/data")
779	>
780	>	## Add ProdAgent dir to tar
781	>	paDir = 'ProdAgentApi'
782	>	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
783	>	if os.path.isdir(pa):
784	>	tar.add(pa,paDir)
785	>
786	>	### FEDE FOR DBS PUBLICATION
787	>	## Add PRODCOMMON dir to tar
788	>	prodcommonDir = 'ProdCommon'
789	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
790	>	if os.path.isdir(prodcommonPath):
791	>	tar.add(prodcommonPath,prodcommonDir)
792	>	#############################
793	>
794	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
795	>	tar.close()
796	>	except :
797	>	raise CrabException('Could not create tar-ball')
798	>
799	>	## check for tarball size
800	>	tarballinfo = os.stat(self.tgzNameWithPath)
801	>	if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
802	>	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
803	>
804	>	## create tar-ball with ML stuff
805	>	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
806	>	try:
807	>	tar = tarfile.open(self.MLtgzfile, "w:gz")
808	>	path=os.environ['CRABDIR'] + '/python/'
809	>	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
810	>	tar.add(path+file,file)
811	>	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
812	>	tar.close()
813	>	except :
814	>	raise CrabException('Could not create ML files tar-ball')
815
816		return
817
818	+	def additionalInputFileTgz(self):
819	+	"""
820	+	Put all additional files into a tar ball and return its name
821	+	"""
822	+	import tarfile
823	+	tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
824	+	tar = tarfile.open(tarName, "w:gz")
825	+	for file in self.additional_inbox_files:
826	+	tar.add(file,string.split(file,'/')[-1])
827	+	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
828	+	tar.close()
829	+	return tarName
830	+
831		def wsSetupEnvironment(self, nj):
832		"""
833		Returns part of a job script which prepares
#	Line 587 | Line 838 | class Cmssw(JobType):
838
839		## OLI_Daniele at this level  middleware already known
840
841	+	txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
842	+	txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
843	+	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
844	+	txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
845		txt += 'if [ $middleware == LCG ]; then \n'
846		txt += self.wsSetupCMSLCGEnvironment_()
847		txt += 'elif [ $middleware == OSG ]; then\n'
848	<	txt += '    time=`date -u +"%s"`\n'
849	<	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
595	<	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
596	<	txt += '    /bin/mkdir -p $WORKING_DIR\n'
848	>	txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
849	>	txt += '    echo "Created working directory: $WORKING_DIR"\n'
850		txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
851		txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
852	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
853	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
854	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
852	>	txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
853	>	txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
854	>	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
855		txt += '        rm -f $RUNTIME_AREA/$repo \n'
856		txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
857		txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 630 | Line 883 | class Cmssw(JobType):
883		txt += '        cd $RUNTIME_AREA\n'
884		txt += '        /bin/rm -rf $WORKING_DIR\n'
885		txt += '        if [ -d $WORKING_DIR ] ;then\n'
886	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
888	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
886	>	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
887	>	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
888	>	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
889	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
890		txt += '            rm -f $RUNTIME_AREA/$repo \n'
891		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
892		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 643 | Line 896 | class Cmssw(JobType):
896		txt += 'fi \n'
897		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
898		txt += 'cd '+self.version+'\n'
899	+	########## FEDE FOR DBS2 ######################
900	+	txt += 'SOFTWARE_DIR=`pwd`\n'
901	+	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
902	+	###############################################
903		### needed grep for bug in scramv1 ###
904	+	txt += scram+' runtime -sh\n'
905		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
906	+	txt += 'echo $PATH\n'
907
908		# Handle the arguments:
909		txt += "\n"
#	Line 666 | Line 925 | class Cmssw(JobType):
925		txt += '        cd $RUNTIME_AREA\n'
926		txt += '        /bin/rm -rf $WORKING_DIR\n'
927		txt += '        if [ -d $WORKING_DIR ] ;then\n'
928	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
930	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
928	>	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
929	>	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
930	>	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
931	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
932		txt += '            rm -f $RUNTIME_AREA/$repo \n'
933		txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
934		txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
#	Line 681 | Line 940 | class Cmssw(JobType):
940
941		# Prepare job-specific part
942		job = common.job_list[nj]
943	<	pset = os.path.basename(job.configFilename())
944	<	txt += '\n'
945	<	if (self.datasetPath): # standard job
946	<	#txt += 'InputFiles=$2\n'
688	<	txt += 'InputFiles=${args[1]}\n'
689	<	txt += 'echo "Inputfiles:<$InputFiles>"\n'
690	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
691	<	else:  # pythia like job
692	<	if (self.sourceSeed):
693	<	#                txt += 'Seed=$2\n'
694	<	txt += 'Seed=${args[1]}\n'
695	<	txt += 'echo "Seed: <$Seed>"\n'
696	<	txt += 'sed "s#\<INPUT\>#$Seed#" $RUNTIME_AREA/'+pset+' > tmp.cfg\n'
697	<	if (self.sourceSeedVtx):
698	<	#                    txt += 'VtxSeed=$3\n'
699	<	txt += 'VtxSeed=${args[2]}\n'
700	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
701	<	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp.cfg > pset.cfg\n'
702	<	else:
703	<	txt += 'mv tmp.cfg pset.cfg\n'
704	<	else:
705	<	txt += '# Copy untouched pset\n'
706	<	txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
943	>	### FEDE FOR DBS OUTPUT PUBLICATION
944	>	if (self.datasetPath):
945	>	txt += '\n'
946	>	txt += 'DatasetPath='+self.datasetPath+'\n'
947
948	+	datasetpath_split = self.datasetPath.split("/")
949	+
950	+	txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
951	+	txt += 'DataTier='+datasetpath_split[2]+'\n'
952	+	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
953	+	txt += 'ApplicationFamily=cmsRun\n'
954	+
955	+	else:
956	+	txt += 'DatasetPath=MCDataTier\n'
957	+	txt += 'PrimaryDataset=null\n'
958	+	txt += 'DataTier=null\n'
959	+	#txt += 'ProcessedDataset=null\n'
960	+	txt += 'ApplicationFamily=MCDataTier\n'
961	+	if self.pset != None: #CarlosDaniele
962	+	pset = os.path.basename(job.configFilename())
963	+	txt += '\n'
964	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
965	+	if (self.datasetPath): # standard job
966	+	#txt += 'InputFiles=$2\n'
967	+	txt += 'InputFiles=${args[1]}\n'
968	+	txt += 'MaxEvents=${args[2]}\n'
969	+	txt += 'SkipEvents=${args[3]}\n'
970	+	txt += 'echo "Inputfiles:<$InputFiles>"\n'
971	+	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
972	+	txt += 'echo "MaxEvents:<$MaxEvents>"\n'
973	+	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
974	+	txt += 'echo "SkipEvents:<$SkipEvents>"\n'
975	+	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
976	+	else:  # pythia like job
977	+	seedIndex=1
978	+	if (self.firstRun):
979	+	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
980	+	txt += 'echo "FirstRun: <$FirstRun>"\n'
981	+	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
982	+	seedIndex=seedIndex+1
983	+
984	+	if (self.sourceSeed):
985	+	txt += 'Seed=${args['+str(seedIndex)+']}\n'
986	+	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
987	+	seedIndex=seedIndex+1
988	+	## the following seeds are not always present
989	+	if (self.sourceSeedVtx):
990	+	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
991	+	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
992	+	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
993	+	seedIndex += 1
994	+	if (self.sourceSeedG4):
995	+	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
996	+	txt += 'echo "G4Seed: <$G4Seed>"\n'
997	+	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
998	+	seedIndex += 1
999	+	if (self.sourceSeedMix):
1000	+	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1001	+	txt += 'echo "MixSeed: <$mixSeed>"\n'
1002	+	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1003	+	seedIndex += 1
1004	+	pass
1005	+	pass
1006	+	txt += 'mv -f '+pset+' pset.cfg\n'
1007
1008		if len(self.additional_inbox_files) > 0:
1009	<	for file in self.additional_inbox_files:
1010	<	relFile = file.split("/")[-1]
1011	<	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
713	<	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
714	<	txt += '   chmod +x '+relFile+'\n'
715	<	txt += 'fi\n'
1009	>	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1010	>	txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1011	>	txt += 'fi\n'
1012		pass
1013
1014	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1015	<
1016	<	txt += '\n'
1017	<	txt += 'echo "***** cat pset.cfg *********"\n'
1018	<	txt += 'cat pset.cfg\n'
1019	<	txt += 'echo "****** end pset.cfg ********"\n'
1020	<	txt += '\n'
1021	<	# txt += 'echo "***** cat pset1.cfg *********"\n'
1022	<	# txt += 'cat pset1.cfg\n'
1023	<	# txt += 'echo "****** end pset1.cfg ********"\n'
1014	>	if self.pset != None: #CarlosDaniele
1015	>	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1016	>
1017	>	txt += '\n'
1018	>	txt += 'echo "***** cat pset.cfg *********"\n'
1019	>	txt += 'cat pset.cfg\n'
1020	>	txt += 'echo "****** end pset.cfg ********"\n'
1021	>	txt += '\n'
1022	>	### FEDE FOR DBS OUTPUT PUBLICATION
1023	>	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
1024	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
1025	>	##############
1026	>	txt += '\n'
1027	>	# txt += 'echo "***** cat pset1.cfg *********"\n'
1028	>	# txt += 'cat pset1.cfg\n'
1029	>	# txt += 'echo "****** end pset1.cfg ********"\n'
1030		return txt
1031
1032	<	def wsBuildExe(self, nj):
1032	>	def wsBuildExe(self, nj=0):
1033		"""
1034		Put in the script the commands to build an executable
1035		or a library.
#	Line 762 | Line 1064 | class Cmssw(JobType):
1064		txt += 'else \n'
1065		txt += '   echo "Successful untar" \n'
1066		txt += 'fi \n'
1067	+	txt += '\n'
1068	+	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
1069	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1070	+	#### FEDE FOR DBS OUTPUT PUBLICATION
1071	+	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1072	+	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1073	+	#txt += '   export PYTHONPATH=ProdAgentApi\n'
1074	+	txt += 'else\n'
1075	+	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1076	+	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1077	+	#txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1078	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1079	+	###################
1080	+	txt += 'fi\n'
1081	+	txt += '\n'
1082	+
1083		pass
1084
1085		return txt
#	Line 773 | Line 1091 | class Cmssw(JobType):
1091		"""
1092
1093		def executableName(self):
1094	<	return self.executable
1094	>	if self.scriptExe: #CarlosDaniele
1095	>	return "sh "
1096	>	else:
1097	>	return self.executable
1098
1099		def executableArgs(self):
1100	<	return " -p pset.cfg"
1100	>	if self.scriptExe:#CarlosDaniele
1101	>	return   self.scriptExe + " $NJob"
1102	>	else:
1103	>	return " -p pset.cfg"
1104
1105		def inputSandbox(self, nj):
1106		"""
1107		Returns a list of filenames to be put in JDL input sandbox.
1108		"""
1109		inp_box = []
1110	<	# dict added to delete duplicate from input sandbox file list
1111	<	seen = {}
1110	>	# # dict added to delete duplicate from input sandbox file list
1111	>	# seen = {}
1112		## code
1113		if os.path.isfile(self.tgzNameWithPath):
1114		inp_box.append(self.tgzNameWithPath)
1115	+	if os.path.isfile(self.MLtgzfile):
1116	+	inp_box.append(self.MLtgzfile)
1117		## config
1118	<	inp_box.append(common.job_list[nj].configFilename())
1118	>	if not self.pset is None:
1119	>	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1120		## additional input files
1121	<	#for file in self.additional_inbox_files:
1122	<	#    inp_box.append(common.work_space.cwdDir()+file)
1121	>	tgz = self.additionalInputFileTgz()
1122	>	inp_box.append(tgz)
1123		return inp_box
1124
1125		def outputSandbox(self, nj):
#	Line 801 | Line 1128 | class Cmssw(JobType):
1128		"""
1129		out_box = []
1130
804	–	stdout=common.job_list[nj].stdout()
805	–	stderr=common.job_list[nj].stderr()
806	–
1131		## User Declared output files
1132	<	for out in self.output_file:
1132	>	for out in (self.output_file+self.output_file_sandbox):
1133		n_out = nj + 1
1134		out_box.append(self.numberFile_(out,str(n_out)))
1135		return out_box
812	–	return []
1136
1137		def prepareSteeringCards(self):
1138		"""
#	Line 825 | Line 1148 | class Cmssw(JobType):
1148		txt = '\n'
1149		txt += '# directory content\n'
1150		txt += 'ls \n'
1151	<	file_list = ''
1152	<	for fileWithSuffix in self.output_file:
1151	>
1152	>	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1153		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
831	–	file_list=file_list+output_file_num+' '
1154		txt += '\n'
1155		txt += '# check output file\n'
1156	<	txt += 'ls '+fileWithSuffix+'\n'
1157	<	txt += 'ls_result=$?\n'
1158	<	#txt += 'exe_result=$?\n'
1159	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1160	<	txt += '   echo "ERROR: Problem with output file"\n'
1161	<	#txt += '   echo "JOB_EXIT_STATUS = $exe_result"\n'
1162	<	#txt += '   echo "JobExitCode=60302" | tee -a $RUNTIME_AREA/$repo\n'
841	<	#txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
842	<	### OLI_DANIELE
1156	>	# txt += 'ls '+fileWithSuffix+'\n'
1157	>	# txt += 'ls_result=$?\n'
1158	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1159	>	txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1160	>	txt += 'else\n'
1161	>	txt += '   exit_status=60302\n'
1162	>	txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1163		if common.scheduler.boss_scheduler_name == 'condor_g':
1164		txt += '    if [ $middleware == OSG ]; then \n'
1165		txt += '        echo "prepare dummy output file"\n'
1166		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1167		txt += '    fi \n'
848	–	txt += 'else\n'
849	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1168		txt += 'fi\n'
1169	+	file_list = []
1170	+	for fileWithSuffix in (self.output_file):
1171	+	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1172	+	txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1173
1174		txt += 'cd $RUNTIME_AREA\n'
1175	<	file_list=file_list[:-1]
854	<	txt += 'file_list="'+file_list+'"\n'
855	<	txt += 'cd $RUNTIME_AREA\n'
1175	>	#### FEDE this is the cleanEnv function
1176		### OLI_DANIELE
1177	<	txt += 'if [ $middleware == OSG ]; then\n'
1178	<	txt += '    cd $RUNTIME_AREA\n'
1179	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1181	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1184	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1186	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189	<	txt += '    fi\n'
1190	<	txt += 'fi\n'
1191	<	txt += '\n'
1177	>	#txt += 'if [ $middleware == OSG ]; then\n'
1178	>	#txt += '    cd $RUNTIME_AREA\n'
1179	>	#txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1180	>	#txt += '    /bin/rm -rf $WORKING_DIR\n'
1181	>	#txt += '    if [ -d $WORKING_DIR ] ;then\n'
1182	>	#txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1183	>	#txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1184	>	#txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1185	>	#txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1186	>	#txt += '        rm -f $RUNTIME_AREA/$repo \n'
1187	>	#txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1188	>	#txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1189	>	#txt += '    fi\n'
1190	>	#txt += 'fi\n'
1191	>	#txt += '\n'
1192	>
1193	>
1194		return txt
1195
1196		def numberFile_(self, file, txt):
#	Line 879 | Line 1201 | class Cmssw(JobType):
1201		# take away last extension
1202		name = p[0]
1203		for x in p[1:-1]:
1204	<	name=name+"."+x
1204	>	name=name+"."+x
1205		# add "_txt"
1206		if len(p)>1:
1207	<	ext = p[len(p)-1]
1208	<	#result = name + '_' + str(txt) + "." + ext
887	<	result = name + '_' + txt + "." + ext
1207	>	ext = p[len(p)-1]
1208	>	result = name + '_' + txt + "." + ext
1209		else:
1210	<	#result = name + '_' + str(txt)
890	<	result = name + '_' + txt
1210	>	result = name + '_' + txt
1211
1212		return result
1213
1214	<	def getRequirements(self, nj):
1214	>	def getRequirements(self, nj=[]):
1215		"""
1216		return job requirements to add to jdl files
1217		"""
1218		req = ''
1219	<	if common.analisys_common_info['sw_version']:
1219	>	if self.version:
1220		req='Member("VO-cms-' + \
1221	<	common.analisys_common_info['sw_version'] + \
1221	>	self.version + \
1222	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1223	>	## SL add requirement for OS version only if SL4
1224	>	reSL4 = re.compile( r'slc4' )
1225	>	if self.executable_arch and reSL4.search(self.executable_arch):
1226	>	req='Member("VO-cms-' + \
1227	>	self.executable_arch + \
1228		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1229
1230		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1231
906	–	## here we should get the requirement for job nj
907	–	sites = common.jobDB.destination(nj)
908	–
909	–	# check for "Any" site, in case no requirement for site
910	–	if len(sites)>0 and sites[0]!="Any":
911	–	req = req + ' && anyMatch(other.storage.CloseSEs, ('
912	–	for site in sites:
913	–	#req = req + 'other.GlueCEInfoHostName == "' + site + '" || '
914	–	req = req + 'target.GlueSEUniqueID=="' + site + '" || '
915	–	pass
916	–	# remove last ||
917	–	req = req[0:-4]
918	–	req = req + '))'
919	–
1232		return req
1233
1234		def configFilename(self):
#	Line 933 | Line 1245 | class Cmssw(JobType):
1245		txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1246		txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1247		txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1248	+	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1249		txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1250	<	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
1251	<	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
1252	<	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
1250	>	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1251	>	txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1252	>	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1253	>	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1254		txt += '   else\n'
1255	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
1255	>	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1256		txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1257		txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1258		txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
#	Line 951 | Line 1265 | class Cmssw(JobType):
1265		txt += '       cd $RUNTIME_AREA\n'
1266		txt += '       /bin/rm -rf $WORKING_DIR\n'
1267		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1268	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
1269	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1270	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1272	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1273	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1268	>	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1269	>	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1270	>	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1271	>	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1272	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1273	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1274	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1275		txt += '       fi\n'
1276		txt += '\n'
1277		txt += '       exit 1\n'
#	Line 1012 | Line 1326 | class Cmssw(JobType):
1326		txt += '       fi\n'
1327		txt += '   fi\n'
1328		txt += '   \n'
1015	–	txt += '   string=`cat /etc/redhat-release`\n'
1016	–	txt += '   echo $string\n'
1017	–	txt += '   if [[ $string = *alhalla* ]]; then\n'
1018	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1019	–	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1020	–	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1021	–	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1022	–	txt += '   else\n'
1023	–	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1024	–	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1025	–	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1026	–	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1027	–	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1028	–	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1029	–	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1030	–	txt += '       exit 1\n'
1031	–	txt += '   fi\n'
1329		txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1330		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1331		return txt
1332
1333	+	### FEDE FOR DBS OUTPUT PUBLICATION
1334	+	def modifyReport(self, nj):
1335	+	"""
1336	+	insert the part of the script that modifies the FrameworkJob Report
1337	+	"""
1338	+
1339	+	txt = ''
1340	+	txt += 'echo "Modify Job Report" \n'
1341	+	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1342	+	################ FEDE FOR DBS2 #############################################
1343	+	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1344	+	#############################################################################
1345	+	try:
1346	+	publish_data = int(self.cfg_params['USER.publish_data'])
1347	+	except KeyError:
1348	+	publish_data = 0
1349	+
1350	+	txt += 'if [ -z "$SE" ]; then\n'
1351	+	txt += '    SE="" \n'
1352	+	txt += 'fi \n'
1353	+	txt += 'if [ -z "$SE_PATH" ]; then\n'
1354	+	txt += '    SE_PATH="" \n'
1355	+	txt += 'fi \n'
1356	+	txt += 'echo "SE = $SE"\n'
1357	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1358	+
1359	+	if (publish_data == 1):
1360	+	#processedDataset = self.cfg_params['USER.processed_datasetname']
1361	+	processedDataset = self.cfg_params['USER.publish_data_name']
1362	+	txt += 'ProcessedDataset='+processedDataset+'\n'
1363	+	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1364	+	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1365	+	#### FEDE: added slash in LFN ##############
1366	+	txt += '    FOR_LFN=/copy_problems/ \n'
1367	+	txt += 'else \n'
1368	+	txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1369	+	#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1370	+	txt += '    FOR_LFN=/store$tmp \n'
1371	+	txt += 'fi \n'
1372	+	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1373	+	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1374	+	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1375	+	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1376	+	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1377	+	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1378	+	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1379	+
1380	+	txt += 'modifyReport_result=$?\n'
1381	+	txt += 'echo modifyReport_result = $modifyReport_result\n'
1382	+	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1383	+	txt += '    exit_status=1\n'
1384	+	txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1385	+	txt += 'else\n'
1386	+	txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1387	+	txt += 'fi\n'
1388	+	else:
1389	+	txt += 'ProcessedDataset=no_data_to_publish \n'
1390	+	#### FEDE: added slash in LFN ##############
1391	+	txt += 'FOR_LFN=/local/ \n'
1392	+	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1393	+	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1394	+	return txt
1395	+
1396	+	def cleanEnv(self):
1397	+	### OLI_DANIELE
1398	+	txt = ''
1399	+	txt += 'if [ $middleware == OSG ]; then\n'
1400	+	txt += '    cd $RUNTIME_AREA\n'
1401	+	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1402	+	txt += '    /bin/rm -rf $WORKING_DIR\n'
1403	+	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1404	+	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1405	+	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1406	+	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1407	+	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1408	+	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1409	+	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1410	+	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1411	+	txt += '    fi\n'
1412	+	txt += 'fi\n'
1413	+	txt += '\n'
1414	+	return txt
1415	+
1416		def setParam_(self, param, value):
1417		self._params[param] = value
1418
#	Line 1045 | Line 1425 | class Cmssw(JobType):
1425		def getTaskid(self):
1426		return self._taskId
1427
1048	–	#######################################################################
1428		def uniquelist(self, old):
1429		"""
1430		remove duplicates from a list

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines