ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.5 by gutsche, Fri Jun 2 22:14:20 2006 UTC vs.
Revision 1.64 by slacapra, Thu Jan 18 18:29:51 2007 UTC

#	Line 5 | Line 5 | from crab_util import *
5		import common
6		import PsetManipulator
7
8	<	import DBSInfo_EDM
9	<	import DataDiscovery_EDM
10	<	import DataLocation_EDM
8	>	import DataDiscovery
9	>	import DataLocation
10		import Scram
11
12	<	import os, string, re
12	>	import os, string, re, shutil
13
14		class Cmssw(JobType):
15	<	def __init__(self, cfg_params):
15	>	def __init__(self, cfg_params, ncjobs):
16		JobType.__init__(self, 'CMSSW')
17		common.logger.debug(3,'CMSSW::__init__')
18
20	–	self.analisys_common_info = {}
19		# Marco.
20		self._params = {}
21		self.cfg_params = cfg_params
22
23	+	# number of jobs requested to be created, limit obj splitting
24	+	self.ncjobs = ncjobs
25	+
26		log = common.logger
27
28		self.scram = Scram.Scram(cfg_params)
28	–	scramArea = ''
29		self.additional_inbox_files = []
30		self.scriptExe = ''
31		self.executable = ''
32		self.tgz_name = 'default.tgz'
33	+	self.scriptName = 'CMSSW.sh'
34	+	self.pset = ''      #scrip use case Da
35	+	self.datasetPath = '' #scrip use case Da
36
37	+	# set FJR file name
38	+	self.fjrFileName = 'crab_fjr.xml'
39
40		self.version = self.scram.getSWVersion()
41	+	common.taskDB.setDict('codeVersion',self.version)
42		self.setParam_('application', self.version)
37	–	common.analisys_common_info['sw_version'] = self.version
38	–	### FEDE
39	–	common.analisys_common_info['copy_input_data'] = 0
40	–	common.analisys_common_info['events_management'] = 1
43
44		### collect Data cards
45		try:
46	<	#   self.owner = cfg_params['CMSSW.owner']
47	<	#   log.debug(6, "CMSSW::CMSSW(): owner = "+self.owner)
48	<	#   self.dataset = cfg_params['CMSSW.dataset']
49	<	self.datasetPath = cfg_params['CMSSW.datasetpath']
50	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+self.datasetPath)
46	>	tmp =  cfg_params['CMSSW.datasetpath']
47	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
48	>	if string.lower(tmp)=='none':
49	>	self.datasetPath = None
50	>	self.selectNoInput = 1
51	>	else:
52	>	self.datasetPath = tmp
53	>	self.selectNoInput = 0
54		except KeyError:
50	–	#    msg = "Error: owner and/or dataset not defined "
55		msg = "Error: datasetpath not defined "
56		raise CrabException(msg)
57
58		# ML monitoring
59		# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
60	<	datasetpath_split = self.datasetPath.split("/")
61	<	self.setParam_('dataset', datasetpath_split[1])
62	<	self.setParam_('owner', datasetpath_split[-1])
63	<
64	<
60	>	if not self.datasetPath:
61	>	self.setParam_('dataset', 'None')
62	>	self.setParam_('owner', 'None')
63	>	else:
64	>	datasetpath_split = self.datasetPath.split("/")
65	>	self.setParam_('dataset', datasetpath_split[1])
66	>	self.setParam_('owner', datasetpath_split[-1])
67
68	+	self.setTaskid_()
69	+	self.setParam_('taskId', self.cfg_params['taskId'])
70
71		self.dataTiers = []
64	–	#       try:
65	–	#           tmpDataTiers = string.split(cfg_params['CMSSW.data_tier'],',')
66	–	#           for tmp in tmpDataTiers:
67	–	#               tmp=string.strip(tmp)
68	–	#               self.dataTiers.append(tmp)
69	–	#               pass
70	–	#           pass
71	–	#       except KeyError:
72	–	#           pass
73	–	#       log.debug(6, "Cmssw::Cmssw(): dataTiers = "+str(self.dataTiers))
72
73		## now the application
74		try:
#	Line 89 | Line 87 | class Cmssw(JobType):
87		try:
88		self.pset = cfg_params['CMSSW.pset']
89		log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
90	<	if (not os.path.exists(self.pset)):
91	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
90	>	if self.pset.lower() != 'none' :
91	>	if (not os.path.exists(self.pset)):
92	>	raise CrabException("User defined PSet file "+self.pset+" does not exist")
93	>	else:
94	>	self.pset = None
95		except KeyError:
96		raise CrabException("PSet file missing. Cannot run cmsRun ")
97
98		# output files
99	+	## stuff which must be returned always via sandbox
100	+	self.output_file_sandbox = []
101	+
102	+	# add fjr report by default via sandbox
103	+	self.output_file_sandbox.append(self.fjrFileName)
104	+
105	+	# other output files to be returned via sandbox or copied to SE
106		try:
107		self.output_file = []
100	–
108		tmp = cfg_params['CMSSW.output_file']
109		if tmp != '':
110		tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
#	Line 107 | Line 114 | class Cmssw(JobType):
114		self.output_file.append(tmp)
115		pass
116		else:
117	<	log.message("No output file defined: only stdout/err will be available")
117	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
118		pass
119		pass
120		except KeyError:
121	<	log.message("No output file defined: only stdout/err will be available")
121	>	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
122		pass
123
124		# script_exe file as additional file in inputSandbox
125		try:
126	<	self.scriptExe = cfg_params['USER.script_exe']
127	<	self.additional_inbox_files.append(self.scriptExe)
126	>	self.scriptExe = cfg_params['USER.script_exe']
127	>	if self.scriptExe != '':
128	>	if not os.path.isfile(self.scriptExe):
129	>	msg ="ERROR. file "+self.scriptExe+" not found"
130	>	raise CrabException(msg)
131	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
132		except KeyError:
133	<	pass
134	<	if self.scriptExe != '':
135	<	if os.path.isfile(self.scriptExe):
136	<	pass
137	<	else:
138	<	log.message("WARNING. file "+self.scriptExe+" not found")
128	<	sys.exit()
129	<
133	>	self.scriptExe = ''
134	>	#CarlosDaniele
135	>	if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
136	>	msg ="WARNING. script_exe  not defined"
137	>	raise CrabException(msg)
138	>
139		## additional input files
140		try:
141	<	tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
142	<	for tmp in tmpAddFiles:
143	<	if not os.path.exists(tmp):
144	<	raise CrabException("Additional input file not found: "+tmp)
145	<	tmp=string.strip(tmp)
146	<	self.additional_inbox_files.append(tmp)
141	>	tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
142	>	common.logger.debug(5,"Additional input files: "+str(tmpAddFiles))
143	>	for tmpFile in tmpAddFiles:
144	>	tmpFile = string.strip(tmpFile)
145	>	if not os.path.exists(tmpFile):
146	>	raise CrabException("Additional input file not found: "+tmpFile)
147	>	pass
148	>	storedFile = common.work_space.shareDir()+ tmpFile
149	>	shutil.copyfile(tmpFile, storedFile)
150	>	self.additional_inbox_files.append(string.strip(storedFile))
151		pass
152	+	common.logger.debug(5,"Inbox files so far : "+str(self.additional_inbox_files))
153		pass
154		except KeyError:
155		pass
156
157	+	# files per job
158		try:
159	<	self.filesPerJob = int(cfg_params['CMSSW.files_per_jobs']) #Daniele
159	>	if (cfg_params['CMSSW.files_per_jobs']):
160	>	raise CrabException("files_per_jobs no longer supported.  Quitting.")
161		except KeyError:
162	<	self.filesPerJob = 1
162	>	pass
163
164	<	## Max event   will be total_number_of_events ???  Daniele
164	>	## Events per job
165		try:
166	<	self.maxEv = cfg_params['CMSSW.event_per_job']
166	>	self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
167	>	self.selectEventsPerJob = 1
168		except KeyError:
169	<	self.maxEv = "-1"
170	<	##
171	<	try:
172	<	self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
156	<	except KeyError:
157	<	msg = 'Must define total_number_of_events'
158	<	raise CrabException(msg)
159	<
160	<	CEBlackList = []
169	>	self.eventsPerJob = -1
170	>	self.selectEventsPerJob = 0
171	>
172	>	## number of jobs
173		try:
174	<	tmpBad = string.split(cfg_params['EDG.ce_black_list'],',')
175	<	for tmp in tmpBad:
164	<	tmp=string.strip(tmp)
165	<	CEBlackList.append(tmp)
174	>	self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
175	>	self.selectNumberOfJobs = 1
176		except KeyError:
177	<	pass
177	>	self.theNumberOfJobs = 0
178	>	self.selectNumberOfJobs = 0
179
180	<	self.reCEBlackList=[]
181	<	for bad in CEBlackList:
182	<	self.reCEBlackList.append(re.compile( bad ))
172	<
173	<	common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
174	<
175	<	CEWhiteList = []
176	<	try:
177	<	tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
178	<	for tmp in tmpGood:
179	<	tmp=string.strip(tmp)
180	<	CEWhiteList.append(tmp)
180	>	try:
181	>	self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
182	>	self.selectTotalNumberEvents = 1
183		except KeyError:
184	<	pass
184	>	self.total_number_of_events = 0
185	>	self.selectTotalNumberEvents = 0
186
187	<	#print 'CEWhiteList: ',CEWhiteList
188	<	self.reCEWhiteList=[]
189	<	for Good in CEWhiteList:
190	<	self.reCEWhiteList.append(re.compile( Good ))
187	>	if self.pset != None: #CarlosDaniele
188	>	if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
189	>	msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
190	>	raise CrabException(msg)
191	>	else:
192	>	if (self.selectNumberOfJobs == 0):
193	>	msg = 'Must specify  number_of_jobs.'
194	>	raise CrabException(msg)
195
196	<	common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
196	>	## source seed for pythia
197	>	try:
198	>	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
199	>	except KeyError:
200	>	self.sourceSeed = None
201	>	common.logger.debug(5,"No seed given")
202
203	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
203	>	try:
204	>	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
205	>	except KeyError:
206	>	self.sourceSeedVtx = None
207	>	common.logger.debug(5,"No vertex seed given")
208	>	try:
209	>	self.firstRun = int(cfg_params['CMSSW.first_run'])
210	>	except KeyError:
211	>	self.firstRun = None
212	>	common.logger.debug(5,"No first run given")
213	>	if self.pset != None: #CarlosDaniele
214	>	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
215
216		#DBSDLS-start
217		## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
218		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
219		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
220	+	self.jobDestination=[]  # Site destination(s) for each job (list of lists)
221		## Perform the data location and discovery (based on DBS/DLS)
222	<	self.DataDiscoveryAndLocation(cfg_params)
222	>	## SL: Don't if NONE is specified as input (pythia use case)
223	>	blockSites = {}
224	>	if self.datasetPath:
225	>	blockSites = self.DataDiscoveryAndLocation(cfg_params)
226		#DBSDLS-end
227
228		self.tgzNameWithPath = self.getTarBall(self.executable)
229	<
230	<	self.jobSplitting()  #Daniele job Splitting
231	<	self.PsetEdit.maxEvent(self.maxEv) #Daniele
232	<	self.PsetEdit.inputModule("INPUT") #Daniele
233	<	self.PsetEdit.psetWriter(self.configFilename())
234	<
235	<
229	>
230	>	## Select Splitting
231	>	if self.selectNoInput:
232	>	if self.pset == None: #CarlosDaniele
233	>	self.jobSplittingForScript()
234	>	else:
235	>	self.jobSplittingNoInput()
236	>	else:
237	>	self.jobSplittingByBlocks(blockSites)
238	>
239	>	# modify Pset
240	>	if self.pset != None: #CarlosDaniele
241	>	try:
242	>	if (self.datasetPath): # standard job
243	>	# allow to processa a fraction of events in a file
244	>	self.PsetEdit.inputModule("INPUT")
245	>	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
246	>	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
247	>	else:  # pythia like job
248	>	self.PsetEdit.maxEvent(self.eventsPerJob)
249	>	if (self.firstRun):
250	>	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
251	>	if (self.sourceSeed) :
252	>	self.PsetEdit.pythiaSeed("INPUT")
253	>	if (self.sourceSeedVtx) :
254	>	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
255	>	# add FrameworkJobReport to parameter-set
256	>	self.PsetEdit.addCrabFJR(self.fjrFileName)
257	>	self.PsetEdit.psetWriter(self.configFilename())
258	>	except:
259	>	msg='Error while manipuliating ParameterSet: exiting...'
260	>	raise CrabException(msg)
261
262		def DataDiscoveryAndLocation(self, cfg_params):
263
264		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
265
214	–	#datasetPath = "/"+self.owner+"/"+self.dataTiers[0]+"/"+self.dataset
215	–
266		datasetPath=self.datasetPath
267
268		## TODO
#	Line 220 | Line 270 | class Cmssw(JobType):
270		dataTiers = dataTiersList.split(',')
271
272		## Contact the DBS
273	+	common.logger.message("Contacting DBS...")
274		try:
275	<	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
275	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, dataTiers, cfg_params)
276		self.pubdata.fetchDBSInfo()
277
278	<	except DataDiscovery_EDM.NotExistingDatasetError, ex :
278	>	except DataDiscovery.NotExistingDatasetError, ex :
279		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
280		raise CrabException(msg)
281
282	<	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
282	>	except DataDiscovery.NoDataTierinProvenanceError, ex :
283		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
284		raise CrabException(msg)
285	<	except DataDiscovery_EDM.DataDiscoveryError, ex:
285	>	except DataDiscovery.DataDiscoveryError, ex:
286		msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
287		raise CrabException(msg)
288
#	Line 239 | Line 290 | class Cmssw(JobType):
290		## self.DBSPaths=self.pubdata.getDBSPaths()
291		common.logger.message("Required data are :"+self.datasetPath)
292
293	<	filesbyblock=self.pubdata.getFiles()
294	<	self.AllInputFiles=filesbyblock.values()
295	<	self.files = self.AllInputFiles
296	<
297	<	## TEMP
298	<	#    self.filesTmp = filesbyblock.values()
248	<	#    self.files = []
249	<	#    locPath='rfio:cmsbose2.bo.infn.it:/flatfiles/SE00/cms/fanfani/ProdTest/'
250	<	#    locPath=''
251	<	#    tmp = []
252	<	#    for file in self.filesTmp[0]:
253	<	#        tmp.append(locPath+file)
254	<	#    self.files.append(tmp)
255	<	## END TEMP
293	>	self.filesbyblock=self.pubdata.getFiles()
294	>	self.eventsbyblock=self.pubdata.getEventsPerBlock()
295	>	self.eventsbyfile=self.pubdata.getEventsPerFile()
296	>	# print str(self.filesbyblock)
297	>	# print 'self.eventsbyfile',len(self.eventsbyfile)
298	>	# print str(self.eventsbyfile)
299
300		## get max number of events
258	–	#common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
301		self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
302	<	common.logger.message("\nThe number of available events is %s"%self.maxEvents)
302	>	common.logger.message("The number of available events is %s\n"%self.maxEvents)
303
304	+	common.logger.message("Contacting DLS...")
305		## Contact the DLS and build a list of sites hosting the fileblocks
306		try:
307	<	dataloc=DataLocation_EDM.DataLocation_EDM(filesbyblock.keys(),cfg_params)
308	<	dataloc.fetchDLSInfo()
309	<
267	<	except DataLocation_EDM.DataLocationError , ex:
307	>	dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
308	>	dataloc.fetchDLSInfo()
309	>	except DataLocation.DataLocationError , ex:
310		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
311		raise CrabException(msg)
312
271	–	allsites=dataloc.getSites()
272	–	common.logger.debug(5,"sites are %s"%allsites)
273	–	sites=self.checkBlackList(allsites)
274	–	common.logger.debug(5,"sites are (after black list) %s"%sites)
275	–	sites=self.checkWhiteList(sites)
276	–	common.logger.debug(5,"sites are (after white list) %s"%sites)
313
314	<	if len(sites)==0:
315	<	msg = 'No sites hosting all the needed data! Exiting... '
316	<	raise CrabException(msg)
314	>	sites = dataloc.getSites()
315	>	allSites = []
316	>	listSites = sites.values()
317	>	for listSite in listSites:
318	>	for oneSite in listSite:
319	>	allSites.append(oneSite)
320	>	allSites = self.uniquelist(allSites)
321
322	<	common.logger.message("List of Sites hosting the data : "+str(sites))
323	<	common.logger.debug(6, "List of Sites: "+str(sites))
324	<	common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
285	<	self.setParam_('TargetCE', ','.join(sites))
286	<	return
322	>	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
323	>	common.logger.debug(6, "List of Sites: "+str(allSites))
324	>	return sites
325
326	<	def jobSplitting(self):
326	>	def jobSplittingByBlocks(self, blockSites):
327		"""
328	<	first implemntation for job splitting
329	<	"""
330	<	#  print 'eventi totali '+str(self.maxEvents)
331	<	#  print 'eventi totali richiesti dallo user '+str(self.total_number_of_events)
332	<	#print 'files per job '+str(self.filesPerJob)
333	<	common.logger.message('Required '+str(self.filesPerJob)+' files per job ')
334	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
328	>	Perform job splitting. Jobs run over an integer number of files
329	>	and no more than one block.
330	>	ARGUMENT: blockSites: dictionary with blocks as keys and list of host sites as values
331	>	REQUIRES: self.selectTotalNumberEvents, self.selectEventsPerJob, self.selectNumberofJobs,
332	>	self.total_number_of_events, self.eventsPerJob, self.theNumberOfJobs,
333	>	self.maxEvents, self.filesbyblock
334	>	SETS: self.jobDestination - Site destination(s) for each job (a list of lists)
335	>	self.total_number_of_jobs - Total # of jobs
336	>	self.list_of_args - File(s) job will run on (a list of lists)
337	>	"""
338	>
339	>	# ---- Handle the possible job splitting configurations ---- #
340	>	if (self.selectTotalNumberEvents):
341	>	totalEventsRequested = self.total_number_of_events
342	>	if (self.selectEventsPerJob):
343	>	eventsPerJobRequested = self.eventsPerJob
344	>	if (self.selectNumberOfJobs):
345	>	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
346	>
347	>	# If user requested all the events in the dataset
348	>	if (totalEventsRequested == -1):
349	>	eventsRemaining=self.maxEvents
350	>	# If user requested more events than are in the dataset
351	>	elif (totalEventsRequested > self.maxEvents):
352	>	eventsRemaining = self.maxEvents
353	>	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
354	>	# If user requested less events than are in the dataset
355	>	else:
356	>	eventsRemaining = totalEventsRequested
357	>
358	>	# If user requested more events per job than are in the dataset
359	>	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
360	>	eventsPerJobRequested = self.maxEvents
361	>
362	>	# For user info at end
363	>	totalEventCount = 0
364	>
365	>	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
366	>	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
367	>
368	>	if (self.selectNumberOfJobs):
369	>	common.logger.message("May not create the exact number_of_jobs requested.")
370	>
371	>	if ( self.ncjobs == 'all' ) :
372	>	totalNumberOfJobs = 999999999
373	>	else :
374	>	totalNumberOfJobs = self.ncjobs
375	>
376	>
377	>	blocks = blockSites.keys()
378	>	blockCount = 0
379	>	# Backup variable in case self.maxEvents counted events in a non-included block
380	>	numBlocksInDataset = len(blocks)
381	>
382	>	jobCount = 0
383	>	list_of_lists = []
384
385	<	## TODO: SL need to have (from DBS) a detailed list of how many events per each file
386	<	n_tot_files = (len(self.files[0]))
387	<	## SL: this is wrong if the files have different number of events
388	<	evPerFile = int(self.maxEvents)/n_tot_files
389	<
390	<	common.logger.debug(5,'Events per File '+str(evPerFile))
391	<
392	<	## if asked to process all events, do it
393	<	if self.total_number_of_events == -1:
394	<	self.total_number_of_events=self.maxEvents
395	<	self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
396	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for all available events '+str(self.total_number_of_events)+' events')
385	>	# ---- Iterate over the blocks in the dataset until ---- #
386	>	# ---- we've met the requested total # of events    ---- #
387	>	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
388	>	block = blocks[blockCount]
389	>	blockCount += 1
390	>
391	>
392	>	numEventsInBlock = self.eventsbyblock[block]
393	>	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
394	>
395	>	files = self.filesbyblock[block]
396	>	numFilesInBlock = len(files)
397	>	if (numFilesInBlock <= 0):
398	>	continue
399	>	fileCount = 0
400	>
401	>	# ---- New block => New job ---- #
402	>	parString = "\\{"
403	>	# counter for number of events in files currently worked on
404	>	filesEventCount = 0
405	>	# flag if next while loop should touch new file
406	>	newFile = 1
407	>	# job event counter
408	>	jobSkipEventCount = 0
409	>
410	>	# ---- Iterate over the files in the block until we've met the requested ---- #
411	>	# ---- total # of events or we've gone over all the files in this block  ---- #
412	>	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
413	>	file = files[fileCount]
414	>	if newFile :
415	>	try:
416	>	numEventsInFile = self.eventsbyfile[file]
417	>	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
418	>	# increase filesEventCount
419	>	filesEventCount += numEventsInFile
420	>	# Add file to current job
421	>	parString += '\\\"' + file + '\\\"\,'
422	>	newFile = 0
423	>	except KeyError:
424	>	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
425	>
426	>
427	>	# if less events in file remain than eventsPerJobRequested
428	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
429	>	# if last file in block
430	>	if ( fileCount == numFilesInBlock-1 ) :
431	>	# end job using last file, use remaining events in block
432	>	# close job and touch new file
433	>	fullString = parString[:-2]
434	>	fullString += '\\}'
435	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
436	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
437	>	self.jobDestination.append(blockSites[block])
438	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
439	>	# reset counter
440	>	jobCount = jobCount + 1
441	>	totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
442	>	eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
443	>	jobSkipEventCount = 0
444	>	# reset file
445	>	parString = "\\{"
446	>	filesEventCount = 0
447	>	newFile = 1
448	>	fileCount += 1
449	>	else :
450	>	# go to next file
451	>	newFile = 1
452	>	fileCount += 1
453	>	# if events in file equal to eventsPerJobRequested
454	>	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
455	>	# close job and touch new file
456	>	fullString = parString[:-2]
457	>	fullString += '\\}'
458	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
459	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
460	>	self.jobDestination.append(blockSites[block])
461	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
462	>	# reset counter
463	>	jobCount = jobCount + 1
464	>	totalEventCount = totalEventCount + eventsPerJobRequested
465	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
466	>	jobSkipEventCount = 0
467	>	# reset file
468	>	parString = "\\{"
469	>	filesEventCount = 0
470	>	newFile = 1
471	>	fileCount += 1
472	>
473	>	# if more events in file remain than eventsPerJobRequested
474	>	else :
475	>	# close job but don't touch new file
476	>	fullString = parString[:-2]
477	>	fullString += '\\}'
478	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
479	>	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
480	>	self.jobDestination.append(blockSites[block])
481	>	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
482	>	# increase counter
483	>	jobCount = jobCount + 1
484	>	totalEventCount = totalEventCount + eventsPerJobRequested
485	>	eventsRemaining = eventsRemaining - eventsPerJobRequested
486	>	# calculate skip events for last file
487	>	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
488	>	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
489	>	# remove all but the last file
490	>	filesEventCount = self.eventsbyfile[file]
491	>	parString = "\\{"
492	>	parString += '\\\"' + file + '\\\"\,'
493	>	pass # END if
494	>	pass # END while (iterate over files in the block)
495	>	pass # END while (iterate over blocks in the dataset)
496	>	self.ncjobs = self.total_number_of_jobs = jobCount
497	>	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
498	>	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
499	>	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
500
501	<	else:
502	<	self.total_number_of_files = int(self.total_number_of_events/evPerFile)
313	<	## SL: if ask for less event than what is computed to be available on a
314	<	##     file, process the first file anyhow.
315	<	if self.total_number_of_files == 0:
316	<	self.total_number_of_files = self.total_number_of_files + 1
501	>	self.list_of_args = list_of_lists
502	>	return
503
504	<	common.logger.debug(5,'N files  '+str(self.total_number_of_files))
504	>	def jobSplittingNoInput(self):
505	>	"""
506	>	Perform job splitting based on number of event per job
507	>	"""
508	>	common.logger.debug(5,'Splitting per events')
509	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
510	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
511	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
512
513	<	check = 0
513	>	if (self.total_number_of_events < 0):
514	>	msg='Cannot split jobs per Events with "-1" as total number of events'
515	>	raise CrabException(msg)
516	>
517	>	if (self.selectEventsPerJob):
518	>	self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
519	>	elif (self.selectNumberOfJobs) :
520	>	self.total_number_of_jobs = self.theNumberOfJobs
521	>	self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
522	>
523	>	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
524	>
525	>	# is there any remainder?
526	>	check = int(self.total_number_of_events) - (int(self.total_number_of_jobs)*self.eventsPerJob)
527	>
528	>	common.logger.debug(5,'Check  '+str(check))
529	>
530	>	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created, each for '+str(self.eventsPerJob)+' for a total of '+str(self.total_number_of_jobs*self.eventsPerJob)+' events')
531	>	if check > 0:
532	>	common.logger.message('Warning: asked '+str(self.total_number_of_events)+' but can do only '+str(int(self.total_number_of_jobs)*self.eventsPerJob))
533	>
534	>	# argument is seed number.$i
535	>	self.list_of_args = []
536	>	for i in range(self.total_number_of_jobs):
537	>	## Since there is no input, any site is good
538	>	# self.jobDestination.append(["Any"])
539	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
540	>	args=''
541	>	if (self.firstRun):
542	>	## pythia first run
543	>	#self.list_of_args.append([(str(self.firstRun)+str(i))])
544	>	args=args+(str(self.firstRun)+str(i))
545	>	else:
546	>	## no first run
547	>	#self.list_of_args.append([str(i)])
548	>	args=args+str(i)
549	>	if (self.sourceSeed):
550	>	if (self.sourceSeedVtx):
551	>	## pythia + vtx random seed
552	>	#self.list_of_args.append([
553	>	#                          str(self.sourceSeed)+str(i),
554	>	#                          str(self.sourceSeedVtx)+str(i)
555	>	#                          ])
556	>	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
557	>	else:
558	>	## only pythia random seed
559	>	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
560	>	args=args +str(',')+str(self.sourceSeed)+str(i)
561	>	else:
562	>	## no random seed
563	>	if str(args)=='': args=args+(str(self.firstRun)+str(i))
564	>	arguments=args.split(',')
565	>	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
566	>	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
567	>	else :self.list_of_args.append([str(arguments[0])])
568
569	<	## Compute the number of jobs
323	<	#self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
324	<	self.total_number_of_jobs = int(self.total_number_of_files/self.filesPerJob)
325	<	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
569	>	#   print self.list_of_args
570
571	<	## is there any remainder?
328	<	check = int(self.total_number_of_files) - (int(self.total_number_of_jobs)*self.filesPerJob)
571	>	return
572
330	–	common.logger.debug(5,'Check  '+str(check))
573
574	<	if check > 0:
575	<	self.total_number_of_jobs =  self.total_number_of_jobs + 1
576	<	common.logger.message('Warning: last job will be created with '+str(check)+' files')
574	>	def jobSplittingForScript(self):#CarlosDaniele
575	>	"""
576	>	Perform job splitting based on number of job
577	>	"""
578	>	common.logger.debug(5,'Splitting per job')
579	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
580
581	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs-1)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
337	<	pass
581	>	self.total_number_of_jobs = self.theNumberOfJobs
582
583	<	list_of_lists = []
584	<	for i in xrange(0, int(n_tot_files), self.filesPerJob):
585	<	list_of_lists.append(self.files[0][i: i+self.filesPerJob])
583	>	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
584	>
585	>	common.logger.message(str(self.total_number_of_jobs)+' jobs can be created')
586
587	<	self.list_of_files = list_of_lists
588	<
587	>	# argument is seed number.$i
588	>	self.list_of_args = []
589	>	for i in range(self.total_number_of_jobs):
590	>	## Since there is no input, any site is good
591	>	# self.jobDestination.append(["Any"])
592	>	self.jobDestination.append([""])
593	>	## no random seed
594	>	self.list_of_args.append([str(i)])
595		return
596
597		def split(self, jobParams):
#	Line 349 | Line 599 | class Cmssw(JobType):
599		common.jobDB.load()
600		#### Fabio
601		njobs = self.total_number_of_jobs
602	<	filelist = self.list_of_files
602	>	arglist = self.list_of_args
603		# create the empty structure
604		for i in range(njobs):
605		jobParams.append("")
606
607		for job in range(njobs):
608	<	jobParams[job] = filelist[job]
608	>	jobParams[job] = arglist[job]
609	>	# print str(arglist[job])
610	>	# print jobParams[job]
611		common.jobDB.setArguments(job, jobParams[job])
612	+	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
613	+	common.jobDB.setDestination(job, self.jobDestination[job])
614
615		common.jobDB.save()
616		return
617
618		def getJobTypeArguments(self, nj, sched):
619	<	params = common.jobDB.arguments(nj)
620	<	#print params
621	<	parString = "\\{"
622	<
369	<	for i in range(len(params) - 1):
370	<	parString += '\\\"' + params[i] + '\\\"\,'
371	<
372	<	parString += '\\\"' + params[len(params) - 1] + '\\\"\\}'
373	<	return parString
619	>	result = ''
620	>	for i in common.jobDB.arguments(nj):
621	>	result=result+str(i)+" "
622	>	return result
623
624		def numberOfJobs(self):
625		# Fabio
377	–
626		return self.total_number_of_jobs
379	–
380	–
381	–
382	–	def checkBlackList(self, allSites):
383	–	if len(self.reCEBlackList)==0: return allSites
384	–	sites = []
385	–	for site in allSites:
386	–	common.logger.debug(10,'Site '+site)
387	–	good=1
388	–	for re in self.reCEBlackList:
389	–	if re.search(site):
390	–	common.logger.message('CE in black list, skipping site '+site)
391	–	good=0
392	–	pass
393	–	if good: sites.append(site)
394	–	if len(sites) == 0:
395	–	common.logger.debug(3,"No sites found after BlackList")
396	–	return sites
397	–
398	–	def checkWhiteList(self, allSites):
399	–
400	–	if len(self.reCEWhiteList)==0: return allSites
401	–	sites = []
402	–	for site in allSites:
403	–	good=0
404	–	for re in self.reCEWhiteList:
405	–	if re.search(site):
406	–	common.logger.debug(5,'CE in white list, adding site '+site)
407	–	good=1
408	–	if not good: continue
409	–	sites.append(site)
410	–	if len(sites) == 0:
411	–	common.logger.message("No sites found after WhiteList\n")
412	–	else:
413	–	common.logger.debug(5,"Selected sites via WhiteList are "+str(sites)+"\n")
414	–	return sites
627
628		def getTarBall(self, exe):
629		"""
#	Line 419 | Line 631 | class Cmssw(JobType):
631		"""
632
633		# if it exist, just return it
634	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
634	>	#
635	>	# Marco. Let's start to use relative path for Boss XML files
636	>	#
637	>	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
638		if os.path.exists(self.tgzNameWithPath):
639		return self.tgzNameWithPath
640
#	Line 433 | Line 648 | class Cmssw(JobType):
648		# First of all declare the user Scram area
649		swArea = self.scram.getSWArea_()
650		#print "swArea = ", swArea
651	<	swVersion = self.scram.getSWVersion()
652	<	#print "swVersion = ", swVersion
651	>	# swVersion = self.scram.getSWVersion()
652	>	# print "swVersion = ", swVersion
653		swReleaseTop = self.scram.getReleaseTop_()
654		#print "swReleaseTop = ", swReleaseTop
655
#	Line 442 | Line 657 | class Cmssw(JobType):
657		if swReleaseTop == '' or swArea == swReleaseTop:
658		return
659
660	<	filesToBeTarred = []
661	<	## First find the executable
662	<	if (self.executable != ''):
663	<	exeWithPath = self.scram.findFile_(executable)
664	<	#           print exeWithPath
665	<	if ( not exeWithPath ):
666	<	raise CrabException('User executable '+executable+' not found')
667	<
668	<	## then check if it's private or not
669	<	if exeWithPath.find(swReleaseTop) == -1:
670	<	# the exe is private, so we must ship
671	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
672	<	path = swArea+'/'
673	<	exe = string.replace(exeWithPath, path,'')
674	<	filesToBeTarred.append(exe)
675	<	pass
676	<	else:
677	<	# the exe is from release, we'll find it on WN
678	<	pass
679	<
680	<	## Now get the libraries: only those in local working area
681	<	libDir = 'lib'
682	<	lib = swArea+'/' +libDir
683	<	common.logger.debug(5,"lib "+lib+" to be tarred")
684	<	if os.path.exists(lib):
685	<	filesToBeTarred.append(libDir)
686	<
687	<	## Now check if module dir is present
688	<	moduleDir = 'module'
689	<	if os.path.isdir(swArea+'/'+moduleDir):
690	<	filesToBeTarred.append(moduleDir)
691	<
692	<	## Now check if the Data dir is present
693	<	dataDir = 'src/Data/'
694	<	if os.path.isdir(swArea+'/'+dataDir):
695	<	filesToBeTarred.append(dataDir)
696	<
697	<	## Create the tar-ball
698	<	if len(filesToBeTarred)>0:
699	<	cwd = os.getcwd()
700	<	os.chdir(swArea)
701	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
702	<	for line in filesToBeTarred:
703	<	tarcmd = tarcmd + line + ' '
704	<	cout = runCommand(tarcmd)
705	<	if not cout:
706	<	raise CrabException('Could not create tar-ball')
707	<	os.chdir(cwd)
708	<	else:
709	<	common.logger.debug(5,"No files to be to be tarred")
660	>	import tarfile
661	>	try: # create tar ball
662	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
663	>	## First find the executable
664	>	if (self.executable != ''):
665	>	exeWithPath = self.scram.findFile_(executable)
666	>	if ( not exeWithPath ):
667	>	raise CrabException('User executable '+executable+' not found')
668	>
669	>	## then check if it's private or not
670	>	if exeWithPath.find(swReleaseTop) == -1:
671	>	# the exe is private, so we must ship
672	>	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
673	>	path = swArea+'/'
674	>	exe = string.replace(exeWithPath, path,'')
675	>	tar.add(path+exe,exe)
676	>	pass
677	>	else:
678	>	# the exe is from release, we'll find it on WN
679	>	pass
680	>
681	>	## Now get the libraries: only those in local working area
682	>	libDir = 'lib'
683	>	lib = swArea+'/' +libDir
684	>	common.logger.debug(5,"lib "+lib+" to be tarred")
685	>	if os.path.exists(lib):
686	>	tar.add(lib,libDir)
687	>
688	>	## Now check if module dir is present
689	>	moduleDir = 'module'
690	>	module = swArea + '/' + moduleDir
691	>	if os.path.isdir(module):
692	>	tar.add(module,moduleDir)
693	>
694	>	## Now check if any data dir(s) is present
695	>	swAreaLen=len(swArea)
696	>	for root, dirs, files in os.walk(swArea):
697	>	if "data" in dirs:
698	>	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
699	>	tar.add(root+"/data",root[swAreaLen:]+"/data")
700	>
701	>	## Add ProdAgent dir to tar
702	>	paDir = 'ProdAgentApi'
703	>	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
704	>	if os.path.isdir(pa):
705	>	tar.add(pa,paDir)
706	>
707	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
708	>	tar.close()
709	>	except :
710	>	raise CrabException('Could not create tar-ball')
711	>
712	>	## create tar-ball with ML stuff
713	>	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
714	>	try:
715	>	tar = tarfile.open(self.MLtgzfile, "w:gz")
716	>	path=os.environ['CRABDIR'] + '/python/'
717	>	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
718	>	tar.add(path+file,file)
719	>	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
720	>	tar.close()
721	>	except :
722	>	raise CrabException('Could not create ML files tar-ball')
723
724		return
725
#	Line 508 | Line 736 | class Cmssw(JobType):
736		txt += 'if [ $middleware == LCG ]; then \n'
737		txt += self.wsSetupCMSLCGEnvironment_()
738		txt += 'elif [ $middleware == OSG ]; then\n'
739	<	txt += '    time=`date -u +"%s"`\n'
740	<	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
513	<	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
514	<	txt += '    /bin/mkdir -p $WORKING_DIR\n'
739	>	txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
740	>	txt += '    echo "Created working directory: $WORKING_DIR"\n'
741		txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
742	<	txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be created on on WN `hostname`"\n'
743	<
744	<	txt += '        echo "JOB_EXIT_STATUS = 1"\n'
742	>	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
743	>	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
744	>	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
745	>	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
746	>	txt += '        rm -f $RUNTIME_AREA/$repo \n'
747	>	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
748	>	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
749		txt += '        exit 1\n'
750		txt += '    fi\n'
751		txt += '\n'
#	Line 531 | Line 761 | class Cmssw(JobType):
761		txt += scram+' project CMSSW '+self.version+'\n'
762		txt += 'status=$?\n'
763		txt += 'if [ $status != 0 ] ; then\n'
764	<	txt += '   echo "SET_EXE_ENV 1 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
764	>	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
765		txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
766	<	txt += '   echo "SanityCheckCode = 10034" | tee -a $RUNTIME_AREA/$repo\n'
766	>	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
767		txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
768	+	txt += '   rm -f $RUNTIME_AREA/$repo \n'
769	+	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
770	+	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
771		## OLI_Daniele
772		txt += '    if [ $middleware == OSG ]; then \n'
773		txt += '        echo "Remove working directory: $WORKING_DIR"\n'
774		txt += '        cd $RUNTIME_AREA\n'
775		txt += '        /bin/rm -rf $WORKING_DIR\n'
776		txt += '        if [ -d $WORKING_DIR ] ;then\n'
777	<	txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
777	>	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
778	>	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
779	>	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
780	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
781	>	txt += '            rm -f $RUNTIME_AREA/$repo \n'
782	>	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
783	>	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
784		txt += '        fi\n'
785		txt += '    fi \n'
786		txt += '   exit 1 \n'
#	Line 549 | Line 788 | class Cmssw(JobType):
788		txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
789		txt += 'cd '+self.version+'\n'
790		### needed grep for bug in scramv1 ###
791	+	txt += scram+' runtime -sh\n'
792		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
793	+	txt += 'echo $PATH\n'
794
795		# Handle the arguments:
796		txt += "\n"
797	<	txt += "## ARGUMNETS: $1 Job Number\n"
557	<	# txt += "## ARGUMNETS: $2 First Event for this job\n"
558	<	# txt += "## ARGUMNETS: $3 Max Event for this job\n"
797	>	txt += "## number of arguments (first argument always jobnumber)\n"
798		txt += "\n"
799	<	txt += "narg=$#\n"
800	<	txt += "if [ $narg -lt 2 ]\n"
799	>	#        txt += "narg=$#\n"
800	>	txt += "if [ $nargs -lt 2 ]\n"
801		txt += "then\n"
802	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
802	>	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
803		txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
804	<	txt += '    echo "SanityCheckCode = 50113" | tee -a $RUNTIME_AREA/$repo\n'
804	>	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
805		txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
806	+	txt += '    rm -f $RUNTIME_AREA/$repo \n'
807	+	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
808	+	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
809		## OLI_Daniele
810		txt += '    if [ $middleware == OSG ]; then \n'
811		txt += '        echo "Remove working directory: $WORKING_DIR"\n'
812		txt += '        cd $RUNTIME_AREA\n'
813		txt += '        /bin/rm -rf $WORKING_DIR\n'
814		txt += '        if [ -d $WORKING_DIR ] ;then\n'
815	<	txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
815	>	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
816	>	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
817	>	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
818	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
819	>	txt += '            rm -f $RUNTIME_AREA/$repo \n'
820	>	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
821	>	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
822		txt += '        fi\n'
823		txt += '    fi \n'
824		txt += "    exit 1\n"
825		txt += "fi\n"
826		txt += "\n"
579	–	txt += "NJob=$1\n"
580	–	txt += "InputFiles=$2\n"
581	–	txt += "echo \"<$InputFiles>\"\n"
582	–	# txt += "Args = ` cat $2 |  sed -e \'s/\\\\//g\' -e \'s/\"/\\x27/g\' `"
583	–
584	–	### OLI_DANIELE
585	–	txt += 'if [ $middleware == LCG ]; then \n'
586	–	txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
587	–	txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
588	–	txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo\n'
589	–	txt += 'elif [ $middleware == OSG ]; then\n'
590	–
591	–	# OLI: added monitoring for dashbord, use hash of crab.cfg
592	–	if common.scheduler.boss_scheduler_name == 'condor_g':
593	–	# create hash of cfg file
594	–	hash = makeCksum(common.work_space.cfgFileName())
595	–	txt += '    echo "MonitorJobID=`echo ${NJob}_'+hash+'_$GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
596	–	txt += '    echo "SyncGridJobId=`echo $GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
597	–	txt += '    echo "SyncCE=`echo $hostname`" | tee -a $RUNTIME_AREA/$repo\n'
598	–	else :
599	–	txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
600	–	txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
601	–	txt += '    echo "SyncCE=`$EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo\n'
602	–
603	–	txt += 'fi\n'
604	–	txt += 'dumpStatus $RUNTIME_AREA/$repo\n'
827
828		# Prepare job-specific part
829		job = common.job_list[nj]
830	<	pset = os.path.basename(job.configFilename())
831	<	txt += '\n'
832	<	#txt += 'echo sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' \n'
833	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
834	<	#txt += 'sed "s#{\'INPUT\'}#${InputFiles}#" $RUNTIME_AREA/'+pset+' > pset1.cfg\n'
830	>	if self.pset != None: #CarlosDaniele
831	>	pset = os.path.basename(job.configFilename())
832	>	txt += '\n'
833	>	if (self.datasetPath): # standard job
834	>	#txt += 'InputFiles=$2\n'
835	>	txt += 'InputFiles=${args[1]}\n'
836	>	txt += 'MaxEvents=${args[2]}\n'
837	>	txt += 'SkipEvents=${args[3]}\n'
838	>	txt += 'echo "Inputfiles:<$InputFiles>"\n'
839	>	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
840	>	txt += 'echo "MaxEvents:<$MaxEvents>"\n'
841	>	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
842	>	txt += 'echo "SkipEvents:<$SkipEvents>"\n'
843	>	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
844	>	else:  # pythia like job
845	>	if (self.sourceSeed):
846	>	txt += 'FirstRun=${args[1]}\n'
847	>	txt += 'echo "FirstRun: <$FirstRun>"\n'
848	>	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
849	>	else:
850	>	txt += '# Copy untouched pset\n'
851	>	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
852	>	if (self.sourceSeed):
853	>	#                    txt += 'Seed=$2\n'
854	>	txt += 'Seed=${args[2]}\n'
855	>	txt += 'echo "Seed: <$Seed>"\n'
856	>	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
857	>	if (self.sourceSeedVtx):
858	>	#                        txt += 'VtxSeed=$3\n'
859	>	txt += 'VtxSeed=${args[3]}\n'
860	>	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
861	>	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
862	>	else:
863	>	txt += 'mv tmp_2.cfg pset.cfg\n'
864	>	else:
865	>	txt += 'mv tmp_1.cfg pset.cfg\n'
866	>	# txt += '# Copy untouched pset\n'
867	>	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
868	>
869
870		if len(self.additional_inbox_files) > 0:
871		for file in self.additional_inbox_files:
872	<	txt += 'if [ -e $RUNTIME_AREA/'+file+' ] ; then\n'
873	<	txt += '   cp $RUNTIME_AREA/'+file+' .\n'
874	<	txt += '   chmod +x '+file+'\n'
872	>	relFile = file.split("/")[-1]
873	>	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
874	>	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
875	>	txt += '   chmod +x '+relFile+'\n'
876		txt += 'fi\n'
877		pass
878
879	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
880	<
881	<	txt += '\n'
882	<	txt += 'echo "***** cat pset.cfg *********"\n'
883	<	txt += 'cat pset.cfg\n'
884	<	txt += 'echo "****** end pset.cfg ********"\n'
885	<	txt += '\n'
886	<	# txt += 'echo "***** cat pset1.cfg *********"\n'
887	<	# txt += 'cat pset1.cfg\n'
888	<	# txt += 'echo "****** end pset1.cfg ********"\n'
879	>	if self.pset != None: #CarlosDaniele
880	>	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
881	>
882	>	txt += '\n'
883	>	txt += 'echo "***** cat pset.cfg *********"\n'
884	>	txt += 'cat pset.cfg\n'
885	>	txt += 'echo "****** end pset.cfg ********"\n'
886	>	txt += '\n'
887	>	# txt += 'echo "***** cat pset1.cfg *********"\n'
888	>	# txt += 'cat pset1.cfg\n'
889	>	# txt += 'echo "****** end pset1.cfg ********"\n'
890		return txt
891
892	<	def wsBuildExe(self, nj):
892	>	def wsBuildExe(self, nj=0):
893		"""
894		Put in the script the commands to build an executable
895		or a library.
#	Line 646 | Line 904 | class Cmssw(JobType):
904		txt += 'if [ $untar_status -ne 0 ]; then \n'
905		txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
906		txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
907	<	txt += '   echo "SanityCheckCode = $untar_status" | tee -a $repo\n'
907	>	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
908		txt += '   if [ $middleware == OSG ]; then \n'
909		txt += '       echo "Remove working directory: $WORKING_DIR"\n'
910		txt += '       cd $RUNTIME_AREA\n'
911		txt += '       /bin/rm -rf $WORKING_DIR\n'
912		txt += '       if [ -d $WORKING_DIR ] ;then\n'
913	<	txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
913	>	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
914	>	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
915	>	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
916	>	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
917	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
918	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
919	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
920		txt += '       fi\n'
921		txt += '   fi \n'
922		txt += '   \n'
923	<	txt += '   exit $untar_status \n'
923	>	txt += '   exit 1 \n'
924		txt += 'else \n'
925		txt += '   echo "Successful untar" \n'
926		txt += 'fi \n'
927	+	txt += '\n'
928	+	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
929	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
930	+	txt += '   export PYTHONPATH=ProdAgentApi\n'
931	+	txt += 'else\n'
932	+	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
933	+	txt += 'fi\n'
934	+	txt += '\n'
935	+
936		pass
937
938		return txt
#	Line 671 | Line 944 | class Cmssw(JobType):
944		"""
945
946		def executableName(self):
947	<	return self.executable
947	>	if self.pset == None: #CarlosDaniele
948	>	return "sh "
949	>	else:
950	>	return self.executable
951
952		def executableArgs(self):
953	<	return " -p pset.cfg"
953	>	if self.pset == None:#CarlosDaniele
954	>	return   self.scriptExe + " $NJob"
955	>	else:
956	>	return " -p pset.cfg"
957
958		def inputSandbox(self, nj):
959		"""
960		Returns a list of filenames to be put in JDL input sandbox.
961		"""
962		inp_box = []
963	<	# dict added to delete duplicate from input sandbox file list
964	<	seen = {}
963	>	# # dict added to delete duplicate from input sandbox file list
964	>	# seen = {}
965		## code
966		if os.path.isfile(self.tgzNameWithPath):
967		inp_box.append(self.tgzNameWithPath)
968	+	if os.path.isfile(self.MLtgzfile):
969	+	inp_box.append(self.MLtgzfile)
970		## config
971	<	inp_box.append(common.job_list[nj].configFilename())
971	>	if not self.pset is None: #CarlosDaniele
972	>	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
973		## additional input files
974		#for file in self.additional_inbox_files:
975		#    inp_box.append(common.work_space.cwdDir()+file)
#	Line 699 | Line 981 | class Cmssw(JobType):
981		"""
982		out_box = []
983
702	–	stdout=common.job_list[nj].stdout()
703	–	stderr=common.job_list[nj].stderr()
704	–
984		## User Declared output files
985	<	for out in self.output_file:
985	>	for out in (self.output_file+self.output_file_sandbox):
986		n_out = nj + 1
987		out_box.append(self.numberFile_(out,str(n_out)))
988		return out_box
710	–	return []
989
990		def prepareSteeringCards(self):
991		"""
#	Line 721 | Line 999 | class Cmssw(JobType):
999		"""
1000
1001		txt = '\n'
1002	<	file_list = ''
1003	<	check = len(self.output_file)
1004	<	i = 0
1005	<	for fileWithSuffix in self.output_file:
728	<	i= i + 1
1002	>	txt += '# directory content\n'
1003	>	txt += 'ls \n'
1004	>
1005	>	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1006		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
730	–	file_list=file_list+output_file_num+''
731	–	txt += '\n'
732	–	txt += 'ls \n'
1007		txt += '\n'
1008	+	txt += '# check output file\n'
1009		txt += 'ls '+fileWithSuffix+'\n'
1010	<	txt += 'exe_result=$?\n'
1011	<	txt += 'if [ $exe_result -ne 0 ] ; then\n'
1012	<	txt += '   echo "ERROR: No output file to manage"\n'
1013	<	### OLI_DANIELE
1014	<	txt += '    if [ $middleware == OSG ]; then \n'
1015	<	txt += '        echo "prepare dummy output file"\n'
1016	<	txt += '        cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1017	<	txt += '    fi \n'
1010	>	txt += 'ls_result=$?\n'
1011	>	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1012	>	txt += '   echo "ERROR: Problem with output file"\n'
1013	>	if common.scheduler.boss_scheduler_name == 'condor_g':
1014	>	txt += '    if [ $middleware == OSG ]; then \n'
1015	>	txt += '        echo "prepare dummy output file"\n'
1016	>	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1017	>	txt += '    fi \n'
1018		txt += 'else\n'
1019		txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1020		txt += 'fi\n'
746	–	if i == check:
747	–	txt += 'cd $RUNTIME_AREA\n'
748	–	pass
749	–	pass
1021
1022	<	file_list=file_list[:-1]
1023	<	txt += 'file_list="'+file_list+'"\n'
1022	>	txt += 'cd $RUNTIME_AREA\n'
1023	>	txt += 'cd $RUNTIME_AREA\n'
1024		### OLI_DANIELE
1025		txt += 'if [ $middleware == OSG ]; then\n'
1026		txt += '    cd $RUNTIME_AREA\n'
1027		txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1028		txt += '    /bin/rm -rf $WORKING_DIR\n'
1029		txt += '    if [ -d $WORKING_DIR ] ;then\n'
1030	<	txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
1030	>	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1031	>	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1032	>	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1033	>	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1034	>	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1035	>	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1036	>	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1037		txt += '    fi\n'
1038		txt += 'fi\n'
1039		txt += '\n'
1040	+
1041	+	file_list = ''
1042	+	## Add to filelist only files to be possibly copied to SE
1043	+	for fileWithSuffix in self.output_file:
1044	+	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1045	+	file_list=file_list+output_file_num+' '
1046	+	file_list=file_list[:-1]
1047	+	txt += 'file_list="'+file_list+'"\n'
1048	+
1049		return txt
1050
1051		def numberFile_(self, file, txt):
#	Line 774 | Line 1060 | class Cmssw(JobType):
1060		# add "_txt"
1061		if len(p)>1:
1062		ext = p[len(p)-1]
777	–	#result = name + '_' + str(txt) + "." + ext
1063		result = name + '_' + txt + "." + ext
1064		else:
780	–	#result = name + '_' + str(txt)
1065		result = name + '_' + txt
1066
1067		return result
1068
1069	<	def getRequirements(self):
1069	>	def getRequirements(self, nj=[]):
1070		"""
1071		return job requirements to add to jdl files
1072		"""
1073		req = ''
1074	<	if common.analisys_common_info['sites']:
1075	<	if common.analisys_common_info['sw_version']:
1076	<	req='Member("VO-cms-' + \
1077	<	common.analisys_common_info['sw_version'] + \
1078	<	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1079	<	if len(common.analisys_common_info['sites'])>0:
1080	<	req = req + ' && ('
797	<	for i in range(len(common.analisys_common_info['sites'])):
798	<	req = req + 'other.GlueCEInfoHostName == "' \
799	<	+ common.analisys_common_info['sites'][i] + '"'
800	<	if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
801	<	req = req + ' || '
802	<	req = req + ')'
803	<	#print "req = ", req
1074	>	if self.version:
1075	>	req='Member("VO-cms-' + \
1076	>	self.version + \
1077	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1078	>
1079	>	req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1080	>
1081		return req
1082
1083		def configFilename(self):
#	Line 818 | Line 1095 | class Cmssw(JobType):
1095		txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1096		txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1097		txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1098	<	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
1099	<	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
1100	<	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
1098	>	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1099	>	txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1100	>	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1101		txt += '   else\n'
1102	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
1102	>	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1103		txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1104		txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1105		txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1106	<	txt += '       exit\n'
1106	>	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1107	>	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1108	>	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1109	>	txt += '       exit 1\n'
1110		txt += '\n'
1111		txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1112		txt += '       cd $RUNTIME_AREA\n'
1113		txt += '       /bin/rm -rf $WORKING_DIR\n'
1114		txt += '       if [ -d $WORKING_DIR ] ;then\n'
1115	<	txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
1115	>	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1116	>	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1117	>	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1118	>	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1119	>	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1120	>	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1121	>	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1122		txt += '       fi\n'
1123		txt += '\n'
1124	<	txt += '       exit\n'
1124	>	txt += '       exit 1\n'
1125		txt += '   fi\n'
1126		txt += '\n'
1127		txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
#	Line 851 | Line 1137 | class Cmssw(JobType):
1137		"""
1138		txt  = '   \n'
1139		txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
854	–	txt += '      echo "JOB_EXIT_STATUS = 0"\n'
1140		txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1141		txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1142		txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1143		txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1144		txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1145	<	txt += '       exit\n'
1145	>	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1146	>	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1147	>	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1148	>	txt += '       exit 1\n'
1149		txt += '   else\n'
1150		txt += '       echo "Sourcing environment... "\n'
1151		txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
#	Line 865 | Line 1153 | class Cmssw(JobType):
1153		txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1154		txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1155		txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1156	<	txt += '           exit\n'
1156	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1157	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1158	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1159	>	txt += '           exit 1\n'
1160		txt += '       fi\n'
1161		txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1162		txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
#	Line 875 | Line 1166 | class Cmssw(JobType):
1166		txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1167		txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1168		txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1169	<	txt += '           exit\n'
1169	>	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1170	>	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1171	>	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1172	>	txt += '           exit 1\n'
1173		txt += '       fi\n'
1174		txt += '   fi\n'
1175		txt += '   \n'
#	Line 887 | Line 1181 | class Cmssw(JobType):
1181		txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1182		txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1183		txt += '   else\n'
1184	<	txt += '       echo "SET_CMS_ENV 1 ==> ERROR OS unknown, LCG environment not initialized"\n'
1184	>	txt += '       echo "SET_CMS_ENV 10033 ==> ERROR OS unknown, LCG environment not initialized"\n'
1185		txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1186		txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1187		txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1188	<	txt += '       exit\n'
1188	>	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1189	>	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1190	>	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1191	>	txt += '       exit 1\n'
1192		txt += '   fi\n'
1193		txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1194		txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
#	Line 902 | Line 1199 | class Cmssw(JobType):
1199
1200		def getParams(self):
1201		return self._params
1202	+
1203	+	def setTaskid_(self):
1204	+	self._taskId = self.cfg_params['taskId']
1205	+
1206	+	def getTaskid(self):
1207	+	return self._taskId
1208	+
1209	+	#######################################################################
1210	+	def uniquelist(self, old):
1211	+	"""
1212	+	remove duplicates from a list
1213	+	"""
1214	+	nd={}
1215	+	for e in old:
1216	+	nd[e]=0
1217	+	return nd.keys()

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines