[ViewVC] Diff of: cvsroot/COMP/CRAB/python/cms

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs.
Revision 1.191 by spiga, Tue May 27 22:14:26 2008 UTC

#	Line 5 \| Line 5 \| from crab_util import *
5		from BlackWhiteListParser import BlackWhiteListParser
6		import common
7		import Scram
8	+	from LFNBaseName import *
9
10		import os, string, glob
11
#	Line 13 \| Line 14 \| class Cmssw(JobType):
14		JobType.__init__(self, 'CMSSW')
15		common.logger.debug(3,'CMSSW::__init__')
16
17	+	self.argsList = []
18	+
19		self._params = {}
20		self.cfg_params = cfg_params
18	–
21		# init BlackWhiteListParser
22		self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23
24	<	try:
23	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24	<	except KeyError:
25	<	self.MaxTarBallSize = 9.5
24	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
25
26		# number of jobs requested to be created, limit obj splitting
27		self.ncjobs = ncjobs
28
29		log = common.logger
30	<
30	>
31		self.scram = Scram.Scram(cfg_params)
32		self.additional_inbox_files = []
33		self.scriptExe = ''
34		self.executable = ''
35		self.executable_arch = self.scram.getArch()
36		self.tgz_name = 'default.tgz'
38	–	self.additional_tgz_name = 'additional.tgz'
37		self.scriptName = 'CMSSW.sh'
38	<	self.pset = '' #scrip use case Da
39	<	self.datasetPath = '' #scrip use case Da
38	>	self.pset = ''
39	>	self.datasetPath = ''
40
41		# set FJR file name
42		self.fjrFileName = 'crab_fjr.xml'
43
44		self.version = self.scram.getSWVersion()
45	<
46	<	#
47	<	# Try to block creation in case of arch/version mismatch
48	<	#
49	<
50	<	a = string.split(self.version, "_")
51	<
52	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
54	<	raise CrabException(msg)
57	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
45	>	version_array = self.version.split('_')
46	>	self.CMSSW_major = 0
47	>	self.CMSSW_minor = 0
48	>	self.CMSSW_patch = 0
49	>	try:
50	>	self.CMSSW_major = int(version_array[1])
51	>	self.CMSSW_minor = int(version_array[2])
52	>	self.CMSSW_patch = int(version_array[3])
53	>	except:
54	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
55		raise CrabException(msg)
60	–
61	–	common.taskDB.setDict('codeVersion',self.version)
62	–	self.setParam_('application', self.version)
56
57		### collect Data cards
58
59	<	## get DBS mode
60	<	try:
68	<	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69	<	except KeyError:
70	<	self.use_dbs_1 = 0
71	<
72	<	try:
73	<	tmp = cfg_params['CMSSW.datasetpath']
74	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75	<	if string.lower(tmp)=='none':
76	<	self.datasetPath = None
77	<	self.selectNoInput = 1
78	<	else:
79	<	self.datasetPath = tmp
80	<	self.selectNoInput = 0
81	<	except KeyError:
82	<	msg = "Error: datasetpath not defined "
59	>	if not cfg_params.has_key('CMSSW.datasetpath'):
60	>	msg = "Error: datasetpath not defined "
61		raise CrabException(msg)
62	<
63	<	# ML monitoring
64	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
65	<	if not self.datasetPath:
66	<	self.setParam_('dataset', 'None')
89	<	self.setParam_('owner', 'None')
62	>	tmp = cfg_params['CMSSW.datasetpath']
63	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
64	>	if string.lower(tmp)=='none':
65	>	self.datasetPath = None
66	>	self.selectNoInput = 1
67		else:
68	<	try:
69	<	datasetpath_split = self.datasetPath.split("/")
93	<	# standard style
94	<	self.setParam_('datasetFull', self.datasetPath)
95	<	if self.use_dbs_1 == 1 :
96	<	self.setParam_('dataset', datasetpath_split[1])
97	<	self.setParam_('owner', datasetpath_split[-1])
98	<	else:
99	<	self.setParam_('dataset', datasetpath_split[1])
100	<	self.setParam_('owner', datasetpath_split[2])
101	<	except:
102	<	self.setParam_('dataset', self.datasetPath)
103	<	self.setParam_('owner', self.datasetPath)
104	<
105	<	self.setTaskid_()
106	<	self.setParam_('taskId', self.cfg_params['taskId'])
68	>	self.datasetPath = tmp
69	>	self.selectNoInput = 0
70
71		self.dataTiers = []
72	+
73	+	self.debug_pset = cfg_params.get('USER.debug_pset',False)
74
75		## now the application
76	<	try:
77	<	self.executable = cfg_params['CMSSW.executable']
113	<	self.setParam_('exe', self.executable)
114	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
115	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
116	<	log.debug(3,msg)
117	<	except KeyError:
118	<	self.executable = 'cmsRun'
119	<	self.setParam_('exe', self.executable)
120	<	msg = "User executable not defined. Use cmsRun"
121	<	log.debug(3,msg)
122	<	pass
76	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
77	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
78
79	<	try:
125	<	self.pset = cfg_params['CMSSW.pset']
126	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
127	<	if self.pset.lower() != 'none' :
128	<	if (not os.path.exists(self.pset)):
129	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
130	<	else:
131	<	self.pset = None
132	<	except KeyError:
79	>	if not cfg_params.has_key('CMSSW.pset'):
80		raise CrabException("PSet file missing. Cannot run cmsRun ")
81	+	self.pset = cfg_params['CMSSW.pset']
82	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
83	+	if self.pset.lower() != 'none' :
84	+	if (not os.path.exists(self.pset)):
85	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
86	+	else:
87	+	self.pset = None
88
89		# output files
90		## stuff which must be returned always via sandbox
#	Line 140 \| Line 94 \| class Cmssw(JobType):
94		self.output_file_sandbox.append(self.fjrFileName)
95
96		# other output files to be returned via sandbox or copied to SE
97	<	try:
98	<	self.output_file = []
99	<	tmp = cfg_params['CMSSW.output_file']
100	<	if tmp != '':
101	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
102	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
103	<	for tmp in tmpOutFiles:
104	<	tmp=string.strip(tmp)
151	<	self.output_file.append(tmp)
152	<	pass
153	<	else:
154	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
97	>	self.output_file = []
98	>	tmp = cfg_params.get('CMSSW.output_file',None)
99	>	if tmp :
100	>	tmpOutFiles = string.split(tmp,',')
101	>	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
102	>	for tmp in tmpOutFiles:
103	>	tmp=string.strip(tmp)
104	>	self.output_file.append(tmp)
105		pass
106	<	pass
157	<	except KeyError:
106	>	else:
107		log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
108	<	pass
108	>	pass
109
110		# script_exe file as additional file in inputSandbox
111	<	try:
112	<	self.scriptExe = cfg_params['USER.script_exe']
113	<	if self.scriptExe != '':
114	<	if not os.path.isfile(self.scriptExe):
115	<	msg ="ERROR. file "+self.scriptExe+" not found"
116	<	raise CrabException(msg)
168	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
169	<	except KeyError:
170	<	self.scriptExe = ''
111	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
112	>	if self.scriptExe :
113	>	if not os.path.isfile(self.scriptExe):
114	>	msg ="ERROR. file "+self.scriptExe+" not found"
115	>	raise CrabException(msg)
116	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
117
172	–	#CarlosDaniele
118		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
119	<	msg ="Error. script_exe not defined"
120	<	raise CrabException(msg)
119	>	msg ="Error. script_exe not defined"
120	>	raise CrabException(msg)
121
122		## additional input files
123	<	try:
123	>	if cfg_params.has_key('USER.additional_input_files'):
124		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
125		for tmp in tmpAddFiles:
126		tmp = string.strip(tmp)
#	Line 192 \| Line 137 \| class Cmssw(JobType):
137		if not os.path.exists(file):
138		raise CrabException("Additional input file not found: "+file)
139		pass
195	–	# fname = string.split(file, '/')[-1]
196	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
197	–	# shutil.copyfile(file, storedFile)
140		self.additional_inbox_files.append(string.strip(file))
141		pass
142		pass
143		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
144	<	except KeyError:
203	<	pass
204	<
205	<	# files per job
206	<	try:
207	<	if (cfg_params['CMSSW.files_per_jobs']):
208	<	raise CrabException("files_per_jobs no longer supported. Quitting.")
209	<	except KeyError:
210	<	pass
144	>	pass
145
146		## Events per job
147	<	try:
147	>	if cfg_params.has_key('CMSSW.events_per_job'):
148		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
149		self.selectEventsPerJob = 1
150	<	except KeyError:
150	>	else:
151		self.eventsPerJob = -1
152		self.selectEventsPerJob = 0
153	<
153	>
154		## number of jobs
155	<	try:
155	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
156		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
157		self.selectNumberOfJobs = 1
158	<	except KeyError:
158	>	else:
159		self.theNumberOfJobs = 0
160		self.selectNumberOfJobs = 0
161
162	<	try:
162	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
163		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
164		self.selectTotalNumberEvents = 1
165	<	except KeyError:
165	>	else:
166		self.total_number_of_events = 0
167		self.selectTotalNumberEvents = 0
168
169	<	if self.pset != None: #CarlosDaniele
169	>	if self.pset != None:
170		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
171		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
172		raise CrabException(msg)
#	Line 241 \| Line 175 \| class Cmssw(JobType):
175		msg = 'Must specify number_of_jobs.'
176		raise CrabException(msg)
177
178	<	## source seed for pythia
179	<	try:
180	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
181	<	except KeyError:
182	<	self.sourceSeed = None
183	<	common.logger.debug(5,"No seed given")
178	>	## New method of dealing with seeds
179	>	self.incrementSeeds = []
180	>	self.preserveSeeds = []
181	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
182	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
183	>	for tmp in tmpList:
184	>	tmp.strip()
185	>	self.preserveSeeds.append(tmp)
186	>	if cfg_params.has_key('CMSSW.increment_seeds'):
187	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
188	>	for tmp in tmpList:
189	>	tmp.strip()
190	>	self.incrementSeeds.append(tmp)
191	>
192	>	## Old method of dealing with seeds
193	>	## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
194	>	## remove
195	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
196	>	if self.sourceSeed:
197	>	print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
198	>	self.incrementSeeds.append('sourceSeed')
199	>	self.incrementSeeds.append('theSource')
200	>
201	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
202	>	if self.sourceSeedVtx:
203	>	print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
204	>	self.incrementSeeds.append('VtxSmeared')
205	>
206	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
207	>	if self.sourceSeedG4:
208	>	print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
209	>	self.incrementSeeds.append('g4SimHits')
210	>
211	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
212	>	if self.sourceSeedMix:
213	>	print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
214	>	self.incrementSeeds.append('mix')
215
216	<	try:
252	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
253	<	except KeyError:
254	<	self.sourceSeedVtx = None
255	<	common.logger.debug(5,"No vertex seed given")
256	<
257	<	try:
258	<	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
259	<	except KeyError:
260	<	self.sourceSeedG4 = None
261	<	common.logger.debug(5,"No g4 sim hits seed given")
216	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
217
263	–	try:
264	–	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
265	–	except KeyError:
266	–	self.sourceSeedMix = None
267	–	common.logger.debug(5,"No mix seed given")
268	–
269	–	try:
270	–	self.firstRun = int(cfg_params['CMSSW.first_run'])
271	–	except KeyError:
272	–	self.firstRun = None
273	–	common.logger.debug(5,"No first run given")
218		if self.pset != None: #CarlosDaniele
219	<	ver = string.split(self.version,"_")
276	<	if (int(ver[1])>=1 and int(ver[2])>=5):
277	<	import PsetManipulator150 as pp
278	<	else:
279	<	import PsetManipulator as pp
219	>	import PsetManipulator as pp
220		PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
221
222	+	# Copy/return
223	+
224	+	self.copy_data = int(cfg_params.get('USER.copy_data',0))
225	+	self.return_data = int(cfg_params.get('USER.return_data',0))
226	+
227		#DBSDLS-start
228	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
228	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
229		self.maxEvents=0 # max events available ( --> check the requested nb. of evts in Creator.py)
230		self.DBSPaths={} # all dbs paths requested ( --> input to the site local discovery script)
231		self.jobDestination=[] # Site destination(s) for each job (list of lists)
#	Line 289 \| Line 234 \| class Cmssw(JobType):
234		blockSites = {}
235		if self.datasetPath:
236		blockSites = self.DataDiscoveryAndLocation(cfg_params)
237	<	#DBSDLS-end
237	>	#DBSDLS-end
238	>
239
294	–	self.tgzNameWithPath = self.getTarBall(self.executable)
295	–
240		## Select Splitting
241	<	if self.selectNoInput:
242	<	if self.pset == None: #CarlosDaniele
241	>	if self.selectNoInput:
242	>	if self.pset == None:
243		self.jobSplittingForScript()
244		else:
245		self.jobSplittingNoInput()
#	Line 303 \| Line 247 \| class Cmssw(JobType):
247		self.jobSplittingByBlocks(blockSites)
248
249		# modify Pset
250	<	if self.pset != None: #CarlosDaniele
250	>	if self.pset != None:
251		try:
252	<	if (self.datasetPath): # standard job
253	<	# allow to processa a fraction of events in a file
254	<	PsetEdit.inputModule("INPUT")
255	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
312	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
313	<	else: # pythia like job
314	<	PsetEdit.maxEvent(self.eventsPerJob)
315	<	if (self.firstRun):
316	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN") #First Run
317	<	if (self.sourceSeed) :
318	<	PsetEdit.pythiaSeed("INPUT")
319	<	if (self.sourceSeedVtx) :
320	<	PsetEdit.vtxSeed("INPUTVTX")
321	<	if (self.sourceSeedG4) :
322	<	PsetEdit.g4Seed("INPUTG4")
323	<	if (self.sourceSeedMix) :
324	<	PsetEdit.mixSeed("INPUTMIX")
325	<	# add FrameworkJobReport to parameter-set
326	<	PsetEdit.addCrabFJR(self.fjrFileName)
252	>	# Add FrameworkJobReport to parameter-set, set max events.
253	>	# Reset later for data jobs by writeCFG which does all modifications
254	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
255	>	PsetEdit.maxEvent(self.eventsPerJob)
256		PsetEdit.psetWriter(self.configFilename())
257		except:
258	<	msg='Error while manipuliating ParameterSet: exiting...'
258	>	msg='Error while manipulating ParameterSet: exiting...'
259		raise CrabException(msg)
260	+	self.tgzNameWithPath = self.getTarBall(self.executable)
261
262		def DataDiscoveryAndLocation(self, cfg_params):
263
264		import DataDiscovery
335	–	import DataDiscovery_DBS2
265		import DataLocation
266		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
267
#	Line 341 \| Line 270 \| class Cmssw(JobType):
270		## Contact the DBS
271		common.logger.message("Contacting Data Discovery Services ...")
272		try:
273	<
345	<	if self.use_dbs_1 == 1 :
346	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
347	<	else :
348	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
273	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
274		self.pubdata.fetchDBSInfo()
275
276		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 357 \| Line 282 \| class Cmssw(JobType):
282		except DataDiscovery.DataDiscoveryError, ex:
283		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
284		raise CrabException(msg)
360	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
361	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
362	–	raise CrabException(msg)
363	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
364	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
365	–	raise CrabException(msg)
366	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
367	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
368	–	raise CrabException(msg)
285
286		self.filesbyblock=self.pubdata.getFiles()
287		self.eventsbyblock=self.pubdata.getEventsPerBlock()
288		self.eventsbyfile=self.pubdata.getEventsPerFile()
289
290		## get max number of events
291	<	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
291	>	self.maxEvents=self.pubdata.getMaxEvents()
292
293		## Contact the DLS and build a list of sites hosting the fileblocks
294		try:
#	Line 381 \| Line 297 \| class Cmssw(JobType):
297		except DataLocation.DataLocationError , ex:
298		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
299		raise CrabException(msg)
300	<
300	>
301
302		sites = dataloc.getSites()
303		allSites = []
#	Line 395 \| Line 311 \| class Cmssw(JobType):
311		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
312
313		return sites
314	<
314	>
315		def jobSplittingByBlocks(self, blockSites):
316		"""
317		Perform job splitting. Jobs run over an integer number of files
#	Line 445 \| Line 361 \| class Cmssw(JobType):
361		totalNumberOfJobs = 999999999
362		else :
363		totalNumberOfJobs = self.ncjobs
448	–
364
365		blocks = blockSites.keys()
366		blockCount = 0
#	Line 465 \| Line 380 \| class Cmssw(JobType):
380		blockCount += 1
381		if block not in jobsOfBlock.keys() :
382		jobsOfBlock[block] = []
383	<
383	>
384		if self.eventsbyblock.has_key(block) :
385		numEventsInBlock = self.eventsbyblock[block]
386		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
387	<
387	>
388		files = self.filesbyblock[block]
389		numFilesInBlock = len(files)
390		if (numFilesInBlock <= 0):
#	Line 477 \| Line 392 \| class Cmssw(JobType):
392		fileCount = 0
393
394		# ---- New block => New job ---- #
395	<	parString = "\\{"
395	>	parString = ""
396		# counter for number of events in files currently worked on
397		filesEventCount = 0
398		# flag if next while loop should touch new file
399		newFile = 1
400		# job event counter
401		jobSkipEventCount = 0
402	<
402	>
403		# ---- Iterate over the files in the block until we've met the requested ---- #
404		# ---- total # of events or we've gone over all the files in this block ---- #
405		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
#	Line 500 \| Line 415 \| class Cmssw(JobType):
415		newFile = 0
416		except KeyError:
417		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
503	–
418
419	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
420		# if less events in file remain than eventsPerJobRequested
421	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
421	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
422		# if last file in block
423		if ( fileCount == numFilesInBlock-1 ) :
424		# end job using last file, use remaining events in block
425		# close job and touch new file
426		fullString = parString[:-2]
512	–	fullString += '\\}'
427		list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
428		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
429		self.jobDestination.append(blockSites[block])
#	Line 522 \| Line 436 \| class Cmssw(JobType):
436		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
437		jobSkipEventCount = 0
438		# reset file
439	<	parString = "\\{"
439	>	parString = ""
440		filesEventCount = 0
441		newFile = 1
442		fileCount += 1
#	Line 534 \| Line 448 \| class Cmssw(JobType):
448		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
449		# close job and touch new file
450		fullString = parString[:-2]
537	–	fullString += '\\}'
451		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
452		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
453		self.jobDestination.append(blockSites[block])
#	Line 546 \| Line 459 \| class Cmssw(JobType):
459		eventsRemaining = eventsRemaining - eventsPerJobRequested
460		jobSkipEventCount = 0
461		# reset file
462	<	parString = "\\{"
462	>	parString = ""
463		filesEventCount = 0
464		newFile = 1
465		fileCount += 1
466	<
466	>
467		# if more events in file remain than eventsPerJobRequested
468		else :
469		# close job but don't touch new file
470		fullString = parString[:-2]
558	–	fullString += '\\}'
471		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
472		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
473		self.jobDestination.append(blockSites[block])
#	Line 570 \| Line 482 \| class Cmssw(JobType):
482		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
483		# remove all but the last file
484		filesEventCount = self.eventsbyfile[file]
485	<	parString = "\\{"
574	<	parString += '\\\"' + file + '\\\"\,'
485	>	parString = '\\\"' + file + '\\\"\,'
486		pass # END if
487		pass # END while (iterate over files in the block)
488		pass # END while (iterate over blocks in the dataset)
#	Line 579 \| Line 490 \| class Cmssw(JobType):
490		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
491		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
492		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
493	<
493	>
494		# screen output
495		screenOutput = "List of jobs and available destination sites:\n\n"
496
#	Line 591 \| Line 502 \| class Cmssw(JobType):
502		for block in blocks:
503		if block in jobsOfBlock.keys() :
504		blockCounter += 1
505	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
505	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
506	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
507		if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
508	<	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
508	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
509		bloskNoSite.append( blockCounter )
510	<
510	>
511		common.logger.message(screenOutput)
512		if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
513		msg = 'WARNING: No sites are hosting any part of data for block:\n '
#	Line 611 \| Line 523 \| class Cmssw(JobType):
523		for range_jobs in noSiteBlock:
524		msg += str(range_jobs) + virgola
525		msg += '\n will not be submitted and this block of data can not be analyzed!\n'
526	+	if self.cfg_params.has_key('EDG.se_white_list'):
527	+	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
528	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
529	+	msg += 'Please check if the dataset is available at this site!)\n'
530	+	if self.cfg_params.has_key('EDG.ce_white_list'):
531	+	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
532	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
533	+	msg += 'Please check if the dataset is available at this site!)\n'
534	+
535		common.logger.message(msg)
536
537		self.list_of_args = list_of_lists
#	Line 621 \| Line 542 \| class Cmssw(JobType):
542		Perform job splitting based on number of event per job
543		"""
544		common.logger.debug(5,'Splitting per events')
545	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
546	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
547	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
545	>
546	>	if (self.selectEventsPerJob):
547	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
548	>	if (self.selectNumberOfJobs):
549	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
550	>	if (self.selectTotalNumberEvents):
551	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
552
553		if (self.total_number_of_events < 0):
554		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 632 \| Line 557 \| class Cmssw(JobType):
557		if (self.selectEventsPerJob):
558		if (self.selectTotalNumberEvents):
559		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
560	<	elif(self.selectNumberOfJobs) :
560	>	elif(self.selectNumberOfJobs) :
561		self.total_number_of_jobs =self.theNumberOfJobs
562	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
562	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
563
564		elif (self.selectNumberOfJobs) :
565		self.total_number_of_jobs = self.theNumberOfJobs
566		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
567	<
567	>
568		common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs))
569
570		# is there any remainder?
#	Line 655 \| Line 580 \| class Cmssw(JobType):
580		self.list_of_args = []
581		for i in range(self.total_number_of_jobs):
582		## Since there is no input, any site is good
583	<	# self.jobDestination.append(["Any"])
659	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
583	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
584		args=[]
585		if (self.firstRun):
586	<	## pythia first run
663	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
586	>	## pythia first run
587		args.append(str(self.firstRun)+str(i))
665	–	else:
666	–	## no first run
667	–	#self.list_of_args.append([str(i)])
668	–	args.append(str(i))
669	–	if (self.sourceSeed):
670	–	args.append(str(self.sourceSeed)+str(i))
671	–	if (self.sourceSeedVtx):
672	–	## + vtx random seed
673	–	args.append(str(self.sourceSeedVtx)+str(i))
674	–	if (self.sourceSeedG4):
675	–	## + G4 random seed
676	–	args.append(str(self.sourceSeedG4)+str(i))
677	–	if (self.sourceSeedMix):
678	–	## + Mix random seed
679	–	args.append(str(self.sourceSeedMix)+str(i))
680	–	pass
681	–	pass
588		self.list_of_args.append(args)
683	–	pass
684	–
685	–	# print self.list_of_args
589
590		return
591
592
593	<	def jobSplittingForScript(self):#CarlosDaniele
593	>	def jobSplittingForScript(self):
594		"""
595		Perform job splitting based on number of job
596		"""
#	Line 703 \| Line 606 \| class Cmssw(JobType):
606		# argument is seed number.$i
607		self.list_of_args = []
608		for i in range(self.total_number_of_jobs):
706	–	## Since there is no input, any site is good
707	–	# self.jobDestination.append(["Any"])
609		self.jobDestination.append([""])
709	–	## no random seed
610		self.list_of_args.append([str(i)])
611		return
612
613		def split(self, jobParams):
614	<
715	<	common.jobDB.load()
716	<	#### Fabio
614	>
615		njobs = self.total_number_of_jobs
616		arglist = self.list_of_args
617		# create the empty structure
618		for i in range(njobs):
619		jobParams.append("")
620	<
620	>
621	>	listID=[]
622	>	listField=[]
623		for job in range(njobs):
624		jobParams[job] = arglist[job]
625	<	# print str(arglist[job])
626	<	# print jobParams[job]
627	<	common.jobDB.setArguments(job, jobParams[job])
628	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
629	<	common.jobDB.setDestination(job, self.jobDestination[job])
625	>	listID.append(job+1)
626	>	job_ToSave ={}
627	>	concString = ' '
628	>	argu=''
629	>	if len(jobParams[job]):
630	>	argu += concString.join(jobParams[job] )
631	>	job_ToSave['arguments']= str(job+1)+' '+argu
632	>	job_ToSave['dlsDestination']= self.jobDestination[job]
633	>	listField.append(job_ToSave)
634	>	msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \
635	>	+" Destination: "+str(self.jobDestination[job])
636	>	common.logger.debug(5,msg)
637	>	common._db.updateJob_(listID,listField)
638	>	self.argsList = (len(jobParams[0])+1)
639
731	–	common.jobDB.save()
640		return
641	<
734	<	def getJobTypeArguments(self, nj, sched):
735	<	result = ''
736	<	for i in common.jobDB.arguments(nj):
737	<	result=result+str(i)+" "
738	<	return result
739	<
641	>
642		def numberOfJobs(self):
741	–	# Fabio
643		return self.total_number_of_jobs
644
645		def getTarBall(self, exe):
646		"""
647		Return the TarBall with lib and exe
648		"""
748	–
749	–	# if it exist, just return it
750	–	#
751	–	# Marco. Let's start to use relative path for Boss XML files
752	–	#
649		self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
650		if os.path.exists(self.tgzNameWithPath):
651		return self.tgzNameWithPath
#	Line 763 \| Line 659 \| class Cmssw(JobType):
659
660		# First of all declare the user Scram area
661		swArea = self.scram.getSWArea_()
766	–	#print "swArea = ", swArea
767	–	# swVersion = self.scram.getSWVersion()
768	–	# print "swVersion = ", swVersion
662		swReleaseTop = self.scram.getReleaseTop_()
663	<	#print "swReleaseTop = ", swReleaseTop
771	<
663	>
664		## check if working area is release top
665		if swReleaseTop == '' or swArea == swReleaseTop:
666	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
667		return
668
669		import tarfile
#	Line 781 \| Line 674 \| class Cmssw(JobType):
674		exeWithPath = self.scram.findFile_(executable)
675		if ( not exeWithPath ):
676		raise CrabException('User executable '+executable+' not found')
677	<
677	>
678		## then check if it's private or not
679		if exeWithPath.find(swReleaseTop) == -1:
680		# the exe is private, so we must ship
#	Line 790 \| Line 683 \| class Cmssw(JobType):
683		# distinguish case when script is in user project area or given by full path somewhere else
684		if exeWithPath.find(path) >= 0 :
685		exe = string.replace(exeWithPath, path,'')
686	<	tar.add(path+exe,os.path.basename(executable))
686	>	tar.add(path+exe,exe)
687		else :
688		tar.add(exeWithPath,os.path.basename(executable))
689		pass
690		else:
691		# the exe is from release, we'll find it on WN
692		pass
693	<
693	>
694		## Now get the libraries: only those in local working area
695		libDir = 'lib'
696		lib = swArea+'/' +libDir
697		common.logger.debug(5,"lib "+lib+" to be tarred")
698		if os.path.exists(lib):
699		tar.add(lib,libDir)
700	<
700	>
701		## Now check if module dir is present
702		moduleDir = 'module'
703		module = swArea + '/' + moduleDir
#	Line 813 \| Line 706 \| class Cmssw(JobType):
706
707		## Now check if any data dir(s) is present
708		swAreaLen=len(swArea)
709	+	self.dataExist = False
710		for root, dirs, files in os.walk(swArea):
711		if "data" in dirs:
712	+	self.dataExist=True
713		common.logger.debug(5,"data "+root+"/data"+" to be tarred")
714		tar.add(root+"/data",root[swAreaLen:]+"/data")
715
716	<	## Add ProdAgent dir to tar
717	<	paDir = 'ProdAgentApi'
718	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
719	<	if os.path.isdir(pa):
720	<	tar.add(pa,paDir)
716	>	### CMSSW ParameterSet
717	>	if not self.pset is None:
718	>	cfg_file = common.work_space.jobDir()+self.configFilename()
719	>	tar.add(cfg_file,self.configFilename())
720	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
721	>
722
723	<	### FEDE FOR DBS PUBLICATION
828	<	## Add PRODCOMMON dir to tar
723	>	## Add ProdCommon dir to tar
724		prodcommonDir = 'ProdCommon'
725		prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
726		if os.path.isdir(prodcommonPath):
727		tar.add(prodcommonPath,prodcommonDir)
833	–	#############################
834	–
728		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
729	+
730	+	##### ML stuff
731	+	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
732	+	path=os.environ['CRABDIR'] + '/python/'
733	+	for file in ML_file_list:
734	+	tar.add(path+file,file)
735	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
736	+
737	+	##### Utils
738	+	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']
739	+	for file in Utils_file_list:
740	+	tar.add(path+file,file)
741	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
742	+
743	+	##### AdditionalFiles
744	+	for file in self.additional_inbox_files:
745	+	tar.add(file,string.split(file,'/')[-1])
746	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
747	+
748		tar.close()
749		except :
750		raise CrabException('Could not create tar-ball')
#	Line 843 \| Line 755 \| class Cmssw(JobType):
755		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
756
757		## create tar-ball with ML stuff
846	–	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
847	–	try:
848	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
849	–	path=os.environ['CRABDIR'] + '/python/'
850	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
851	–	tar.add(path+file,file)
852	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
853	–	tar.close()
854	–	except :
855	–	raise CrabException('Could not create ML files tar-ball')
856	–
857	–	return
858	–
859	–	def additionalInputFileTgz(self):
860	–	"""
861	–	Put all additional files into a tar ball and return its name
862	–	"""
863	–	import tarfile
864	–	tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
865	–	tar = tarfile.open(tarName, "w:gz")
866	–	for file in self.additional_inbox_files:
867	–	tar.add(file,string.split(file,'/')[-1])
868	–	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
869	–	tar.close()
870	–	return tarName
758
759	<	def wsSetupEnvironment(self, nj):
759	>	def wsSetupEnvironment(self, nj=0):
760		"""
761		Returns part of a job script which prepares
762		the execution environment for the job 'nj'.
763		"""
764	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
765	+	psetName = 'pset.py'
766	+	else:
767	+	psetName = 'pset.cfg'
768		# Prepare JobType-independent part
769	<	txt = ''
770	<
771	<	## OLI_Daniele at this level middleware already known
881	<
882	<	txt += 'if [ $middleware == LCG ]; then \n'
883	<	txt += ' echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
884	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
885	<	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
769	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
770	>	txt += 'echo ">>> setup environment"\n'
771	>	txt += 'if [ $middleware == LCG ]; then \n'
772		txt += self.wsSetupCMSLCGEnvironment_()
773		txt += 'elif [ $middleware == OSG ]; then\n'
774		txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
775	<	txt += ' echo "Created working directory: $WORKING_DIR"\n'
776	<	txt += ' if [ ! -d $WORKING_DIR ] ;then\n'
777	<	txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
778	<	txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
893	<	txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
894	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
895	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
896	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
897	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
898	<	txt += ' exit 1\n'
775	>	txt += ' if [ ! $? == 0 ] ;then\n'
776	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
777	>	txt += ' job_exit_code=10016\n'
778	>	txt += ' func_exit\n'
779		txt += ' fi\n'
780	+	txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
781		txt += '\n'
782		txt += ' echo "Change to working directory: $WORKING_DIR"\n'
783		txt += ' cd $WORKING_DIR\n'
784	<	txt += self.wsSetupCMSOSGEnvironment_()
785	<	txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
905	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
784	>	txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
785	>	txt += self.wsSetupCMSOSGEnvironment_()
786		txt += 'fi\n'
787
788		# Prepare JobType-specific part
789		scram = self.scram.commandName()
790		txt += '\n\n'
791	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
791	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
792	>	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
793		txt += scram+' project CMSSW '+self.version+'\n'
794		txt += 'status=$?\n'
795		txt += 'if [ $status != 0 ] ; then\n'
796	<	txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
797	<	txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
798	<	txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
918	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
919	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
920	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
921	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
922	<	## OLI_Daniele
923	<	txt += ' if [ $middleware == OSG ]; then \n'
924	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
925	<	txt += ' cd $RUNTIME_AREA\n'
926	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
927	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
928	<	txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
929	<	txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
930	<	txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
932	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
933	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
934	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
935	<	txt += ' fi\n'
936	<	txt += ' fi \n'
937	<	txt += ' exit 1 \n'
796	>	txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
797	>	txt += ' job_exit_code=10034\n'
798	>	txt += ' func_exit\n'
799		txt += 'fi \n'
939	–	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
800		txt += 'cd '+self.version+'\n'
941	–	########## FEDE FOR DBS2 ######################
801		txt += 'SOFTWARE_DIR=`pwd`\n'
802	<	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
944	<	###############################################
945	<	### needed grep for bug in scramv1 ###
946	<	txt += scram+' runtime -sh\n'
802	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
803		txt += 'eval `'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME`\n'
804	<	txt += 'echo $PATH\n'
805	<
804	>	txt += 'if [ $? != 0 ] ; then\n'
805	>	txt += ' echo "ERROR ==> Problem with the command: "\n'
806	>	txt += ' echo "eval \`'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
807	>	txt += ' job_exit_code=10034\n'
808	>	txt += ' func_exit\n'
809	>	txt += 'fi \n'
810		# Handle the arguments:
811		txt += "\n"
812		txt += "## number of arguments (first argument always jobnumber)\n"
813		txt += "\n"
814	<	# txt += "narg=$#\n"
955	<	txt += "if [ $nargs -lt 2 ]\n"
814	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
815		txt += "then\n"
816	<	txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
817	<	txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
818	<	txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
960	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
961	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
962	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
963	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
964	<	## OLI_Daniele
965	<	txt += ' if [ $middleware == OSG ]; then \n'
966	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
967	<	txt += ' cd $RUNTIME_AREA\n'
968	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
969	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
970	<	txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
971	<	txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
972	<	txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
973	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
974	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
975	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
976	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
977	<	txt += ' fi\n'
978	<	txt += ' fi \n'
979	<	txt += " exit 1\n"
816	>	txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
817	>	txt += ' job_exit_code=50113\n'
818	>	txt += " func_exit\n"
819		txt += "fi\n"
820		txt += "\n"
821
822		# Prepare job-specific part
823		job = common.job_list[nj]
824	<	### FEDE FOR DBS OUTPUT PUBLICATION
986	<	if (self.datasetPath):
824	>	if (self.datasetPath):
825		txt += '\n'
826		txt += 'DatasetPath='+self.datasetPath+'\n'
827
828		datasetpath_split = self.datasetPath.split("/")
829	<
829	>
830		txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
831		txt += 'DataTier='+datasetpath_split[2]+'\n'
994	–	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
832		txt += 'ApplicationFamily=cmsRun\n'
833
834		else:
835		txt += 'DatasetPath=MCDataTier\n'
836		txt += 'PrimaryDataset=null\n'
837		txt += 'DataTier=null\n'
1001	–	#txt += 'ProcessedDataset=null\n'
838		txt += 'ApplicationFamily=MCDataTier\n'
839	<	if self.pset != None: #CarlosDaniele
839	>	if self.pset != None:
840		pset = os.path.basename(job.configFilename())
841		txt += '\n'
842		txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
843		if (self.datasetPath): # standard job
844	<	#txt += 'InputFiles=$2\n'
845	<	txt += 'InputFiles=${args[1]}\n'
846	<	txt += 'MaxEvents=${args[2]}\n'
1011	<	txt += 'SkipEvents=${args[3]}\n'
844	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
845	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
846	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
847		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1013	–	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
848		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1015	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
849		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1017	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
850		else: # pythia like job
851	<	seedIndex=1
851	>	txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
852	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
853	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
854	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
855		if (self.firstRun):
856	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
856	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
857		txt += 'echo "FirstRun: <$FirstRun>"\n'
1023	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1024	–	seedIndex=seedIndex+1
858
859	<	if (self.sourceSeed):
1027	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1028	<	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1029	<	seedIndex=seedIndex+1
1030	<	## the following seeds are not always present
1031	<	if (self.sourceSeedVtx):
1032	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1033	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1034	<	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1035	<	seedIndex += 1
1036	<	if (self.sourceSeedG4):
1037	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1038	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
1039	<	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1040	<	seedIndex += 1
1041	<	if (self.sourceSeedMix):
1042	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1043	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
1044	<	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1045	<	seedIndex += 1
1046	<	pass
1047	<	pass
1048	<	txt += 'mv -f '+pset+' pset.cfg\n'
859	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
860
1050	–	if len(self.additional_inbox_files) > 0:
1051	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1052	–	txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1053	–	txt += 'fi\n'
1054	–	pass
861
862	<	if self.pset != None: #CarlosDaniele
863	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1058	<
1059	<	txt += '\n'
1060	<	txt += 'echo "*** cat pset.cfg *******"\n'
1061	<	txt += 'cat pset.cfg\n'
1062	<	txt += 'echo "**** end pset.cfg ******"\n'
862	>	if self.pset != None:
863	>	# FUTURE: Can simply for 2_1_x and higher
864		txt += '\n'
865	<	### FEDE FOR DBS OUTPUT PUBLICATION
866	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
865	>	if self.debug_pset==True:
866	>	txt += 'echo "*** cat ' + psetName + ' *******"\n'
867	>	txt += 'cat ' + psetName + '\n'
868	>	txt += 'echo "**** end ' + psetName + ' ******"\n'
869	>	txt += '\n'
870	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
871		txt += 'echo "PSETHASH = $PSETHASH" \n'
1067	–	##############
872		txt += '\n'
1069	–	# txt += 'echo "*** cat pset1.cfg *******"\n'
1070	–	# txt += 'cat pset1.cfg\n'
1071	–	# txt += 'echo "**** end pset1.cfg ******"\n'
873		return txt
874
875	<	def wsBuildExe(self, nj=0):
875	>	def wsUntarSoftware(self, nj=0):
876		"""
877		Put in the script the commands to build an executable
878		or a library.
879		"""
880
881	<	txt = ""
881	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
882
883		if os.path.isfile(self.tgzNameWithPath):
884	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
884	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
885		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
886	+	txt += 'ls -Al \n'
887		txt += 'untar_status=$? \n'
888		txt += 'if [ $untar_status -ne 0 ]; then \n'
889	<	txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
890	<	txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n'
891	<	txt += ' echo "JobExitCode=$untar_status" \| tee -a $RUNTIME_AREA/$repo\n'
1090	<	txt += ' if [ $middleware == OSG ]; then \n'
1091	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1092	<	txt += ' cd $RUNTIME_AREA\n'
1093	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1094	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1095	<	txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1096	<	txt += ' echo "JOB_EXIT_STATUS = 50999"\n'
1097	<	txt += ' echo "JobExitCode=50999" \| tee -a $RUNTIME_AREA/$repo\n'
1098	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1099	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1100	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1101	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1102	<	txt += ' fi\n'
1103	<	txt += ' fi \n'
1104	<	txt += ' \n'
1105	<	txt += ' exit 1 \n'
889	>	txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
890	>	txt += ' job_exit_code=$untar_status\n'
891	>	txt += ' func_exit\n'
892		txt += 'else \n'
893		txt += ' echo "Successful untar" \n'
894		txt += 'fi \n'
895		txt += '\n'
896	<	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
896	>	txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
897		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
898	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1113	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1114	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1115	<	#txt += ' export PYTHONPATH=ProdAgentApi\n'
898	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
899		txt += 'else\n'
900	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1118	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1119	<	#txt += ' export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
900	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
901		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1121	–	###################
902		txt += 'fi\n'
903		txt += '\n'
904
905		pass
906	<
906	>
907	>	return txt
908	>
909	>	def wsBuildExe(self, nj=0):
910	>	"""
911	>	Put in the script the commands to build an executable
912	>	or a library.
913	>	"""
914	>
915	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
916	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
917	>
918	>	txt += 'rm -r lib/ module/ \n'
919	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
920	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
921	>	if self.dataExist == True:
922	>	txt += 'rm -r src/ \n'
923	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
924	>	if len(self.additional_inbox_files)>0:
925	>	for file in self.additional_inbox_files:
926	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
927	>	txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
928	>
929	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
930	>	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
931	>	txt += 'else\n'
932	>	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
933	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
934	>	txt += 'fi\n'
935	>	txt += '\n'
936	>
937		return txt
938
939		def modifySteeringCards(self, nj):
940		"""
941	<	modify the card provided by the user,
941	>	modify the card provided by the user,
942		writing a new card into share dir
943		"""
944	<
944	>
945		def executableName(self):
946	<	if self.scriptExe: #CarlosDaniele
946	>	if self.scriptExe:
947		return "sh "
948		else:
949		return self.executable
950
951		def executableArgs(self):
952	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
953		if self.scriptExe:#CarlosDaniele
954		return self.scriptExe + " $NJob"
955		else:
956	<	# if >= CMSSW_1_5_X, add -e
957	<	version_array = self.scram.getSWVersion().split('_')
958	<	major = 0
959	<	minor = 0
960	<	try:
961	<	major = int(version_array[1])
962	<	minor = int(version_array[2])
963	<	except:
1153	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1154	<	raise CrabException(msg)
1155	<	if major >= 1 and minor >= 5 :
1156	<	return " -e -p pset.cfg"
956	>	ex_args = ""
957	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
958	>	# Framework job report
959	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
960	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
961	>	# Type of config file
962	>	if self.CMSSW_major >= 2 :
963	>	ex_args += " -p pset.py"
964		else:
965	<	return " -p pset.cfg"
965	>	ex_args += " -p pset.cfg"
966	>	return ex_args
967
968		def inputSandbox(self, nj):
969		"""
970		Returns a list of filenames to be put in JDL input sandbox.
971		"""
972		inp_box = []
1165	–	# # dict added to delete duplicate from input sandbox file list
1166	–	# seen = {}
1167	–	## code
973		if os.path.isfile(self.tgzNameWithPath):
974		inp_box.append(self.tgzNameWithPath)
975	<	if os.path.isfile(self.MLtgzfile):
976	<	inp_box.append(self.MLtgzfile)
1172	<	## config
1173	<	if not self.pset is None:
1174	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1175	<	## additional input files
1176	<	tgz = self.additionalInputFileTgz()
1177	<	inp_box.append(tgz)
975	>	wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
976	>	inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
977		return inp_box
978
979		def outputSandbox(self, nj):
#	Line 1185 \| Line 984 \| class Cmssw(JobType):
984
985		## User Declared output files
986		for out in (self.output_file+self.output_file_sandbox):
987	<	n_out = nj + 1
987	>	n_out = nj + 1
988		out_box.append(self.numberFile_(out,str(n_out)))
989		return out_box
990
#	Line 1200 \| Line 999 \| class Cmssw(JobType):
999		Returns part of a job script which renames the produced files.
1000		"""
1001
1002	<	txt = '\n'
1003	<	txt += '# directory content\n'
1002	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1003	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1004	>	txt += 'echo ">>> current directory content:"\n'
1005		txt += 'ls \n'
1006	+	txt += '\n'
1007
1207	–	txt += 'output_exit_status=0\n'
1208	–
1209	–	for fileWithSuffix in (self.output_file_sandbox):
1210	–	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1211	–	txt += '\n'
1212	–	txt += '# check output file\n'
1213	–	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1214	–	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1215	–	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1216	–	txt += 'else\n'
1217	–	txt += ' exit_status=60302\n'
1218	–	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1219	–	if common.scheduler.boss_scheduler_name == 'condor_g':
1220	–	txt += ' if [ $middleware == OSG ]; then \n'
1221	–	txt += ' echo "prepare dummy output file"\n'
1222	–	txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1223	–	txt += ' fi \n'
1224	–	txt += 'fi\n'
1225	–
1008		for fileWithSuffix in (self.output_file):
1009		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1010		txt += '\n'
1011		txt += '# check output file\n'
1012		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1013	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1014	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1013	>	if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1014	>	txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
1015	>	txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1016	>	else:
1017	>	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1018	>	txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1019		txt += 'else\n'
1020	<	txt += ' exit_status=60302\n'
1021	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1022	<	txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n'
1237	<	txt += ' output_exit_status=$exit_status\n'
1238	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1020	>	txt += ' job_exit_code=60302\n'
1021	>	txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1022	>	if common.scheduler.name().upper() == 'CONDOR_G':
1023		txt += ' if [ $middleware == OSG ]; then \n'
1024		txt += ' echo "prepare dummy output file"\n'
1025		txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1244 \| Line 1028 \| class Cmssw(JobType):
1028		file_list = []
1029		for fileWithSuffix in (self.output_file):
1030		file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1031	<
1031	>
1032		txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1033	+	txt += '\n'
1034	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1035	+	txt += 'echo ">>> current directory content:"\n'
1036	+	txt += 'ls \n'
1037	+	txt += '\n'
1038		txt += 'cd $RUNTIME_AREA\n'
1039	+	txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1040		return txt
1041
1042		def numberFile_(self, file, txt):
#	Line 1264 \| Line 1054 \| class Cmssw(JobType):
1054		result = name + '_' + txt + "." + ext
1055		else:
1056		result = name + '_' + txt
1057	<
1057	>
1058		return result
1059
1060		def getRequirements(self, nj=[]):
1061		"""
1062	<	return job requirements to add to jdl files
1062	>	return job requirements to add to jdl files
1063		"""
1064		req = ''
1065		if self.version:
1066		req='Member("VO-cms-' + \
1067		self.version + \
1068		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1069	<	## SL add requirement for OS version only if SL4
1280	<	#reSL4 = re.compile( r'slc4' )
1281	<	if self.executable_arch: # and reSL4.search(self.executable_arch):
1069	>	if self.executable_arch:
1070		req+=' && Member("VO-cms-' + \
1071		self.executable_arch + \
1072		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1073
1074		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1075	+	if common.scheduler.name() == "glitecoll":
1076	+	req += ' && other.GlueCEStateStatus == "Production" '
1077
1078		return req
1079
1080		def configFilename(self):
1081		""" return the config filename """
1082	<	return self.name()+'.cfg'
1082	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1083	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1084	>	return self.name()+'.py'
1085	>	else:
1086	>	return self.name()+'.cfg'
1087
1294	–	### OLI_DANIELE
1088		def wsSetupCMSOSGEnvironment_(self):
1089		"""
1090		Returns part of a job script which is prepares
1091		the execution environment and which is common for all CMS jobs.
1092		"""
1093	<	txt = '\n'
1094	<	txt += ' echo "### SETUP CMS OSG ENVIRONMENT ###"\n'
1095	<	txt += ' if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1096	<	txt += ' # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1097	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1098	<	txt += ' source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1306	<	txt += ' elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1093	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1094	>	txt += ' echo ">>> setup CMS OSG environment:"\n'
1095	>	txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1096	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1097	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1098	>	txt += ' if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1099		txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1100	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1101	<	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1102	<	txt += ' else\n'
1103	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1104	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1105	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1314	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1315	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1316	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1317	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1318	<	txt += ' exit 1\n'
1319	<	txt += '\n'
1320	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1321	<	txt += ' cd $RUNTIME_AREA\n'
1322	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1323	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1324	<	txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1325	<	txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1326	<	txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1327	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1328	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1329	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1330	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1331	<	txt += ' fi\n'
1332	<	txt += '\n'
1333	<	txt += ' exit 1\n'
1334	<	txt += ' fi\n'
1100	>	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1101	>	txt += ' else\n'
1102	>	txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1103	>	txt += ' job_exit_code=10020\n'
1104	>	txt += ' func_exit\n'
1105	>	txt += ' fi\n'
1106		txt += '\n'
1107	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1108	<	txt += ' echo " END SETUP CMS OSG ENVIRONMENT "\n'
1107	>	txt += ' echo "==> setup cms environment ok"\n'
1108	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1109
1110		return txt
1111	<
1341	<	### OLI_DANIELE
1111	>
1112		def wsSetupCMSLCGEnvironment_(self):
1113		"""
1114		Returns part of a job script which is prepares
1115		the execution environment and which is common for all CMS jobs.
1116		"""
1117	<	txt = ' \n'
1118	<	txt += ' echo " ### SETUP CMS LCG ENVIRONMENT ### "\n'
1119	<	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1120	<	txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1121	<	txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1122	<	txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1123	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1124	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1125	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1126	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1127	<	txt += ' exit 1\n'
1128	<	txt += ' else\n'
1129	<	txt += ' echo "Sourcing environment... "\n'
1130	<	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1131	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1132	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1133	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1134	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1135	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1136	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1137	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1138	<	txt += ' exit 1\n'
1139	<	txt += ' fi\n'
1140	<	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1141	<	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1142	<	txt += ' result=$?\n'
1143	<	txt += ' if [ $result -ne 0 ]; then\n'
1374	<	txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1375	<	txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1376	<	txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1377	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1378	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1379	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1380	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1381	<	txt += ' exit 1\n'
1382	<	txt += ' fi\n'
1383	<	txt += ' fi\n'
1384	<	txt += ' \n'
1385	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1386	<	txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1117	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1118	>	txt += ' echo ">>> setup CMS LCG environment:"\n'
1119	>	txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1120	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1121	>	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
1122	>	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1123	>	txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1124	>	txt += ' job_exit_code=10031\n'
1125	>	txt += ' func_exit\n'
1126	>	txt += ' else\n'
1127	>	txt += ' echo "Sourcing environment... "\n'
1128	>	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1129	>	txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1130	>	txt += ' job_exit_code=10020\n'
1131	>	txt += ' func_exit\n'
1132	>	txt += ' fi\n'
1133	>	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1134	>	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1135	>	txt += ' result=$?\n'
1136	>	txt += ' if [ $result -ne 0 ]; then\n'
1137	>	txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1138	>	txt += ' job_exit_code=10032\n'
1139	>	txt += ' func_exit\n'
1140	>	txt += ' fi\n'
1141	>	txt += ' fi\n'
1142	>	txt += ' \n'
1143	>	txt += ' echo "==> setup cms environment ok"\n'
1144		return txt
1145
1389	–	### FEDE FOR DBS OUTPUT PUBLICATION
1146		def modifyReport(self, nj):
1147		"""
1148	<	insert the part of the script that modifies the FrameworkJob Report
1148	>	insert the part of the script that modifies the FrameworkJob Report
1149		"""
1150	+	txt = '\n#Written by cms_cmssw::modifyReport\n'
1151	+	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1152	+	if (publish_data == 1):
1153	+	processedDataset = self.cfg_params['USER.publish_data_name']
1154	+	LFNBaseName = LFNBase(processedDataset)
1155
1156	<	txt = ''
1157	<	try:
1158	<	publish_data = int(self.cfg_params['USER.publish_data'])
1159	<	except KeyError:
1160	<	publish_data = 0
1161	<	if (publish_data == 1):
1162	<	txt += 'echo "Modify Job Report" \n'
1402	<	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1403	<	################ FEDE FOR DBS2 #############################################
1404	<	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1405	<	#############################################################################
1406	<	#try:
1407	<	# publish_data = int(self.cfg_params['USER.publish_data'])
1408	<	#except KeyError:
1409	<	# publish_data = 0
1410	<
1411	<	txt += 'if [ -z "$SE" ]; then\n'
1412	<	txt += ' SE="" \n'
1413	<	txt += 'fi \n'
1414	<	txt += 'if [ -z "$SE_PATH" ]; then\n'
1415	<	txt += ' SE_PATH="" \n'
1416	<	txt += 'fi \n'
1417	<	txt += 'echo "SE = $SE"\n'
1418	<	txt += 'echo "SE_PATH = $SE_PATH"\n'
1156	>	txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1157	>	txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1158	>	txt += 'else\n'
1159	>	txt += ' FOR_LFN=/copy_problems/ \n'
1160	>	txt += ' SE=""\n'
1161	>	txt += ' SE_PATH=""\n'
1162	>	txt += 'fi\n'
1163
1164	<	#if (publish_data == 1):
1165	<	#processedDataset = self.cfg_params['USER.processed_datasetname']
1422	<	processedDataset = self.cfg_params['USER.publish_data_name']
1164	>	txt += 'echo ">>> Modify Job Report:" \n'
1165	>	txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1166		txt += 'ProcessedDataset='+processedDataset+'\n'
1424	–	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1425	–	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1426	–	#### FEDE: added slash in LFN ##############
1427	–	txt += ' FOR_LFN=/copy_problems/ \n'
1428	–	txt += 'else \n'
1429	–	txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1430	–	##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1431	–	txt += ' FOR_LFN=/store$tmp \n'
1432	–	txt += 'fi \n'
1167		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1168	+	txt += 'echo "SE = $SE"\n'
1169	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1170		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1171		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1172	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1173	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1438	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1439	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1440	<
1172	>	txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1173	>	txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1174		txt += 'modifyReport_result=$?\n'
1442	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1175		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1176	<	txt += ' exit_status=1\n'
1177	<	txt += ' echo "ERROR: Problem with ModifyJobReport"\n'
1176	>	txt += ' modifyReport_result=70500\n'
1177	>	txt += ' job_exit_code=$modifyReport_result\n'
1178	>	txt += ' echo "ModifyReportResult=$modifyReport_result" \| tee -a $RUNTIME_AREA/$repo\n'
1179	>	txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
1180		txt += 'else\n'
1181	<	txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1181	>	txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1182		txt += 'fi\n'
1449	–	else:
1450	–	txt += 'echo "no data publication required"\n'
1451	–	#txt += 'ProcessedDataset=no_data_to_publish \n'
1452	–	#### FEDE: added slash in LFN ##############
1453	–	#txt += 'FOR_LFN=/local/ \n'
1454	–	#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1455	–	#txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1183		return txt
1184
1185	<	def cleanEnv(self):
1186	<	### OLI_DANIELE
1187	<	txt = ''
1188	<	txt += 'if [ $middleware == OSG ]; then\n'
1189	<	txt += ' cd $RUNTIME_AREA\n'
1190	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1191	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1192	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1193	<	txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1194	<	txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1195	<	txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1196	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1197	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1198	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1199	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1185	>	def wsParseFJR(self):
1186	>	"""
1187	>	Parse the FrameworkJobReport to obtain useful infos
1188	>	"""
1189	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1190	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1191	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1192	>	txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1193	>	txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --MonitorID $MonitorID --MonitorJobID $MonitorJobID`\n'
1194	>	txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1195	>	txt += ' executable_exit_status=`echo $cmd_out \| awk -F\; "{print $1}" \| awk -F ' ' "{print $NF}"`\n'
1196	>	txt += ' if [ $executable_exit_status -eq 50115 ];then\n'
1197	>	txt += ' echo ">>> crab_fjr.xml contents: "\n'
1198	>	txt += ' cat $RUNTIME_AREA/crab_fjr_NJob.xml\n'
1199	>	txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1200	>	txt += ' else\n'
1201	>	txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1202	>	txt += ' fi\n'
1203	>	txt += ' else\n'
1204	>	txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1205		txt += ' fi\n'
1206	+	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1207	+
1208	+	if self.datasetPath:
1209	+	# VERIFY PROCESSED DATA
1210	+	txt += ' if [ $executable_exit_status -eq 0 ];then\n'
1211	+	txt += ' echo ">>> Verify list of processed files:"\n'
1212	+	txt += ' echo $InputFiles \|tr -d "\\" \|tr "," \n"\|tr -d "\"" > input-files.txt\n'
1213	+	txt += ' grep LFN $RUNTIME_AREA/crab_fjr_$NJob.xml \|cut -d">" -f2\|cut -d"<" -f1\|grep "/" > processed-files.txt\n'
1214	+	txt += ' cat input-files.txt \| sort \| uniq > tmp.txt\n'
1215	+	txt += ' mv tmp.txt input-files.txt\n'
1216	+	txt += ' echo "cat input-files.txt"\n'
1217	+	txt += ' echo "----------------------"\n'
1218	+	txt += ' cat input-files.txt\n'
1219	+	txt += ' cat processed-files.txt \| sort \| uniq > tmp.txt\n'
1220	+	txt += ' mv tmp.txt processed-files.txt\n'
1221	+	txt += ' echo "----------------------"\n'
1222	+	txt += ' echo "cat processed-files.txt"\n'
1223	+	txt += ' echo "----------------------"\n'
1224	+	txt += ' cat processed-files.txt\n'
1225	+	txt += ' echo "----------------------"\n'
1226	+	txt += ' diff -q input-files.txt processed-files.txt\n'
1227	+	txt += ' fileverify_status=$?\n'
1228	+	txt += ' if [ $fileverify_status -ne 0 ]; then\n'
1229	+	txt += ' executable_exit_status=30001\n'
1230	+	txt += ' echo "ERROR ==> not all input files processed"\n'
1231	+	txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1232	+	txt += ' echo " ==> diff input-files.txt processed-files.txt"\n'
1233	+	txt += ' fi\n'
1234	+	txt += ' fi\n'
1235	+	txt += '\n'
1236	+	txt += 'else\n'
1237	+	txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1238		txt += 'fi\n'
1239		txt += '\n'
1240	+	txt += 'echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1241	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1242	+	txt += 'job_exit_code=$executable_exit_status\n'
1243	+
1244		return txt
1245
1246		def setParam_(self, param, value):
#	Line 1481 \| Line 1249 \| class Cmssw(JobType):
1249		def getParams(self):
1250		return self._params
1251
1484	–	def setTaskid_(self):
1485	–	self._taskId = self.cfg_params['taskId']
1486	–
1487	–	def getTaskid(self):
1488	–	return self._taskId
1489	–
1252		def uniquelist(self, old):
1253		"""
1254		remove duplicates from a list
#	Line 1496 \| Line 1258 \| class Cmssw(JobType):
1258		nd[e]=0
1259		return nd.keys()
1260
1261	<
1500	<	def checkOut(self, limit):
1261	>	def outList(self):
1262		"""
1263		check the dimension of the output files
1264		"""
1265	<	txt = 'echo "*****************************************"\n'
1266	<	txt += 'echo " Starting output sandbox limit check "\n'
1506	<	txt += 'echo "*****************************************"\n'
1507	<	allOutFiles = ""
1265	>	txt = ''
1266	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1267		listOutFiles = []
1268	<	for fileOut in (self.output_file+self.output_file_sandbox):
1269	<	if fileOut.find('crab_fjr') == -1:
1270	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1271	<	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1272	<	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1273	<	txt += 'ls -gGhrta;\n'
1274	<	txt += 'sum=0;\n'
1275	<	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1276	<	txt += ' if [ -e $file ]; then\n'
1277	<	txt += ' tt=`ls -gGrta $file \| awk \'{ print $3 }\'`\n'
1278	<	txt += ' sum=`expr $sum + $tt`\n'
1279	<	txt += ' else\n'
1280	<	txt += ' echo "WARNING: output file $file not found!"\n'
1281	<	txt += ' fi\n'
1282	<	txt += 'done\n'
1524	<	txt += 'echo "Total Output dimension: $sum";\n'
1525	<	txt += 'limit='+str(limit)+';\n'
1526	<	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1527	<	txt += 'if [ $limit -lt $sum ]; then\n'
1528	<	txt += ' echo "WARNING: output files have to big size - something will be lost;"\n'
1529	<	txt += ' echo " checking the output file sizes..."\n'
1530	<	"""
1531	<	txt += ' dim=0;\n'
1532	<	txt += ' exclude=0;\n'
1533	<	txt += ' for files in '+str(allOutFiles)+' ; do\n'
1534	<	txt += ' sumTemp=0;\n'
1535	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1536	<	txt += ' if [ $file != $file2 ]; then\n'
1537	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1538	<	txt += ' sumTemp=`expr $sumTemp + $tt`;\n'
1539	<	txt += ' fi\n'
1540	<	txt += ' done\n'
1541	<	txt += ' if [ $sumTemp -lt $limit ]; then\n'
1542	<	txt += ' if [ $dim -lt $sumTemp ]; then\n'
1543	<	txt += ' dim=$sumTemp;\n'
1544	<	txt += ' exclude=$file;\n'
1545	<	txt += ' fi\n'
1546	<	txt += ' fi\n'
1547	<	txt += ' done\n'
1548	<	txt += ' echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1549	<	"""
1550	<	txt += ' tot=0;\n'
1551	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1552	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1553	<	txt += ' tot=`expr $tot + $tt`;\n'
1554	<	txt += ' if [ $limit -lt $tot ]; then\n'
1555	<	txt += ' tot=`expr $tot - $tt`;\n'
1556	<	txt += ' fileLast=$file;\n'
1557	<	txt += ' break;\n'
1558	<	txt += ' fi\n'
1559	<	txt += ' done\n'
1560	<	txt += ' echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1561	<	txt += ' flag=0;\n'
1562	<	txt += ' for filess in '+str(allOutFiles)+' ; do\n'
1563	<	txt += ' if [ $fileLast = $filess ]; then\n'
1564	<	txt += ' flag=1;\n'
1565	<	txt += ' fi\n'
1566	<	txt += ' if [ $flag -eq 1 ]; then\n'
1567	<	txt += ' rm -f $filess;\n'
1568	<	txt += ' fi\n'
1569	<	txt += ' done\n'
1570	<	txt += ' ls -agGhrt;\n'
1571	<	txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1572	<	txt += ' echo "JOB_EXIT_STATUS = 70000";\n'
1573	<	txt += ' exit_status=70000;\n'
1574	<	txt += 'else'
1575	<	txt += ' echo "Total Output dimension $sum is fine.";\n'
1576	<	txt += 'fi\n'
1577	<	txt += 'echo "*****************************************"\n'
1578	<	txt += 'echo "* Ending output sandbox limit check *"\n'
1579	<	txt += 'echo "*****************************************"\n'
1268	>	stdout = 'CMSSW_$NJob.stdout'
1269	>	stderr = 'CMSSW_$NJob.stderr'
1270	>	if (self.return_data == 1):
1271	>	for file in (self.output_file+self.output_file_sandbox):
1272	>	listOutFiles.append(self.numberFile_(file, '$NJob'))
1273	>	listOutFiles.append(stdout)
1274	>	listOutFiles.append(stderr)
1275	>	else:
1276	>	for file in (self.output_file_sandbox):
1277	>	listOutFiles.append(self.numberFile_(file, '$NJob'))
1278	>	listOutFiles.append(stdout)
1279	>	listOutFiles.append(stderr)
1280	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1281	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1282	>	txt += 'export filesToCheck\n'
1283		return txt

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing COMP/CRAB/python/cms_cmssw.py (file contents): Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs. Revision 1.191 by spiga, Tue May 27 22:14:26 2008 UTC

Diff Legend

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs.
Revision 1.191 by spiga, Tue May 27 22:14:26 2008 UTC