ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.112 by corvo, Thu Aug 9 16:53:21 2007 UTC vs.
Revision 1.191 by spiga, Tue May 27 22:14:26 2008 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from BlackWhiteListParser import BlackWhiteListParser
6		import common
7		import Scram
8	+	from LFNBaseName import *
9
10		import os, string, glob
11
#	Line 12 | Line 14 | class Cmssw(JobType):
14		JobType.__init__(self, 'CMSSW')
15		common.logger.debug(3,'CMSSW::__init__')
16
17	+	self.argsList = []
18	+
19		self._params = {}
20		self.cfg_params = cfg_params
21	+	# init BlackWhiteListParser
22	+	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23
24	<	try:
19	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
20	<	except KeyError:
21	<	self.MaxTarBallSize = 9.5
24	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',9.5))
25
26		# number of jobs requested to be created, limit obj splitting
27		self.ncjobs = ncjobs
28
29		log = common.logger
30	<
30	>
31		self.scram = Scram.Scram(cfg_params)
32		self.additional_inbox_files = []
33		self.scriptExe = ''
34		self.executable = ''
35		self.executable_arch = self.scram.getArch()
36		self.tgz_name = 'default.tgz'
34	–	self.additional_tgz_name = 'additional.tgz'
37		self.scriptName = 'CMSSW.sh'
38	<	self.pset = ''      #scrip use case Da
39	<	self.datasetPath = '' #scrip use case Da
38	>	self.pset = ''
39	>	self.datasetPath = ''
40
41		# set FJR file name
42		self.fjrFileName = 'crab_fjr.xml'
43
44		self.version = self.scram.getSWVersion()
45	<
46	<	#
47	<	# Try to block creation in case of arch/version mismatch
48	<	#
49	<
50	<	a = string.split(self.version, "_")
51	<
52	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
53	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
54	<	raise CrabException(msg)
53	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
54	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
45	>	version_array = self.version.split('_')
46	>	self.CMSSW_major = 0
47	>	self.CMSSW_minor = 0
48	>	self.CMSSW_patch = 0
49	>	try:
50	>	self.CMSSW_major = int(version_array[1])
51	>	self.CMSSW_minor = int(version_array[2])
52	>	self.CMSSW_patch = int(version_array[3])
53	>	except:
54	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
55		raise CrabException(msg)
56	–
57	–	common.taskDB.setDict('codeVersion',self.version)
58	–	self.setParam_('application', self.version)
56
57		### collect Data cards
58
59	<	## get DBS mode
60	<	try:
64	<	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
65	<	except KeyError:
66	<	self.use_dbs_1 = 0
67	<
68	<	try:
69	<	tmp =  cfg_params['CMSSW.datasetpath']
70	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
71	<	if string.lower(tmp)=='none':
72	<	self.datasetPath = None
73	<	self.selectNoInput = 1
74	<	else:
75	<	self.datasetPath = tmp
76	<	self.selectNoInput = 0
77	<	except KeyError:
78	<	msg = "Error: datasetpath not defined "
59	>	if not cfg_params.has_key('CMSSW.datasetpath'):
60	>	msg = "Error: datasetpath not defined "
61		raise CrabException(msg)
62	<
63	<	# ML monitoring
64	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
65	<	if not self.datasetPath:
66	<	self.setParam_('dataset', 'None')
85	<	self.setParam_('owner', 'None')
62	>	tmp =  cfg_params['CMSSW.datasetpath']
63	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
64	>	if string.lower(tmp)=='none':
65	>	self.datasetPath = None
66	>	self.selectNoInput = 1
67		else:
68	<	try:
69	<	datasetpath_split = self.datasetPath.split("/")
89	<	# standard style
90	<	if self.use_dbs_1 == 1 :
91	<	self.setParam_('dataset', datasetpath_split[1])
92	<	self.setParam_('owner', datasetpath_split[-1])
93	<	else:
94	<	self.setParam_('dataset', datasetpath_split[1])
95	<	self.setParam_('owner', datasetpath_split[2])
96	<	except:
97	<	self.setParam_('dataset', self.datasetPath)
98	<	self.setParam_('owner', self.datasetPath)
99	<
100	<	self.setTaskid_()
101	<	self.setParam_('taskId', self.cfg_params['taskId'])
68	>	self.datasetPath = tmp
69	>	self.selectNoInput = 0
70
71		self.dataTiers = []
72	+
73	+	self.debug_pset = cfg_params.get('USER.debug_pset',False)
74
75		## now the application
76	<	try:
77	<	self.executable = cfg_params['CMSSW.executable']
108	<	self.setParam_('exe', self.executable)
109	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
110	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
111	<	log.debug(3,msg)
112	<	except KeyError:
113	<	self.executable = 'cmsRun'
114	<	self.setParam_('exe', self.executable)
115	<	msg = "User executable not defined. Use cmsRun"
116	<	log.debug(3,msg)
117	<	pass
76	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
77	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
78
79	<	try:
120	<	self.pset = cfg_params['CMSSW.pset']
121	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
122	<	if self.pset.lower() != 'none' :
123	<	if (not os.path.exists(self.pset)):
124	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
125	<	else:
126	<	self.pset = None
127	<	except KeyError:
79	>	if not cfg_params.has_key('CMSSW.pset'):
80		raise CrabException("PSet file missing. Cannot run cmsRun ")
81	+	self.pset = cfg_params['CMSSW.pset']
82	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
83	+	if self.pset.lower() != 'none' :
84	+	if (not os.path.exists(self.pset)):
85	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
86	+	else:
87	+	self.pset = None
88
89		# output files
90		## stuff which must be returned always via sandbox
#	Line 135 | Line 94 | class Cmssw(JobType):
94		self.output_file_sandbox.append(self.fjrFileName)
95
96		# other output files to be returned via sandbox or copied to SE
97	<	try:
98	<	self.output_file = []
99	<	tmp = cfg_params['CMSSW.output_file']
100	<	if tmp != '':
101	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
102	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
103	<	for tmp in tmpOutFiles:
104	<	tmp=string.strip(tmp)
146	<	self.output_file.append(tmp)
147	<	pass
148	<	else:
149	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
97	>	self.output_file = []
98	>	tmp = cfg_params.get('CMSSW.output_file',None)
99	>	if tmp :
100	>	tmpOutFiles = string.split(tmp,',')
101	>	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
102	>	for tmp in tmpOutFiles:
103	>	tmp=string.strip(tmp)
104	>	self.output_file.append(tmp)
105		pass
106	<	pass
152	<	except KeyError:
106	>	else:
107		log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
108	<	pass
108	>	pass
109
110		# script_exe file as additional file in inputSandbox
111	<	try:
112	<	self.scriptExe = cfg_params['USER.script_exe']
113	<	if self.scriptExe != '':
114	<	if not os.path.isfile(self.scriptExe):
115	<	msg ="ERROR. file "+self.scriptExe+" not found"
116	<	raise CrabException(msg)
163	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
164	<	except KeyError:
165	<	self.scriptExe = ''
111	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
112	>	if self.scriptExe :
113	>	if not os.path.isfile(self.scriptExe):
114	>	msg ="ERROR. file "+self.scriptExe+" not found"
115	>	raise CrabException(msg)
116	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
117
167	–	#CarlosDaniele
118		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
119	<	msg ="Error. script_exe  not defined"
120	<	raise CrabException(msg)
119	>	msg ="Error. script_exe  not defined"
120	>	raise CrabException(msg)
121
122		## additional input files
123	<	try:
123	>	if cfg_params.has_key('USER.additional_input_files'):
124		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
125		for tmp in tmpAddFiles:
126		tmp = string.strip(tmp)
#	Line 187 | Line 137 | class Cmssw(JobType):
137		if not os.path.exists(file):
138		raise CrabException("Additional input file not found: "+file)
139		pass
190	–	# fname = string.split(file, '/')[-1]
191	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
192	–	# shutil.copyfile(file, storedFile)
140		self.additional_inbox_files.append(string.strip(file))
141		pass
142		pass
143		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
144	<	except KeyError:
198	<	pass
199	<
200	<	# files per job
201	<	try:
202	<	if (cfg_params['CMSSW.files_per_jobs']):
203	<	raise CrabException("files_per_jobs no longer supported.  Quitting.")
204	<	except KeyError:
205	<	pass
144	>	pass
145
146		## Events per job
147	<	try:
147	>	if cfg_params.has_key('CMSSW.events_per_job'):
148		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
149		self.selectEventsPerJob = 1
150	<	except KeyError:
150	>	else:
151		self.eventsPerJob = -1
152		self.selectEventsPerJob = 0
153	<
153	>
154		## number of jobs
155	<	try:
155	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
156		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
157		self.selectNumberOfJobs = 1
158	<	except KeyError:
158	>	else:
159		self.theNumberOfJobs = 0
160		self.selectNumberOfJobs = 0
161
162	<	try:
162	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
163		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
164		self.selectTotalNumberEvents = 1
165	<	except KeyError:
165	>	else:
166		self.total_number_of_events = 0
167		self.selectTotalNumberEvents = 0
168
169	<	if self.pset != None: #CarlosDaniele
169	>	if self.pset != None:
170		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
171		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
172		raise CrabException(msg)
#	Line 236 | Line 175 | class Cmssw(JobType):
175		msg = 'Must specify  number_of_jobs.'
176		raise CrabException(msg)
177
178	<	## source seed for pythia
179	<	try:
180	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
181	<	except KeyError:
182	<	self.sourceSeed = None
183	<	common.logger.debug(5,"No seed given")
178	>	## New method of dealing with seeds
179	>	self.incrementSeeds = []
180	>	self.preserveSeeds = []
181	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
182	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
183	>	for tmp in tmpList:
184	>	tmp.strip()
185	>	self.preserveSeeds.append(tmp)
186	>	if cfg_params.has_key('CMSSW.increment_seeds'):
187	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
188	>	for tmp in tmpList:
189	>	tmp.strip()
190	>	self.incrementSeeds.append(tmp)
191	>
192	>	## Old method of dealing with seeds
193	>	## FUTURE: This is for old CMSSW and old CRAB. Can throw exceptions after a couple of CRAB releases and then
194	>	## remove
195	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
196	>	if self.sourceSeed:
197	>	print "pythia_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
198	>	self.incrementSeeds.append('sourceSeed')
199	>	self.incrementSeeds.append('theSource')
200	>
201	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
202	>	if self.sourceSeedVtx:
203	>	print "vtx_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
204	>	self.incrementSeeds.append('VtxSmeared')
205	>
206	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
207	>	if self.sourceSeedG4:
208	>	print "g4_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
209	>	self.incrementSeeds.append('g4SimHits')
210	>
211	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
212	>	if self.sourceSeedMix:
213	>	print "mix_seed is a deprecated parameter. Use preserve_seeds or increment_seeds in the future.\n","Added to increment_seeds."
214	>	self.incrementSeeds.append('mix')
215
216	<	try:
247	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
248	<	except KeyError:
249	<	self.sourceSeedVtx = None
250	<	common.logger.debug(5,"No vertex seed given")
216	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
217
252	–	try:
253	–	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
254	–	except KeyError:
255	–	self.sourceSeedG4 = None
256	–	common.logger.debug(5,"No g4 sim hits seed given")
257	–
258	–	try:
259	–	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
260	–	except KeyError:
261	–	self.sourceSeedMix = None
262	–	common.logger.debug(5,"No mix seed given")
263	–
264	–	try:
265	–	self.firstRun = int(cfg_params['CMSSW.first_run'])
266	–	except KeyError:
267	–	self.firstRun = None
268	–	common.logger.debug(5,"No first run given")
218		if self.pset != None: #CarlosDaniele
219	<	ver = string.split(self.version,"_")
271	<	if (int(ver[1])>=1 and int(ver[2])>=5):
272	<	import PsetManipulator150 as pp
273	<	else:
274	<	import PsetManipulator as pp
219	>	import PsetManipulator as pp
220		PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
221
222	+	# Copy/return
223	+
224	+	self.copy_data = int(cfg_params.get('USER.copy_data',0))
225	+	self.return_data = int(cfg_params.get('USER.return_data',0))
226	+
227		#DBSDLS-start
228	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
228	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
229		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
230		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
231		self.jobDestination=[]  # Site destination(s) for each job (list of lists)
#	Line 284 | Line 234 | class Cmssw(JobType):
234		blockSites = {}
235		if self.datasetPath:
236		blockSites = self.DataDiscoveryAndLocation(cfg_params)
237	<	#DBSDLS-end
237	>	#DBSDLS-end
238	>
239
289	–	self.tgzNameWithPath = self.getTarBall(self.executable)
290	–
240		## Select Splitting
241	<	if self.selectNoInput:
242	<	if self.pset == None: #CarlosDaniele
241	>	if self.selectNoInput:
242	>	if self.pset == None:
243		self.jobSplittingForScript()
244		else:
245		self.jobSplittingNoInput()
#	Line 298 | Line 247 | class Cmssw(JobType):
247		self.jobSplittingByBlocks(blockSites)
248
249		# modify Pset
250	<	if self.pset != None: #CarlosDaniele
250	>	if self.pset != None:
251		try:
252	<	if (self.datasetPath): # standard job
253	<	# allow to processa a fraction of events in a file
254	<	PsetEdit.inputModule("INPUT")
255	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
307	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
308	<	else:  # pythia like job
309	<	PsetEdit.maxEvent(self.eventsPerJob)
310	<	if (self.firstRun):
311	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
312	<	if (self.sourceSeed) :
313	<	PsetEdit.pythiaSeed("INPUT")
314	<	if (self.sourceSeedVtx) :
315	<	PsetEdit.vtxSeed("INPUTVTX")
316	<	if (self.sourceSeedG4) :
317	<	self.PsetEdit.g4Seed("INPUTG4")
318	<	if (self.sourceSeedMix) :
319	<	self.PsetEdit.mixSeed("INPUTMIX")
320	<	# add FrameworkJobReport to parameter-set
321	<	PsetEdit.addCrabFJR(self.fjrFileName)
252	>	# Add FrameworkJobReport to parameter-set, set max events.
253	>	# Reset later for data jobs by writeCFG which does all modifications
254	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
255	>	PsetEdit.maxEvent(self.eventsPerJob)
256		PsetEdit.psetWriter(self.configFilename())
257		except:
258	<	msg='Error while manipuliating ParameterSet: exiting...'
258	>	msg='Error while manipulating ParameterSet: exiting...'
259		raise CrabException(msg)
260	+	self.tgzNameWithPath = self.getTarBall(self.executable)
261
262		def DataDiscoveryAndLocation(self, cfg_params):
263
264		import DataDiscovery
330	–	import DataDiscovery_DBS2
265		import DataLocation
266		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
267
#	Line 336 | Line 270 | class Cmssw(JobType):
270		## Contact the DBS
271		common.logger.message("Contacting Data Discovery Services ...")
272		try:
273	<
340	<	if self.use_dbs_1 == 1 :
341	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
342	<	else :
343	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
273	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
274		self.pubdata.fetchDBSInfo()
275
276		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 352 | Line 282 | class Cmssw(JobType):
282		except DataDiscovery.DataDiscoveryError, ex:
283		msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
284		raise CrabException(msg)
355	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
356	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
357	–	raise CrabException(msg)
358	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
359	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
360	–	raise CrabException(msg)
361	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
362	–	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
363	–	raise CrabException(msg)
285
286		self.filesbyblock=self.pubdata.getFiles()
287		self.eventsbyblock=self.pubdata.getEventsPerBlock()
288		self.eventsbyfile=self.pubdata.getEventsPerFile()
289
290		## get max number of events
291	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
291	>	self.maxEvents=self.pubdata.getMaxEvents()
292
293		## Contact the DLS and build a list of sites hosting the fileblocks
294		try:
#	Line 376 | Line 297 | class Cmssw(JobType):
297		except DataLocation.DataLocationError , ex:
298		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
299		raise CrabException(msg)
300	<
300	>
301
302		sites = dataloc.getSites()
303		allSites = []
#	Line 390 | Line 311 | class Cmssw(JobType):
311		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
312
313		return sites
314	<
314	>
315		def jobSplittingByBlocks(self, blockSites):
316		"""
317		Perform job splitting. Jobs run over an integer number of files
#	Line 440 | Line 361 | class Cmssw(JobType):
361		totalNumberOfJobs = 999999999
362		else :
363		totalNumberOfJobs = self.ncjobs
443	–
364
365		blocks = blockSites.keys()
366		blockCount = 0
#	Line 460 | Line 380 | class Cmssw(JobType):
380		blockCount += 1
381		if block not in jobsOfBlock.keys() :
382		jobsOfBlock[block] = []
383	<
383	>
384		if self.eventsbyblock.has_key(block) :
385		numEventsInBlock = self.eventsbyblock[block]
386		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
387	<
387	>
388		files = self.filesbyblock[block]
389		numFilesInBlock = len(files)
390		if (numFilesInBlock <= 0):
#	Line 472 | Line 392 | class Cmssw(JobType):
392		fileCount = 0
393
394		# ---- New block => New job ---- #
395	<	parString = "\\{"
395	>	parString = ""
396		# counter for number of events in files currently worked on
397		filesEventCount = 0
398		# flag if next while loop should touch new file
399		newFile = 1
400		# job event counter
401		jobSkipEventCount = 0
402	<
402	>
403		# ---- Iterate over the files in the block until we've met the requested ---- #
404		# ---- total # of events or we've gone over all the files in this block  ---- #
405		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
#	Line 495 | Line 415 | class Cmssw(JobType):
415		newFile = 0
416		except KeyError:
417		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
498	–
418
419	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
420		# if less events in file remain than eventsPerJobRequested
421	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
421	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
422		# if last file in block
423		if ( fileCount == numFilesInBlock-1 ) :
424		# end job using last file, use remaining events in block
425		# close job and touch new file
426		fullString = parString[:-2]
507	–	fullString += '\\}'
427		list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
428		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
429		self.jobDestination.append(blockSites[block])
#	Line 517 | Line 436 | class Cmssw(JobType):
436		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
437		jobSkipEventCount = 0
438		# reset file
439	<	parString = "\\{"
439	>	parString = ""
440		filesEventCount = 0
441		newFile = 1
442		fileCount += 1
#	Line 529 | Line 448 | class Cmssw(JobType):
448		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
449		# close job and touch new file
450		fullString = parString[:-2]
532	–	fullString += '\\}'
451		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
452		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
453		self.jobDestination.append(blockSites[block])
#	Line 541 | Line 459 | class Cmssw(JobType):
459		eventsRemaining = eventsRemaining - eventsPerJobRequested
460		jobSkipEventCount = 0
461		# reset file
462	<	parString = "\\{"
462	>	parString = ""
463		filesEventCount = 0
464		newFile = 1
465		fileCount += 1
466	<
466	>
467		# if more events in file remain than eventsPerJobRequested
468		else :
469		# close job but don't touch new file
470		fullString = parString[:-2]
553	–	fullString += '\\}'
471		list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
472		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
473		self.jobDestination.append(blockSites[block])
#	Line 565 | Line 482 | class Cmssw(JobType):
482		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
483		# remove all but the last file
484		filesEventCount = self.eventsbyfile[file]
485	<	parString = "\\{"
569	<	parString += '\\\"' + file + '\\\"\,'
485	>	parString = '\\\"' + file + '\\\"\,'
486		pass # END if
487		pass # END while (iterate over files in the block)
488		pass # END while (iterate over blocks in the dataset)
#	Line 574 | Line 490 | class Cmssw(JobType):
490		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
491		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
492		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
493	<
493	>
494		# screen output
495		screenOutput = "List of jobs and available destination sites:\n\n"
496
497	+	# keep trace of block with no sites to print a warning at the end
498	+	noSiteBlock = []
499	+	bloskNoSite = []
500	+
501		blockCounter = 0
502		for block in blocks:
503		if block in jobsOfBlock.keys() :
504		blockCounter += 1
505	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(blockSites[block]))
505	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
506	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
507	>	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
508	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
509	>	bloskNoSite.append( blockCounter )
510	>
511	>	common.logger.message(screenOutput)
512	>	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
513	>	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
514	>	virgola = ""
515	>	if len(bloskNoSite) > 1:
516	>	virgola = ","
517	>	for block in bloskNoSite:
518	>	msg += ' ' + str(block) + virgola
519	>	msg += '\n               Related jobs:\n                 '
520	>	virgola = ""
521	>	if len(noSiteBlock) > 1:
522	>	virgola = ","
523	>	for range_jobs in noSiteBlock:
524	>	msg += str(range_jobs) + virgola
525	>	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
526	>	if self.cfg_params.has_key('EDG.se_white_list'):
527	>	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
528	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
529	>	msg += 'Please check if the dataset is available at this site!)\n'
530	>	if self.cfg_params.has_key('EDG.ce_white_list'):
531	>	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
532	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
533	>	msg += 'Please check if the dataset is available at this site!)\n'
534
535	<	# common.logger.message(screenOutput)
535	>	common.logger.message(msg)
536
537		self.list_of_args = list_of_lists
538		return
#	Line 594 | Line 542 | class Cmssw(JobType):
542		Perform job splitting based on number of event per job
543		"""
544		common.logger.debug(5,'Splitting per events')
545	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
546	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
547	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
545	>
546	>	if (self.selectEventsPerJob):
547	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
548	>	if (self.selectNumberOfJobs):
549	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
550	>	if (self.selectTotalNumberEvents):
551	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
552
553		if (self.total_number_of_events < 0):
554		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 605 | Line 557 | class Cmssw(JobType):
557		if (self.selectEventsPerJob):
558		if (self.selectTotalNumberEvents):
559		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
560	<	elif(self.selectNumberOfJobs) :
560	>	elif(self.selectNumberOfJobs) :
561		self.total_number_of_jobs =self.theNumberOfJobs
562	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
562	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
563
564		elif (self.selectNumberOfJobs) :
565		self.total_number_of_jobs = self.theNumberOfJobs
566		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
567	<
567	>
568		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
569
570		# is there any remainder?
#	Line 628 | Line 580 | class Cmssw(JobType):
580		self.list_of_args = []
581		for i in range(self.total_number_of_jobs):
582		## Since there is no input, any site is good
583	<	# self.jobDestination.append(["Any"])
632	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
583	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
584		args=[]
585		if (self.firstRun):
586	<	## pythia first run
636	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
586	>	## pythia first run
587		args.append(str(self.firstRun)+str(i))
638	–	else:
639	–	## no first run
640	–	#self.list_of_args.append([str(i)])
641	–	args.append(str(i))
642	–	if (self.sourceSeed):
643	–	args.append(str(self.sourceSeed)+str(i))
644	–	if (self.sourceSeedVtx):
645	–	## + vtx random seed
646	–	args.append(str(self.sourceSeedVtx)+str(i))
647	–	if (self.sourceSeedG4):
648	–	## + G4 random seed
649	–	args.append(str(self.sourceSeedG4)+str(i))
650	–	if (self.sourceSeedMix):
651	–	## + Mix random seed
652	–	args.append(str(self.sourceSeedMix)+str(i))
653	–	pass
654	–	pass
588		self.list_of_args.append(args)
656	–	pass
657	–
658	–	# print self.list_of_args
589
590		return
591
592
593	<	def jobSplittingForScript(self):#CarlosDaniele
593	>	def jobSplittingForScript(self):
594		"""
595		Perform job splitting based on number of job
596		"""
#	Line 676 | Line 606 | class Cmssw(JobType):
606		# argument is seed number.$i
607		self.list_of_args = []
608		for i in range(self.total_number_of_jobs):
679	–	## Since there is no input, any site is good
680	–	# self.jobDestination.append(["Any"])
609		self.jobDestination.append([""])
682	–	## no random seed
610		self.list_of_args.append([str(i)])
611		return
612
613		def split(self, jobParams):
614	<
688	<	common.jobDB.load()
689	<	#### Fabio
614	>
615		njobs = self.total_number_of_jobs
616		arglist = self.list_of_args
617		# create the empty structure
618		for i in range(njobs):
619		jobParams.append("")
620	<
620	>
621	>	listID=[]
622	>	listField=[]
623		for job in range(njobs):
624		jobParams[job] = arglist[job]
625	<	# print str(arglist[job])
626	<	# print jobParams[job]
627	<	common.jobDB.setArguments(job, jobParams[job])
628	<	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
629	<	common.jobDB.setDestination(job, self.jobDestination[job])
625	>	listID.append(job+1)
626	>	job_ToSave ={}
627	>	concString = ' '
628	>	argu=''
629	>	if len(jobParams[job]):
630	>	argu +=   concString.join(jobParams[job] )
631	>	job_ToSave['arguments']= str(job+1)+' '+argu
632	>	job_ToSave['dlsDestination']= self.jobDestination[job]
633	>	listField.append(job_ToSave)
634	>	msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
635	>	+"                     Destination: "+str(self.jobDestination[job])
636	>	common.logger.debug(5,msg)
637	>	common._db.updateJob_(listID,listField)
638	>	self.argsList = (len(jobParams[0])+1)
639
704	–	common.jobDB.save()
640		return
641	<
707	<	def getJobTypeArguments(self, nj, sched):
708	<	result = ''
709	<	for i in common.jobDB.arguments(nj):
710	<	result=result+str(i)+" "
711	<	return result
712	<
641	>
642		def numberOfJobs(self):
714	–	# Fabio
643		return self.total_number_of_jobs
644
645		def getTarBall(self, exe):
646		"""
647		Return the TarBall with lib and exe
648		"""
721	–
722	–	# if it exist, just return it
723	–	#
724	–	# Marco. Let's start to use relative path for Boss XML files
725	–	#
649		self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
650		if os.path.exists(self.tgzNameWithPath):
651		return self.tgzNameWithPath
#	Line 736 | Line 659 | class Cmssw(JobType):
659
660		# First of all declare the user Scram area
661		swArea = self.scram.getSWArea_()
739	–	#print "swArea = ", swArea
740	–	# swVersion = self.scram.getSWVersion()
741	–	# print "swVersion = ", swVersion
662		swReleaseTop = self.scram.getReleaseTop_()
663	<	#print "swReleaseTop = ", swReleaseTop
744	<
663	>
664		## check if working area is release top
665		if swReleaseTop == '' or swArea == swReleaseTop:
666	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
667		return
668
669		import tarfile
#	Line 754 | Line 674 | class Cmssw(JobType):
674		exeWithPath = self.scram.findFile_(executable)
675		if ( not exeWithPath ):
676		raise CrabException('User executable '+executable+' not found')
677	<
677	>
678		## then check if it's private or not
679		if exeWithPath.find(swReleaseTop) == -1:
680		# the exe is private, so we must ship
#	Line 763 | Line 683 | class Cmssw(JobType):
683		# distinguish case when script is in user project area or given by full path somewhere else
684		if exeWithPath.find(path) >= 0 :
685		exe = string.replace(exeWithPath, path,'')
686	<	tar.add(path+exe,os.path.basename(executable))
686	>	tar.add(path+exe,exe)
687		else :
688		tar.add(exeWithPath,os.path.basename(executable))
689		pass
690		else:
691		# the exe is from release, we'll find it on WN
692		pass
693	<
693	>
694		## Now get the libraries: only those in local working area
695		libDir = 'lib'
696		lib = swArea+'/' +libDir
697		common.logger.debug(5,"lib "+lib+" to be tarred")
698		if os.path.exists(lib):
699		tar.add(lib,libDir)
700	<
700	>
701		## Now check if module dir is present
702		moduleDir = 'module'
703		module = swArea + '/' + moduleDir
#	Line 786 | Line 706 | class Cmssw(JobType):
706
707		## Now check if any data dir(s) is present
708		swAreaLen=len(swArea)
709	+	self.dataExist = False
710		for root, dirs, files in os.walk(swArea):
711		if "data" in dirs:
712	+	self.dataExist=True
713		common.logger.debug(5,"data "+root+"/data"+" to be tarred")
714		tar.add(root+"/data",root[swAreaLen:]+"/data")
715
716	<	## Add ProdAgent dir to tar
717	<	paDir = 'ProdAgentApi'
718	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
719	<	if os.path.isdir(pa):
720	<	tar.add(pa,paDir)
716	>	### CMSSW ParameterSet
717	>	if not self.pset is None:
718	>	cfg_file = common.work_space.jobDir()+self.configFilename()
719	>	tar.add(cfg_file,self.configFilename())
720	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
721	>
722
723	<	### FEDE FOR DBS PUBLICATION
801	<	## Add PRODCOMMON dir to tar
723	>	## Add ProdCommon dir to tar
724		prodcommonDir = 'ProdCommon'
725		prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
726		if os.path.isdir(prodcommonPath):
727		tar.add(prodcommonPath,prodcommonDir)
806	–	#############################
807	–
728		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
729	+
730	+	##### ML stuff
731	+	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
732	+	path=os.environ['CRABDIR'] + '/python/'
733	+	for file in ML_file_list:
734	+	tar.add(path+file,file)
735	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
736	+
737	+	##### Utils
738	+	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'JobReportErrorCode.py']
739	+	for file in Utils_file_list:
740	+	tar.add(path+file,file)
741	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
742	+
743	+	##### AdditionalFiles
744	+	for file in self.additional_inbox_files:
745	+	tar.add(file,string.split(file,'/')[-1])
746	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
747	+
748		tar.close()
749		except :
750		raise CrabException('Could not create tar-ball')
#	Line 816 | Line 755 | class Cmssw(JobType):
755		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
756
757		## create tar-ball with ML stuff
819	–	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
820	–	try:
821	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
822	–	path=os.environ['CRABDIR'] + '/python/'
823	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
824	–	tar.add(path+file,file)
825	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
826	–	tar.close()
827	–	except :
828	–	raise CrabException('Could not create ML files tar-ball')
829	–
830	–	return
831	–
832	–	def additionalInputFileTgz(self):
833	–	"""
834	–	Put all additional files into a tar ball and return its name
835	–	"""
836	–	import tarfile
837	–	tarName=  common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
838	–	tar = tarfile.open(tarName, "w:gz")
839	–	for file in self.additional_inbox_files:
840	–	tar.add(file,string.split(file,'/')[-1])
841	–	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
842	–	tar.close()
843	–	return tarName
758
759	<	def wsSetupEnvironment(self, nj):
759	>	def wsSetupEnvironment(self, nj=0):
760		"""
761		Returns part of a job script which prepares
762		the execution environment for the job 'nj'.
763		"""
764	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
765	+	psetName = 'pset.py'
766	+	else:
767	+	psetName = 'pset.cfg'
768		# Prepare JobType-independent part
769	<	txt = ''
770	<
771	<	## OLI_Daniele at this level  middleware already known
854	<
855	<	txt += 'echo "### Firtst set SCRAM ARCH and BUILD_ARCH ###"\n'
856	<	txt += 'echo "Setting SCRAM_ARCH='+self.executable_arch+'"\n'
857	<	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
858	<	txt += 'export BUILD_ARCH='+self.executable_arch+'\n'
859	<	txt += 'if [ $middleware == LCG ]; then \n'
769	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
770	>	txt += 'echo ">>> setup environment"\n'
771	>	txt += 'if [ $middleware == LCG ]; then \n'
772		txt += self.wsSetupCMSLCGEnvironment_()
773		txt += 'elif [ $middleware == OSG ]; then\n'
774		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
775	<	txt += '    echo "Created working directory: $WORKING_DIR"\n'
776	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
777	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
778	<	txt += '    echo "JOB_EXIT_STATUS = 10016"\n'
867	<	txt += '    echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
868	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
869	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
870	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
871	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
872	<	txt += '        exit 1\n'
775	>	txt += '    if [ ! $? == 0 ] ;then\n'
776	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
777	>	txt += '        job_exit_code=10016\n'
778	>	txt += '        func_exit\n'
779		txt += '    fi\n'
780	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
781		txt += '\n'
782		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
783		txt += '    cd $WORKING_DIR\n'
784	<	txt += self.wsSetupCMSOSGEnvironment_()
784	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
785	>	txt += self.wsSetupCMSOSGEnvironment_()
786		txt += 'fi\n'
787
788		# Prepare JobType-specific part
789		scram = self.scram.commandName()
790		txt += '\n\n'
791	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
791	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
792	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
793		txt += scram+' project CMSSW '+self.version+'\n'
794		txt += 'status=$?\n'
795		txt += 'if [ $status != 0 ] ; then\n'
796	<	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
797	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
798	<	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
890	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
891	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
892	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
893	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
894	<	## OLI_Daniele
895	<	txt += '    if [ $middleware == OSG ]; then \n'
896	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
897	<	txt += '        cd $RUNTIME_AREA\n'
898	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
899	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
900	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
901	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
902	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
903	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
904	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
905	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
906	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
907	<	txt += '        fi\n'
908	<	txt += '    fi \n'
909	<	txt += '   exit 1 \n'
796	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
797	>	txt += '    job_exit_code=10034\n'
798	>	txt += '    func_exit\n'
799		txt += 'fi \n'
911	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
800		txt += 'cd '+self.version+'\n'
913	–	########## FEDE FOR DBS2 ######################
801		txt += 'SOFTWARE_DIR=`pwd`\n'
802	<	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
916	<	###############################################
917	<	### needed grep for bug in scramv1 ###
918	<	txt += scram+' runtime -sh\n'
802	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
803		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
804	<	txt += 'echo $PATH\n'
805	<
804	>	txt += 'if [ $? != 0 ] ; then\n'
805	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
806	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
807	>	txt += '    job_exit_code=10034\n'
808	>	txt += '    func_exit\n'
809	>	txt += 'fi \n'
810		# Handle the arguments:
811		txt += "\n"
812		txt += "## number of arguments (first argument always jobnumber)\n"
813		txt += "\n"
814	<	#        txt += "narg=$#\n"
927	<	txt += "if [ $nargs -lt 2 ]\n"
814	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
815		txt += "then\n"
816	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
817	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
818	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
932	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
933	<	txt += '    rm -f $RUNTIME_AREA/$repo \n'
934	<	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
935	<	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
936	<	## OLI_Daniele
937	<	txt += '    if [ $middleware == OSG ]; then \n'
938	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
939	<	txt += '        cd $RUNTIME_AREA\n'
940	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
941	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
942	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
943	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
944	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
945	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
946	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
947	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
948	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
949	<	txt += '        fi\n'
950	<	txt += '    fi \n'
951	<	txt += "    exit 1\n"
816	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
817	>	txt += '    job_exit_code=50113\n'
818	>	txt += "    func_exit\n"
819		txt += "fi\n"
820		txt += "\n"
821
822		# Prepare job-specific part
823		job = common.job_list[nj]
824	<	### FEDE FOR DBS OUTPUT PUBLICATION
958	<	if (self.datasetPath):
824	>	if (self.datasetPath):
825		txt += '\n'
826		txt += 'DatasetPath='+self.datasetPath+'\n'
827
828		datasetpath_split = self.datasetPath.split("/")
829	<
829	>
830		txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
831		txt += 'DataTier='+datasetpath_split[2]+'\n'
966	–	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
832		txt += 'ApplicationFamily=cmsRun\n'
833
834		else:
835		txt += 'DatasetPath=MCDataTier\n'
836		txt += 'PrimaryDataset=null\n'
837		txt += 'DataTier=null\n'
973	–	#txt += 'ProcessedDataset=null\n'
838		txt += 'ApplicationFamily=MCDataTier\n'
839	<	if self.pset != None: #CarlosDaniele
839	>	if self.pset != None:
840		pset = os.path.basename(job.configFilename())
841		txt += '\n'
842		txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
843		if (self.datasetPath): # standard job
844	<	#txt += 'InputFiles=$2\n'
845	<	txt += 'InputFiles=${args[1]}\n'
846	<	txt += 'MaxEvents=${args[2]}\n'
983	<	txt += 'SkipEvents=${args[3]}\n'
844	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
845	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
846	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
847		txt += 'echo "Inputfiles:<$InputFiles>"\n'
985	–	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
848		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
987	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
849		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
989	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
850		else:  # pythia like job
851	<	seedIndex=1
851	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
852	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
853	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
854	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
855		if (self.firstRun):
856	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
856	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
857		txt += 'echo "FirstRun: <$FirstRun>"\n'
995	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
996	–	seedIndex=seedIndex+1
858
859	<	if (self.sourceSeed):
999	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1000	<	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1001	<	seedIndex=seedIndex+1
1002	<	## the following seeds are not always present
1003	<	if (self.sourceSeedVtx):
1004	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1005	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1006	<	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1007	<	seedIndex += 1
1008	<	if (self.sourceSeedG4):
1009	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1010	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
1011	<	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1012	<	seedIndex += 1
1013	<	if (self.sourceSeedMix):
1014	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1015	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
1016	<	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1017	<	seedIndex += 1
1018	<	pass
1019	<	pass
1020	<	txt += 'mv -f '+pset+' pset.cfg\n'
859	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
860
1022	–	if len(self.additional_inbox_files) > 0:
1023	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1024	–	txt += '  tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1025	–	txt += 'fi\n'
1026	–	pass
861
862	<	if self.pset != None: #CarlosDaniele
863	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1030	<
1031	<	txt += '\n'
1032	<	txt += 'echo "***** cat pset.cfg *********"\n'
1033	<	txt += 'cat pset.cfg\n'
1034	<	txt += 'echo "****** end pset.cfg ********"\n'
862	>	if self.pset != None:
863	>	# FUTURE: Can simply for 2_1_x and higher
864		txt += '\n'
865	<	### FEDE FOR DBS OUTPUT PUBLICATION
866	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
865	>	if self.debug_pset==True:
866	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
867	>	txt += 'cat ' + psetName + '\n'
868	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
869	>	txt += '\n'
870	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
871		txt += 'echo "PSETHASH = $PSETHASH" \n'
1039	–	##############
872		txt += '\n'
1041	–	# txt += 'echo "***** cat pset1.cfg *********"\n'
1042	–	# txt += 'cat pset1.cfg\n'
1043	–	# txt += 'echo "****** end pset1.cfg ********"\n'
873		return txt
874
875	<	def wsBuildExe(self, nj=0):
875	>	def wsUntarSoftware(self, nj=0):
876		"""
877		Put in the script the commands to build an executable
878		or a library.
879		"""
880
881	<	txt = ""
881	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
882
883		if os.path.isfile(self.tgzNameWithPath):
884	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
884	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
885		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
886	+	txt += 'ls -Al \n'
887		txt += 'untar_status=$? \n'
888		txt += 'if [ $untar_status -ne 0 ]; then \n'
889	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
890	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
891	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
1062	<	txt += '   if [ $middleware == OSG ]; then \n'
1063	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1064	<	txt += '       cd $RUNTIME_AREA\n'
1065	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
1066	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
1067	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1068	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
1069	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
1070	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1071	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1072	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1073	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1074	<	txt += '       fi\n'
1075	<	txt += '   fi \n'
1076	<	txt += '   \n'
1077	<	txt += '   exit 1 \n'
889	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
890	>	txt += '   job_exit_code=$untar_status\n'
891	>	txt += '   func_exit\n'
892		txt += 'else \n'
893		txt += '   echo "Successful untar" \n'
894		txt += 'fi \n'
895		txt += '\n'
896	<	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
896	>	txt += 'echo ">>> Include ProdCommon in PYTHONPATH:"\n'
897		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
898	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1085	<	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1086	<	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1087	<	#txt += '   export PYTHONPATH=ProdAgentApi\n'
898	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon\n'
899		txt += 'else\n'
900	<	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1090	<	#txt += '   export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1091	<	#txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
900	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/ProdCommon:${PYTHONPATH}\n'
901		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1093	–	###################
902		txt += 'fi\n'
903		txt += '\n'
904
905		pass
906	<
906	>
907	>	return txt
908	>
909	>	def wsBuildExe(self, nj=0):
910	>	"""
911	>	Put in the script the commands to build an executable
912	>	or a library.
913	>	"""
914	>
915	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
916	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
917	>
918	>	txt += 'rm -r lib/ module/ \n'
919	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
920	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
921	>	if self.dataExist == True:
922	>	txt += 'rm -r src/ \n'
923	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
924	>	if len(self.additional_inbox_files)>0:
925	>	for file in self.additional_inbox_files:
926	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
927	>	txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
928	>
929	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
930	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon\n'
931	>	txt += 'else\n'
932	>	txt += '   export PYTHONPATH=$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
933	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
934	>	txt += 'fi\n'
935	>	txt += '\n'
936	>
937		return txt
938
939		def modifySteeringCards(self, nj):
940		"""
941	<	modify the card provided by the user,
941	>	modify the card provided by the user,
942		writing a new card into share dir
943		"""
944	<
944	>
945		def executableName(self):
946	<	if self.scriptExe: #CarlosDaniele
946	>	if self.scriptExe:
947		return "sh "
948		else:
949		return self.executable
950
951		def executableArgs(self):
952	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
953		if self.scriptExe:#CarlosDaniele
954		return   self.scriptExe + " $NJob"
955	<	else:
956	<	return " -p pset.cfg"
955	>	else:
956	>	ex_args = ""
957	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
958	>	# Framework job report
959	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
960	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
961	>	# Type of config file
962	>	if self.CMSSW_major >= 2 :
963	>	ex_args += " -p pset.py"
964	>	else:
965	>	ex_args += " -p pset.cfg"
966	>	return ex_args
967
968		def inputSandbox(self, nj):
969		"""
970		Returns a list of filenames to be put in JDL input sandbox.
971		"""
972		inp_box = []
1124	–	# # dict added to delete duplicate from input sandbox file list
1125	–	# seen = {}
1126	–	## code
973		if os.path.isfile(self.tgzNameWithPath):
974		inp_box.append(self.tgzNameWithPath)
975	<	if os.path.isfile(self.MLtgzfile):
976	<	inp_box.append(self.MLtgzfile)
1131	<	## config
1132	<	if not self.pset is None:
1133	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1134	<	## additional input files
1135	<	tgz = self.additionalInputFileTgz()
1136	<	inp_box.append(tgz)
975	>	wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
976	>	inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
977		return inp_box
978
979		def outputSandbox(self, nj):
#	Line 1144 | Line 984 | class Cmssw(JobType):
984
985		## User Declared output files
986		for out in (self.output_file+self.output_file_sandbox):
987	<	n_out = nj + 1
987	>	n_out = nj + 1
988		out_box.append(self.numberFile_(out,str(n_out)))
989		return out_box
990
#	Line 1159 | Line 999 | class Cmssw(JobType):
999		Returns part of a job script which renames the produced files.
1000		"""
1001
1002	<	txt = '\n'
1003	<	txt += '# directory content\n'
1002	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1003	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1004	>	txt += 'echo ">>> current directory content:"\n'
1005		txt += 'ls \n'
1006	+	txt += '\n'
1007
1008	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1008	>	for fileWithSuffix in (self.output_file):
1009		output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1010		txt += '\n'
1011		txt += '# check output file\n'
1170	–	# txt += 'ls '+fileWithSuffix+'\n'
1171	–	# txt += 'ls_result=$?\n'
1012		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1013	<	txt += '   mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1013	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1014	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1015	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1016	>	else:
1017	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1018	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1019		txt += 'else\n'
1020	<	txt += '   exit_status=60302\n'
1021	<	txt += '   echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1022	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1020	>	txt += '    job_exit_code=60302\n'
1021	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1022	>	if common.scheduler.name().upper() == 'CONDOR_G':
1023		txt += '    if [ $middleware == OSG ]; then \n'
1024		txt += '        echo "prepare dummy output file"\n'
1025		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1183 | Line 1028 | class Cmssw(JobType):
1028		file_list = []
1029		for fileWithSuffix in (self.output_file):
1030		file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1031	+
1032		txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1033	<
1033	>	txt += '\n'
1034	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1035	>	txt += 'echo ">>> current directory content:"\n'
1036	>	txt += 'ls \n'
1037	>	txt += '\n'
1038		txt += 'cd $RUNTIME_AREA\n'
1039	<	#### FEDE this is the cleanEnv function
1190	<	### OLI_DANIELE
1191	<	#txt += 'if [ $middleware == OSG ]; then\n'
1192	<	#txt += '    cd $RUNTIME_AREA\n'
1193	<	#txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1194	<	#txt += '    /bin/rm -rf $WORKING_DIR\n'
1195	<	#txt += '    if [ -d $WORKING_DIR ] ;then\n'
1196	<	#txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1197	<	#txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1198	<	#txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1199	<	#txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1200	<	#txt += '        rm -f $RUNTIME_AREA/$repo \n'
1201	<	#txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1202	<	#txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1203	<	#txt += '    fi\n'
1204	<	#txt += 'fi\n'
1205	<	#txt += '\n'
1206	<
1207	<
1039	>	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1040		return txt
1041
1042		def numberFile_(self, file, txt):
#	Line 1222 | Line 1054 | class Cmssw(JobType):
1054		result = name + '_' + txt + "." + ext
1055		else:
1056		result = name + '_' + txt
1057	<
1057	>
1058		return result
1059
1060		def getRequirements(self, nj=[]):
1061		"""
1062	<	return job requirements to add to jdl files
1062	>	return job requirements to add to jdl files
1063		"""
1064		req = ''
1065		if self.version:
1066		req='Member("VO-cms-' + \
1067		self.version + \
1068		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1069	<	## SL add requirement for OS version only if SL4
1238	<	#reSL4 = re.compile( r'slc4' )
1239	<	if self.executable_arch: # and reSL4.search(self.executable_arch):
1069	>	if self.executable_arch:
1070		req+=' && Member("VO-cms-' + \
1071		self.executable_arch + \
1072		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1073
1074		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1075	+	if common.scheduler.name() == "glitecoll":
1076	+	req += ' && other.GlueCEStateStatus == "Production" '
1077
1078		return req
1079
1080		def configFilename(self):
1081		""" return the config filename """
1082	<	return self.name()+'.cfg'
1082	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1083	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1084	>	return self.name()+'.py'
1085	>	else:
1086	>	return self.name()+'.cfg'
1087
1252	–	### OLI_DANIELE
1088		def wsSetupCMSOSGEnvironment_(self):
1089		"""
1090		Returns part of a job script which is prepares
1091		the execution environment and which is common for all CMS jobs.
1092		"""
1093	<	txt = '\n'
1094	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1095	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1096	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1097	<	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1098	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1264	<	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1093	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1094	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
1095	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1096	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1097	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1098	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1099		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1100	<	txt += '       export SCRAM_ARCH='+self.executable_arch+'\n'
1101	<	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1102	<	txt += '   else\n'
1103	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1104	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1105	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1272	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1273	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1274	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1275	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1276	<	txt += '       exit 1\n'
1277	<	txt += '\n'
1278	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1279	<	txt += '       cd $RUNTIME_AREA\n'
1280	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
1281	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
1282	<	txt += '           echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1283	<	txt += '           echo "JOB_EXIT_STATUS = 10017"\n'
1284	<	txt += '           echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1285	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1286	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1287	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1288	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1289	<	txt += '       fi\n'
1290	<	txt += '\n'
1291	<	txt += '       exit 1\n'
1292	<	txt += '   fi\n'
1100	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1101	>	txt += '    else\n'
1102	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1103	>	txt += '        job_exit_code=10020\n'
1104	>	txt += '        func_exit\n'
1105	>	txt += '    fi\n'
1106		txt += '\n'
1107	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1108	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1107	>	txt += '    echo "==> setup cms environment ok"\n'
1108	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1109
1110		return txt
1111	<
1299	<	### OLI_DANIELE
1111	>
1112		def wsSetupCMSLCGEnvironment_(self):
1113		"""
1114		Returns part of a job script which is prepares
1115		the execution environment and which is common for all CMS jobs.
1116		"""
1117	<	txt  = '   \n'
1118	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1119	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1120	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1121	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1122	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1123	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1124	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1125	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1126	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1127	<	txt += '       exit 1\n'
1128	<	txt += '   else\n'
1129	<	txt += '       echo "Sourcing environment... "\n'
1130	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1131	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1132	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1133	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1134	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1135	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1136	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1137	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1138	<	txt += '           exit 1\n'
1139	<	txt += '       fi\n'
1140	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1141	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1142	<	txt += '       result=$?\n'
1143	<	txt += '       if [ $result -ne 0 ]; then\n'
1332	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1333	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1334	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1335	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1336	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1337	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1338	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1339	<	txt += '           exit 1\n'
1340	<	txt += '       fi\n'
1341	<	txt += '   fi\n'
1342	<	txt += '   \n'
1343	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1344	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1117	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1118	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1119	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1120	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1121	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1122	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1123	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1124	>	txt += '        job_exit_code=10031\n'
1125	>	txt += '        func_exit\n'
1126	>	txt += '    else\n'
1127	>	txt += '        echo "Sourcing environment... "\n'
1128	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1129	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1130	>	txt += '            job_exit_code=10020\n'
1131	>	txt += '            func_exit\n'
1132	>	txt += '        fi\n'
1133	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1134	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1135	>	txt += '        result=$?\n'
1136	>	txt += '        if [ $result -ne 0 ]; then\n'
1137	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1138	>	txt += '            job_exit_code=10032\n'
1139	>	txt += '            func_exit\n'
1140	>	txt += '        fi\n'
1141	>	txt += '    fi\n'
1142	>	txt += '    \n'
1143	>	txt += '    echo "==> setup cms environment ok"\n'
1144		return txt
1145
1347	–	### FEDE FOR DBS OUTPUT PUBLICATION
1146		def modifyReport(self, nj):
1147		"""
1148	<	insert the part of the script that modifies the FrameworkJob Report
1148	>	insert the part of the script that modifies the FrameworkJob Report
1149		"""
1150	+	txt = '\n#Written by cms_cmssw::modifyReport\n'
1151	+	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1152	+	if (publish_data == 1):
1153	+	processedDataset = self.cfg_params['USER.publish_data_name']
1154	+	LFNBaseName = LFNBase(processedDataset)
1155
1156	<	txt = ''
1157	<	txt += 'echo "Modify Job Report" \n'
1158	<	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1159	<	################ FEDE FOR DBS2 #############################################
1160	<	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1161	<	#############################################################################
1162	<	try:
1360	<	publish_data = int(self.cfg_params['USER.publish_data'])
1361	<	except KeyError:
1362	<	publish_data = 0
1363	<
1364	<	txt += 'if [ -z "$SE" ]; then\n'
1365	<	txt += '    SE="" \n'
1366	<	txt += 'fi \n'
1367	<	txt += 'if [ -z "$SE_PATH" ]; then\n'
1368	<	txt += '    SE_PATH="" \n'
1369	<	txt += 'fi \n'
1370	<	txt += 'echo "SE = $SE"\n'
1371	<	txt += 'echo "SE_PATH = $SE_PATH"\n'
1156	>	txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1157	>	txt += '    FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1158	>	txt += 'else\n'
1159	>	txt += '    FOR_LFN=/copy_problems/ \n'
1160	>	txt += '    SE=""\n'
1161	>	txt += '    SE_PATH=""\n'
1162	>	txt += 'fi\n'
1163
1164	<	if (publish_data == 1):
1165	<	#processedDataset = self.cfg_params['USER.processed_datasetname']
1375	<	processedDataset = self.cfg_params['USER.publish_data_name']
1164	>	txt += 'echo ">>> Modify Job Report:" \n'
1165	>	txt += 'chmod a+x $SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1166		txt += 'ProcessedDataset='+processedDataset+'\n'
1377	–	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1378	–	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1379	–	#### FEDE: added slash in LFN ##############
1380	–	txt += '    FOR_LFN=/copy_problems/ \n'
1381	–	txt += 'else \n'
1382	–	txt += '    tmp=`echo $SE_PATH | awk -F \'store\' \'{print$2}\'` \n'
1383	–	#####  FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1384	–	txt += '    FOR_LFN=/store$tmp \n'
1385	–	txt += 'fi \n'
1167		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1168	+	txt += 'echo "SE = $SE"\n'
1169	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1170		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1171		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1172	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1173	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1391	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1392	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1393	<
1172	>	txt += 'echo "$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1173	>	txt += '$SOFTWARE_DIR/ProdCommon/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1174		txt += 'modifyReport_result=$?\n'
1395	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1175		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1176	<	txt += '    exit_status=1\n'
1177	<	txt += '    echo "ERROR: Problem with ModifyJobReport"\n'
1176	>	txt += '    modifyReport_result=70500\n'
1177	>	txt += '    job_exit_code=$modifyReport_result\n'
1178	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1179	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1180		txt += 'else\n'
1181	<	txt += '    mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1181	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1182		txt += 'fi\n'
1402	–	else:
1403	–	txt += 'ProcessedDataset=no_data_to_publish \n'
1404	–	#### FEDE: added slash in LFN ##############
1405	–	txt += 'FOR_LFN=/local/ \n'
1406	–	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1407	–	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1183		return txt
1184
1185	<	def cleanEnv(self):
1186	<	### OLI_DANIELE
1187	<	txt = ''
1188	<	txt += 'if [ $middleware == OSG ]; then\n'
1189	<	txt += '    cd $RUNTIME_AREA\n'
1190	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1191	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1192	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1193	<	txt += '              echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1194	<	txt += '              echo "JOB_EXIT_STATUS = 60999"\n'
1195	<	txt += '              echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1196	<	txt += '              dumpStatus $RUNTIME_AREA/$repo\n'
1197	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1198	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1199	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1185	>	def wsParseFJR(self):
1186	>	"""
1187	>	Parse the FrameworkJobReport to obtain useful infos
1188	>	"""
1189	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1190	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1191	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1192	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1193	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --MonitorID $MonitorID --MonitorJobID $MonitorJobID`\n'
1194	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1195	>	txt += '        executable_exit_status=`echo $cmd_out | awk -F\; "{print $1}" | awk -F ' ' "{print $NF}"`\n'
1196	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1197	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1198	>	txt += '            cat $RUNTIME_AREA/crab_fjr_NJob.xml\n'
1199	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1200	>	txt += '        else\n'
1201	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1202	>	txt += '        fi\n'
1203	>	txt += '    else\n'
1204	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1205		txt += '    fi\n'
1206	+	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1207	+
1208	+	if self.datasetPath:
1209	+	# VERIFY PROCESSED DATA
1210	+	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1211	+	txt += '      echo ">>> Verify list of processed files:"\n'
1212	+	txt += '      echo $InputFiles |tr -d "\\" |tr "," \n"|tr -d "\"" > input-files.txt\n'
1213	+	txt += '      grep LFN $RUNTIME_AREA/crab_fjr_$NJob.xml |cut -d">" -f2|cut -d"<" -f1|grep "/" > processed-files.txt\n'
1214	+	txt += '      cat input-files.txt  | sort | uniq > tmp.txt\n'
1215	+	txt += '      mv tmp.txt input-files.txt\n'
1216	+	txt += '      echo "cat input-files.txt"\n'
1217	+	txt += '      echo "----------------------"\n'
1218	+	txt += '      cat input-files.txt\n'
1219	+	txt += '      cat processed-files.txt | sort | uniq > tmp.txt\n'
1220	+	txt += '      mv tmp.txt processed-files.txt\n'
1221	+	txt += '      echo "----------------------"\n'
1222	+	txt += '      echo "cat processed-files.txt"\n'
1223	+	txt += '      echo "----------------------"\n'
1224	+	txt += '      cat processed-files.txt\n'
1225	+	txt += '      echo "----------------------"\n'
1226	+	txt += '      diff -q input-files.txt processed-files.txt\n'
1227	+	txt += '      fileverify_status=$?\n'
1228	+	txt += '      if [ $fileverify_status -ne 0 ]; then\n'
1229	+	txt += '         executable_exit_status=30001\n'
1230	+	txt += '         echo "ERROR ==> not all input files processed"\n'
1231	+	txt += '         echo "      ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1232	+	txt += '         echo "      ==> diff input-files.txt processed-files.txt"\n'
1233	+	txt += '      fi\n'
1234	+	txt += '    fi\n'
1235	+	txt += '\n'
1236	+	txt += 'else\n'
1237	+	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1238		txt += 'fi\n'
1239		txt += '\n'
1240	+	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1241	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1242	+	txt += 'job_exit_code=$executable_exit_status\n'
1243	+
1244		return txt
1245
1246		def setParam_(self, param, value):
#	Line 1433 | Line 1249 | class Cmssw(JobType):
1249		def getParams(self):
1250		return self._params
1251
1436	–	def setTaskid_(self):
1437	–	self._taskId = self.cfg_params['taskId']
1438	–
1439	–	def getTaskid(self):
1440	–	return self._taskId
1441	–
1252		def uniquelist(self, old):
1253		"""
1254		remove duplicates from a list
#	Line 1447 | Line 1257 | class Cmssw(JobType):
1257		for e in old:
1258		nd[e]=0
1259		return nd.keys()
1260	+
1261	+	def outList(self):
1262	+	"""
1263	+	check the dimension of the output files
1264	+	"""
1265	+	txt = ''
1266	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1267	+	listOutFiles = []
1268	+	stdout = 'CMSSW_$NJob.stdout'
1269	+	stderr = 'CMSSW_$NJob.stderr'
1270	+	if (self.return_data == 1):
1271	+	for file in (self.output_file+self.output_file_sandbox):
1272	+	listOutFiles.append(self.numberFile_(file, '$NJob'))
1273	+	listOutFiles.append(stdout)
1274	+	listOutFiles.append(stderr)
1275	+	else:
1276	+	for file in (self.output_file_sandbox):
1277	+	listOutFiles.append(self.numberFile_(file, '$NJob'))
1278	+	listOutFiles.append(stdout)
1279	+	listOutFiles.append(stderr)
1280	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1281	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1282	+	txt += 'export filesToCheck\n'
1283	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines