ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.75 by gutsche, Sun Apr 8 23:50:33 2007 UTC vs.
Revision 1.254 by ewv, Thu Oct 30 16:25:24 2008 UTC

#	Line 2 | Line 2 | from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	+	from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
6		import common
6	–	import PsetManipulator
7	–	import DataDiscovery
8	–	import DataDiscovery_DBS2
9	–	import DataLocation
7		import Scram
8
9	<	import os, string, re, shutil, glob
9	>	import os, string, glob
10
11		class Cmssw(JobType):
12	<	def __init__(self, cfg_params, ncjobs):
12	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
13		JobType.__init__(self, 'CMSSW')
14		common.logger.debug(3,'CMSSW::__init__')
15	+	self.skip_blocks = skip_blocks
16	+
17	+	self.argsList = []
18
19	–	# Marco.
19		self._params = {}
20		self.cfg_params = cfg_params
21
22	<	try:
23	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
24	<	except KeyError:
25	<	self.MaxTarBallSize = 100.0
22	>	# init BlackWhiteListParser
23	>	seWhiteList = cfg_params.get('EDG.se_white_list',[])
24	>	seBlackList = cfg_params.get('EDG.se_black_list',[])
25	>	self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
26	>
27	>	### Temporary patch to automatically skip the ISB size check:
28	>	server=self.cfg_params.get('CRAB.server_name',None)
29	>	size = 9.5
30	>	if server or common.scheduler.name().upper() in ['LSF','CAF']: size = 99999
31	>	### D.S.
32	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
33
34		# number of jobs requested to be created, limit obj splitting
35		self.ncjobs = ncjobs
36
37		log = common.logger
38	<
38	>
39		self.scram = Scram.Scram(cfg_params)
40		self.additional_inbox_files = []
41		self.scriptExe = ''
#	Line 37 | Line 43 | class Cmssw(JobType):
43		self.executable_arch = self.scram.getArch()
44		self.tgz_name = 'default.tgz'
45		self.scriptName = 'CMSSW.sh'
46	<	self.pset = ''      #scrip use case Da
47	<	self.datasetPath = '' #scrip use case Da
46	>	self.pset = ''
47	>	self.datasetPath = ''
48
49		# set FJR file name
50		self.fjrFileName = 'crab_fjr.xml'
51
52		self.version = self.scram.getSWVersion()
53	<	common.taskDB.setDict('codeVersion',self.version)
54	<	self.setParam_('application', self.version)
53	>	version_array = self.version.split('_')
54	>	self.CMSSW_major = 0
55	>	self.CMSSW_minor = 0
56	>	self.CMSSW_patch = 0
57	>	try:
58	>	self.CMSSW_major = int(version_array[1])
59	>	self.CMSSW_minor = int(version_array[2])
60	>	self.CMSSW_patch = int(version_array[3])
61	>	except:
62	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
63	>	raise CrabException(msg)
64
65		### collect Data cards
66
67	<	## get DBS mode
68	<	try:
54	<	self.use_dbs_2 = int(self.cfg_params['CMSSW.use_dbs_2'])
55	<	except KeyError:
56	<	self.use_dbs_2 = 0
57	<
58	<	try:
59	<	tmp =  cfg_params['CMSSW.datasetpath']
60	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
61	<	if string.lower(tmp)=='none':
62	<	self.datasetPath = None
63	<	self.selectNoInput = 1
64	<	else:
65	<	self.datasetPath = tmp
66	<	self.selectNoInput = 0
67	<	except KeyError:
68	<	msg = "Error: datasetpath not defined "
67	>	if not cfg_params.has_key('CMSSW.datasetpath'):
68	>	msg = "Error: datasetpath not defined "
69		raise CrabException(msg)
70
71	<	# ML monitoring
72	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
73	<	if not self.datasetPath:
74	<	self.setParam_('dataset', 'None')
75	<	self.setParam_('owner', 'None')
76	<	else:
77	<	datasetpath_split = self.datasetPath.split("/")
78	<	if self.use_dbs_2 == 1 :
79	<	self.setParam_('dataset', datasetpath_split[1])
80	<	self.setParam_('owner', datasetpath_split[2])
81	<	else :
82	<	self.setParam_('dataset', datasetpath_split[1])
83	<	self.setParam_('owner', datasetpath_split[-1])
71	>	### Temporary: added to remove input file control in the case of PU
72	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
73
74	<	self.setTaskid_()
75	<	self.setParam_('taskId', self.cfg_params['taskId'])
74	>	tmp =  cfg_params['CMSSW.datasetpath']
75	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
76
77	<	self.dataTiers = []
77	>	if tmp =='':
78	>	msg = "Error: datasetpath not defined "
79	>	raise CrabException(msg)
80	>	elif string.lower(tmp)=='none':
81	>	self.datasetPath = None
82	>	self.selectNoInput = 1
83	>	else:
84	>	self.datasetPath = tmp
85	>	self.selectNoInput = 0
86
87	+	self.dataTiers = []
88	+	self.debugWrap = ''
89	+	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
90	+	if self.debug_wrapper: self.debugWrap='--debug'
91		## now the application
92	<	try:
93	<	self.executable = cfg_params['CMSSW.executable']
93	<	self.setParam_('exe', self.executable)
94	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
95	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
96	<	log.debug(3,msg)
97	<	except KeyError:
98	<	self.executable = 'cmsRun'
99	<	self.setParam_('exe', self.executable)
100	<	msg = "User executable not defined. Use cmsRun"
101	<	log.debug(3,msg)
102	<	pass
92	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
93	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
94
95	<	try:
105	<	self.pset = cfg_params['CMSSW.pset']
106	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
107	<	if self.pset.lower() != 'none' :
108	<	if (not os.path.exists(self.pset)):
109	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
110	<	else:
111	<	self.pset = None
112	<	except KeyError:
95	>	if not cfg_params.has_key('CMSSW.pset'):
96		raise CrabException("PSet file missing. Cannot run cmsRun ")
97	+	self.pset = cfg_params['CMSSW.pset']
98	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
99	+	if self.pset.lower() != 'none' :
100	+	if (not os.path.exists(self.pset)):
101	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
102	+	else:
103	+	self.pset = None
104
105		# output files
106		## stuff which must be returned always via sandbox
#	Line 120 | Line 110 | class Cmssw(JobType):
110		self.output_file_sandbox.append(self.fjrFileName)
111
112		# other output files to be returned via sandbox or copied to SE
113	<	try:
114	<	self.output_file = []
115	<	tmp = cfg_params['CMSSW.output_file']
116	<	if tmp != '':
117	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
118	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
119	<	for tmp in tmpOutFiles:
120	<	tmp=string.strip(tmp)
131	<	self.output_file.append(tmp)
132	<	pass
133	<	else:
134	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
135	<	pass
136	<	pass
137	<	except KeyError:
138	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available")
139	<	pass
113	>	outfileflag = False
114	>	self.output_file = []
115	>	tmp = cfg_params.get('CMSSW.output_file',None)
116	>	if tmp :
117	>	self.output_file = [x.strip() for x in tmp.split(',')]
118	>	outfileflag = True #output found
119	>	#else:
120	>	#    log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
121
122		# script_exe file as additional file in inputSandbox
123	<	try:
124	<	self.scriptExe = cfg_params['USER.script_exe']
125	<	if self.scriptExe != '':
126	<	if not os.path.isfile(self.scriptExe):
127	<	msg ="ERROR. file "+self.scriptExe+" not found"
128	<	raise CrabException(msg)
148	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
149	<	except KeyError:
150	<	self.scriptExe = ''
123	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
124	>	if self.scriptExe :
125	>	if not os.path.isfile(self.scriptExe):
126	>	msg ="ERROR. file "+self.scriptExe+" not found"
127	>	raise CrabException(msg)
128	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
129
152	–	#CarlosDaniele
130		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
131	<	msg ="Error. script_exe  not defined"
132	<	raise CrabException(msg)
131	>	msg ="Error. script_exe  not defined"
132	>	raise CrabException(msg)
133	>
134	>	# use parent files...
135	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
136
137		## additional input files
138	<	try:
138	>	if cfg_params.has_key('USER.additional_input_files'):
139		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
140		for tmp in tmpAddFiles:
141		tmp = string.strip(tmp)
142		dirname = ''
143		if not tmp[0]=="/": dirname = "."
144	<	files = glob.glob(os.path.join(dirname, tmp))
144	>	files = []
145	>	if string.find(tmp,"*")>-1:
146	>	files = glob.glob(os.path.join(dirname, tmp))
147	>	if len(files)==0:
148	>	raise CrabException("No additional input file found with this pattern: "+tmp)
149	>	else:
150	>	files.append(tmp)
151		for file in files:
152		if not os.path.exists(file):
153		raise CrabException("Additional input file not found: "+file)
154		pass
155	<	storedFile = common.work_space.shareDir()+file
170	<	shutil.copyfile(file, storedFile)
171	<	self.additional_inbox_files.append(string.strip(storedFile))
155	>	self.additional_inbox_files.append(string.strip(file))
156		pass
157		pass
158		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
159	<	except KeyError:
176	<	pass
177	<
178	<	# files per job
179	<	try:
180	<	if (cfg_params['CMSSW.files_per_jobs']):
181	<	raise CrabException("files_per_jobs no longer supported.  Quitting.")
182	<	except KeyError:
183	<	pass
159	>	pass
160
161		## Events per job
162	<	try:
162	>	if cfg_params.has_key('CMSSW.events_per_job'):
163		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
164		self.selectEventsPerJob = 1
165	<	except KeyError:
165	>	else:
166		self.eventsPerJob = -1
167		self.selectEventsPerJob = 0
168	<
168	>
169		## number of jobs
170	<	try:
170	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
171		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
172		self.selectNumberOfJobs = 1
173	<	except KeyError:
173	>	else:
174		self.theNumberOfJobs = 0
175		self.selectNumberOfJobs = 0
176
177	<	try:
177	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
178		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
179		self.selectTotalNumberEvents = 1
180	<	except KeyError:
180	>	if self.selectNumberOfJobs  == 1:
181	>	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
182	>	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
183	>	raise CrabException(msg)
184	>	else:
185		self.total_number_of_events = 0
186		self.selectTotalNumberEvents = 0
187
188	<	if self.pset != None: #CarlosDaniele
188	>	if self.pset != None:
189		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
190		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
191		raise CrabException(msg)
#	Line 214 | Line 194 | class Cmssw(JobType):
194		msg = 'Must specify  number_of_jobs.'
195		raise CrabException(msg)
196
197	<	## source seed for pythia
198	<	try:
199	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
200	<	except KeyError:
201	<	self.sourceSeed = None
202	<	common.logger.debug(5,"No seed given")
197	>	## New method of dealing with seeds
198	>	self.incrementSeeds = []
199	>	self.preserveSeeds = []
200	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
201	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
202	>	for tmp in tmpList:
203	>	tmp.strip()
204	>	self.preserveSeeds.append(tmp)
205	>	if cfg_params.has_key('CMSSW.increment_seeds'):
206	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
207	>	for tmp in tmpList:
208	>	tmp.strip()
209	>	self.incrementSeeds.append(tmp)
210	>
211	>	## FUTURE: Can remove in CRAB 2.4.0
212	>	self.sourceSeed    = cfg_params.get('CMSSW.pythia_seed',None)
213	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
214	>	self.sourceSeedG4  = cfg_params.get('CMSSW.g4_seed',None)
215	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
216	>	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
217	>	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
218	>	raise CrabException(msg)
219
220	<	try:
221	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
222	<	except KeyError:
223	<	self.sourceSeedVtx = None
224	<	common.logger.debug(5,"No vertex seed given")
229	<	try:
230	<	self.firstRun = int(cfg_params['CMSSW.first_run'])
231	<	except KeyError:
232	<	self.firstRun = None
233	<	common.logger.debug(5,"No first run given")
234	<	if self.pset != None: #CarlosDaniele
235	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
220	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
221	>
222	>	# Copy/return
223	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
224	>	self.return_data = int(cfg_params.get('USER.return_data',0))
225
226		#DBSDLS-start
227	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
227	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
228		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
229		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
230		self.jobDestination=[]  # Site destination(s) for each job (list of lists)
#	Line 244 | Line 233 | class Cmssw(JobType):
233		blockSites = {}
234		if self.datasetPath:
235		blockSites = self.DataDiscoveryAndLocation(cfg_params)
236	<	#DBSDLS-end
236	>	#DBSDLS-end
237
249	–	self.tgzNameWithPath = self.getTarBall(self.executable)
250	–
238		## Select Splitting
239	<	if self.selectNoInput:
240	<	if self.pset == None: #CarlosDaniele
239	>	if self.selectNoInput:
240	>	if self.pset == None:
241		self.jobSplittingForScript()
242		else:
243		self.jobSplittingNoInput()
244	<	else:
244	>	elif (cfg_params.get('CMSSW.noblockboundary',0)):
245	>	self.jobSplittingNoBlockBoundary(blockSites)
246	>	else:
247		self.jobSplittingByBlocks(blockSites)
248
249	<	# modify Pset
250	<	if self.pset != None: #CarlosDaniele
251	<	try:
252	<	if (self.datasetPath): # standard job
253	<	# allow to processa a fraction of events in a file
254	<	self.PsetEdit.inputModule("INPUT")
255	<	self.PsetEdit.maxEvent("INPUTMAXEVENTS")
256	<	self.PsetEdit.skipEvent("INPUTSKIPEVENTS")
257	<	else:  # pythia like job
258	<	self.PsetEdit.maxEvent(self.eventsPerJob)
259	<	if (self.firstRun):
260	<	self.PsetEdit.pythiaFirstRun("INPUTFIRSTRUN")  #First Run
261	<	if (self.sourceSeed) :
262	<	self.PsetEdit.pythiaSeed("INPUT")
263	<	if (self.sourceSeedVtx) :
264	<	self.PsetEdit.pythiaSeedVtx("INPUTVTX")
265	<	# add FrameworkJobReport to parameter-set
266	<	self.PsetEdit.addCrabFJR(self.fjrFileName)
267	<	self.PsetEdit.psetWriter(self.configFilename())
268	<	except:
269	<	msg='Error while manipuliating ParameterSet: exiting...'
270	<	raise CrabException(msg)
249	>	# modify Pset only the first time
250	>	if isNew:
251	>	if self.pset != None:
252	>	import PsetManipulator as pp
253	>	PsetEdit = pp.PsetManipulator(self.pset)
254	>	try:
255	>	# Add FrameworkJobReport to parameter-set, set max events.
256	>	# Reset later for data jobs by writeCFG which does all modifications
257	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
258	>	PsetEdit.maxEvent(self.eventsPerJob)
259	>	PsetEdit.psetWriter(self.configFilename())
260	>	## If present, add TFileService to output files
261	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
262	>	tfsOutput = PsetEdit.getTFileService()
263	>	if tfsOutput:
264	>	if tfsOutput in self.output_file:
265	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
266	>	else:
267	>	outfileflag = True #output found
268	>	self.output_file.append(tfsOutput)
269	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
270	>	pass
271	>	pass
272	>	## If present and requested, add PoolOutputModule to output files
273	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
274	>	edmOutput = PsetEdit.getPoolOutputModule()
275	>	if edmOutput:
276	>	if edmOutput in self.output_file:
277	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
278	>	else:
279	>	self.output_file.append(edmOutput)
280	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
281	>	pass
282	>	pass
283	>	except CrabException:
284	>	msg='Error while manipulating ParameterSet: exiting...'
285	>	raise CrabException(msg)
286	>	## Prepare inputSandbox TarBall (only the first time)
287	>	self.tgzNameWithPath = self.getTarBall(self.executable)
288
289		def DataDiscoveryAndLocation(self, cfg_params):
290
291	+	import DataDiscovery
292	+	import DataLocation
293		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
294
295		datasetPath=self.datasetPath
296
297		## Contact the DBS
298	<	common.logger.message("Contacting DBS...")
298	>	common.logger.message("Contacting Data Discovery Services ...")
299		try:
300	<
293	<	if self.use_dbs_2 == 1 :
294	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
295	<	else :
296	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
300	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
301		self.pubdata.fetchDBSInfo()
302
303		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 305 | Line 309 | class Cmssw(JobType):
309		except DataDiscovery.DataDiscoveryError, ex:
310		msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
311		raise CrabException(msg)
308	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
309	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
310	–	raise CrabException(msg)
311	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
312	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
313	–	raise CrabException(msg)
314	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
315	–	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
316	–	raise CrabException(msg)
317	–
318	–	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
319	–	common.logger.message("Required data are :"+self.datasetPath)
312
313		self.filesbyblock=self.pubdata.getFiles()
314		self.eventsbyblock=self.pubdata.getEventsPerBlock()
315		self.eventsbyfile=self.pubdata.getEventsPerFile()
316	+	self.parentFiles=self.pubdata.getParent()
317
318		## get max number of events
319	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
327	<	common.logger.message("The number of available events is %s\n"%self.maxEvents)
319	>	self.maxEvents=self.pubdata.getMaxEvents()
320
329	–	common.logger.message("Contacting DLS...")
321		## Contact the DLS and build a list of sites hosting the fileblocks
322		try:
323		dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
#	Line 334 | Line 325 | class Cmssw(JobType):
325		except DataLocation.DataLocationError , ex:
326		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
327		raise CrabException(msg)
328	<
328	>
329
330		sites = dataloc.getSites()
331		allSites = []
#	Line 344 | Line 335 | class Cmssw(JobType):
335		allSites.append(oneSite)
336		allSites = self.uniquelist(allSites)
337
338	<	common.logger.message("Sites ("+str(len(allSites))+") hosting part/all of dataset: "+str(allSites))
339	<	common.logger.debug(6, "List of Sites: "+str(allSites))
338	>	# screen output
339	>	common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
340	>
341		return sites
342	<
342	>
343		def jobSplittingByBlocks(self, blockSites):
344		"""
345		Perform job splitting. Jobs run over an integer number of files
#	Line 397 | Line 389 | class Cmssw(JobType):
389		totalNumberOfJobs = 999999999
390		else :
391		totalNumberOfJobs = self.ncjobs
400	–
392
393		blocks = blockSites.keys()
394		blockCount = 0
#	Line 407 | Line 398 | class Cmssw(JobType):
398		jobCount = 0
399		list_of_lists = []
400
401	+	# list tracking which jobs are in which jobs belong to which block
402	+	jobsOfBlock = {}
403	+
404		# ---- Iterate over the blocks in the dataset until ---- #
405		# ---- we've met the requested total # of events    ---- #
406		while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
407		block = blocks[blockCount]
408		blockCount += 1
409	<
409	>	if block not in jobsOfBlock.keys() :
410	>	jobsOfBlock[block] = []
411	>
412		if self.eventsbyblock.has_key(block) :
413		numEventsInBlock = self.eventsbyblock[block]
414		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
415	<
415	>
416		files = self.filesbyblock[block]
417		numFilesInBlock = len(files)
418		if (numFilesInBlock <= 0):
#	Line 424 | Line 420 | class Cmssw(JobType):
420		fileCount = 0
421
422		# ---- New block => New job ---- #
423	<	parString = "\\{"
423	>	parString = ""
424		# counter for number of events in files currently worked on
425		filesEventCount = 0
426		# flag if next while loop should touch new file
427		newFile = 1
428		# job event counter
429		jobSkipEventCount = 0
430	<
430	>
431		# ---- Iterate over the files in the block until we've met the requested ---- #
432		# ---- total # of events or we've gone over all the files in this block  ---- #
433	+	pString=''
434		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
435		file = files[fileCount]
436	+	if self.useParent:
437	+	parent = self.parentFiles[file]
438	+	for f in parent :
439	+	pString += '\\\"' + f + '\\\"\,'
440	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
441	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
442		if newFile :
443		try:
444		numEventsInFile = self.eventsbyfile[file]
#	Line 447 | Line 450 | class Cmssw(JobType):
450		newFile = 0
451		except KeyError:
452		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
450	–
453
454	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
455		# if less events in file remain than eventsPerJobRequested
456	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
456	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
457		# if last file in block
458		if ( fileCount == numFilesInBlock-1 ) :
459		# end job using last file, use remaining events in block
460		# close job and touch new file
461		fullString = parString[:-2]
462	<	fullString += '\\}'
463	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
462	>	if self.useParent:
463	>	fullParentString = pString[:-2]
464	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
465	>	else:
466	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
467		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
468		self.jobDestination.append(blockSites[block])
469		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
470	+	# fill jobs of block dictionary
471	+	jobsOfBlock[block].append(jobCount+1)
472		# reset counter
473		jobCount = jobCount + 1
474		totalEventCount = totalEventCount + filesEventCount - jobSkipEventCount
475		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
476		jobSkipEventCount = 0
477		# reset file
478	<	parString = "\\{"
478	>	pString = ""
479	>	parString = ""
480		filesEventCount = 0
481		newFile = 1
482		fileCount += 1
#	Line 479 | Line 488 | class Cmssw(JobType):
488		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
489		# close job and touch new file
490		fullString = parString[:-2]
491	<	fullString += '\\}'
492	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
491	>	if self.useParent:
492	>	fullParentString = pString[:-2]
493	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
494	>	else:
495	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
496		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
497		self.jobDestination.append(blockSites[block])
498		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
499	+	jobsOfBlock[block].append(jobCount+1)
500		# reset counter
501		jobCount = jobCount + 1
502		totalEventCount = totalEventCount + eventsPerJobRequested
503		eventsRemaining = eventsRemaining - eventsPerJobRequested
504		jobSkipEventCount = 0
505		# reset file
506	<	parString = "\\{"
506	>	pString = ""
507	>	parString = ""
508		filesEventCount = 0
509		newFile = 1
510		fileCount += 1
511	<
511	>
512		# if more events in file remain than eventsPerJobRequested
513		else :
514		# close job but don't touch new file
515		fullString = parString[:-2]
516	<	fullString += '\\}'
517	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
516	>	if self.useParent:
517	>	fullParentString = pString[:-2]
518	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
519	>	else:
520	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
521		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
522		self.jobDestination.append(blockSites[block])
523		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
524	+	jobsOfBlock[block].append(jobCount+1)
525		# increase counter
526		jobCount = jobCount + 1
527		totalEventCount = totalEventCount + eventsPerJobRequested
#	Line 513 | Line 531 | class Cmssw(JobType):
531		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
532		# remove all but the last file
533		filesEventCount = self.eventsbyfile[file]
534	<	parString = "\\{"
535	<	parString += '\\\"' + file + '\\\"\,'
534	>	if self.useParent:
535	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
536	>	parString = '\\\"' + file + '\\\"\,'
537		pass # END if
538		pass # END while (iterate over files in the block)
539		pass # END while (iterate over blocks in the dataset)
540		self.ncjobs = self.total_number_of_jobs = jobCount
541		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
542		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
543	<	common.logger.message("\n"+str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
544	<
543	>	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
544	>
545	>	# screen output
546	>	screenOutput = "List of jobs and available destination sites:\n\n"
547	>
548	>	# keep trace of block with no sites to print a warning at the end
549	>	noSiteBlock = []
550	>	bloskNoSite = []
551	>
552	>	blockCounter = 0
553	>	for block in blocks:
554	>	if block in jobsOfBlock.keys() :
555	>	blockCounter += 1
556	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
557	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
558	>	if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
559	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
560	>	bloskNoSite.append( blockCounter )
561	>
562	>	common.logger.message(screenOutput)
563	>	if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
564	>	msg = 'WARNING: No sites are hosting any part of data for block:\n                '
565	>	virgola = ""
566	>	if len(bloskNoSite) > 1:
567	>	virgola = ","
568	>	for block in bloskNoSite:
569	>	msg += ' ' + str(block) + virgola
570	>	msg += '\n               Related jobs:\n                 '
571	>	virgola = ""
572	>	if len(noSiteBlock) > 1:
573	>	virgola = ","
574	>	for range_jobs in noSiteBlock:
575	>	msg += str(range_jobs) + virgola
576	>	msg += '\n               will not be submitted and this block of data can not be analyzed!\n'
577	>	if self.cfg_params.has_key('EDG.se_white_list'):
578	>	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
579	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
580	>	msg += 'Please check if the dataset is available at this site!)\n'
581	>	if self.cfg_params.has_key('EDG.ce_white_list'):
582	>	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
583	>	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
584	>	msg += 'Please check if the dataset is available at this site!)\n'
585	>
586	>	common.logger.message(msg)
587	>
588		self.list_of_args = list_of_lists
589		return
590
591	+	def jobSplittingNoBlockBoundary(self,blockSites):
592	+	"""
593	+	"""
594	+	# ---- Handle the possible job splitting configurations ---- #
595	+	if (self.selectTotalNumberEvents):
596	+	totalEventsRequested = self.total_number_of_events
597	+	if (self.selectEventsPerJob):
598	+	eventsPerJobRequested = self.eventsPerJob
599	+	if (self.selectNumberOfJobs):
600	+	totalEventsRequested = self.theNumberOfJobs * self.eventsPerJob
601	+
602	+	# If user requested all the events in the dataset
603	+	if (totalEventsRequested == -1):
604	+	eventsRemaining=self.maxEvents
605	+	# If user requested more events than are in the dataset
606	+	elif (totalEventsRequested > self.maxEvents):
607	+	eventsRemaining = self.maxEvents
608	+	common.logger.message("Requested "+str(self.total_number_of_events)+ " events, but only "+str(self.maxEvents)+" events are available.")
609	+	# If user requested less events than are in the dataset
610	+	else:
611	+	eventsRemaining = totalEventsRequested
612	+
613	+	# If user requested more events per job than are in the dataset
614	+	if (self.selectEventsPerJob and eventsPerJobRequested > self.maxEvents):
615	+	eventsPerJobRequested = self.maxEvents
616	+
617	+	# For user info at end
618	+	totalEventCount = 0
619	+
620	+	if (self.selectTotalNumberEvents and self.selectNumberOfJobs):
621	+	eventsPerJobRequested = int(eventsRemaining/self.theNumberOfJobs)
622	+
623	+	if (self.selectNumberOfJobs):
624	+	common.logger.message("May not create the exact number_of_jobs requested.")
625	+
626	+	if ( self.ncjobs == 'all' ) :
627	+	totalNumberOfJobs = 999999999
628	+	else :
629	+	totalNumberOfJobs = self.ncjobs
630	+
631	+	blocks = blockSites.keys()
632	+	blockCount = 0
633	+	# Backup variable in case self.maxEvents counted events in a non-included block
634	+	numBlocksInDataset = len(blocks)
635	+
636	+	jobCount = 0
637	+	list_of_lists = []
638	+
639	+	#AF
640	+	#AF do not reset input files and event count on block boundary
641	+	#AF
642	+	parString=""
643	+	filesEventCount = 0
644	+	#AF
645	+
646	+	# list tracking which jobs are in which jobs belong to which block
647	+	jobsOfBlock = {}
648	+	while ( (eventsRemaining > 0) and (blockCount < numBlocksInDataset) and (jobCount < totalNumberOfJobs)):
649	+	block = blocks[blockCount]
650	+	blockCount += 1
651	+	if block not in jobsOfBlock.keys() :
652	+	jobsOfBlock[block] = []
653	+
654	+	if self.eventsbyblock.has_key(block) :
655	+	numEventsInBlock = self.eventsbyblock[block]
656	+	common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
657	+	files = self.filesbyblock[block]
658	+	numFilesInBlock = len(files)
659	+	if (numFilesInBlock <= 0):
660	+	continue
661	+	fileCount = 0
662	+	#AF
663	+	#AF do not reset input files and event count of block boundary
664	+	#AF
665	+	## ---- New block => New job ---- #
666	+	#parString = ""
667	+	# counter for number of events in files currently worked on
668	+	#filesEventCount = 0
669	+	#AF
670	+	# flag if next while loop should touch new file
671	+	newFile = 1
672	+	# job event counter
673	+	jobSkipEventCount = 0
674	+
675	+	# ---- Iterate over the files in the block until we've met the requested ---- #
676	+	# ---- total # of events or we've gone over all the files in this block  ---- #
677	+	pString=''
678	+	while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
679	+	file = files[fileCount]
680	+	if self.useParent:
681	+	parent = self.parentFiles[file]
682	+	for f in parent :
683	+	pString += '\\\"' + f + '\\\"\,'
684	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
685	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
686	+	if newFile :
687	+	try:
688	+	numEventsInFile = self.eventsbyfile[file]
689	+	common.logger.debug(6, "File "+str(file)+" has "+str(numEventsInFile)+" events")
690	+	# increase filesEventCount
691	+	filesEventCount += numEventsInFile
692	+	# Add file to current job
693	+	parString += '\\\"' + file + '\\\"\,'
694	+	newFile = 0
695	+	except KeyError:
696	+	common.logger.message("File "+str(file)+" has unknown number of events: skipping")
697	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
698	+	#common.logger.message("AF filesEventCount %s - jobSkipEventCount %s "%(filesEventCount,jobSkipEventCount))
699	+	# if less events in file remain than eventsPerJobRequested
700	+	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
701	+	#AF
702	+	#AF skip fileboundary part
703	+	#AF
704	+	# go to next file
705	+	newFile = 1
706	+	fileCount += 1
707	+	# if events in file equal to eventsPerJobRequested
708	+	elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
709	+	# close job and touch new file
710	+	fullString = parString[:-2]
711	+	if self.useParent:
712	+	fullParentString = pString[:-2]
713	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
714	+	else:
715	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
716	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
717	+	self.jobDestination.append(blockSites[block])
718	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
719	+	jobsOfBlock[block].append(jobCount+1)
720	+	# reset counter
721	+	jobCount = jobCount + 1
722	+	totalEventCount = totalEventCount + eventsPerJobRequested
723	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
724	+	jobSkipEventCount = 0
725	+	# reset file
726	+	pString = ""
727	+	parString = ""
728	+	filesEventCount = 0
729	+	newFile = 1
730	+	fileCount += 1
731	+
732	+	# if more events in file remain than eventsPerJobRequested
733	+	else :
734	+	# close job but don't touch new file
735	+	fullString = parString[:-2]
736	+	if self.useParent:
737	+	fullParentString = pString[:-2]
738	+	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
739	+	else:
740	+	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
741	+	common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
742	+	self.jobDestination.append(blockSites[block])
743	+	common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
744	+	jobsOfBlock[block].append(jobCount+1)
745	+	# increase counter
746	+	jobCount = jobCount + 1
747	+	totalEventCount = totalEventCount + eventsPerJobRequested
748	+	eventsRemaining = eventsRemaining - eventsPerJobRequested
749	+	# calculate skip events for last file
750	+	# use filesEventCount (contains several files), jobSkipEventCount and eventsPerJobRequest
751	+	jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
752	+	# remove all but the last file
753	+	filesEventCount = self.eventsbyfile[file]
754	+	if self.useParent:
755	+	for f in parent : pString += '\\\"' + f + '\\\"\,'
756	+	parString = '\\\"' + file + '\\\"\,'
757	+	pass # END if
758	+	pass # END while (iterate over files in the block)
759	+	pass # END while (iterate over blocks in the dataset)
760	+	self.ncjobs = self.total_number_of_jobs = jobCount
761	+	if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
762	+	common.logger.message("eventsRemaining "+str(eventsRemaining))
763	+	common.logger.message("jobCount "+str(jobCount))
764	+	common.logger.message(" totalNumberOfJobs "+str(totalNumberOfJobs))
765	+	common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
766	+	common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
767	+
768	+	# screen output
769	+	screenOutput = "List of jobs and available destination sites:\n\n"
770	+
771	+	#AF
772	+	#AF   skip check on  block with no sites
773	+	#AF
774	+	self.list_of_args = list_of_lists
775	+
776	+	return
777	+
778	+
779	+
780		def jobSplittingNoInput(self):
781		"""
782		Perform job splitting based on number of event per job
783		"""
784		common.logger.debug(5,'Splitting per events')
785	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
786	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
787	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
785	>
786	>	if (self.selectEventsPerJob):
787	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
788	>	if (self.selectNumberOfJobs):
789	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
790	>	if (self.selectTotalNumberEvents):
791	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
792
793		if (self.total_number_of_events < 0):
794		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 542 | Line 797 | class Cmssw(JobType):
797		if (self.selectEventsPerJob):
798		if (self.selectTotalNumberEvents):
799		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
800	<	elif(self.selectNumberOfJobs) :
800	>	elif(self.selectNumberOfJobs) :
801		self.total_number_of_jobs =self.theNumberOfJobs
802	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
802	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
803
804		elif (self.selectNumberOfJobs) :
805		self.total_number_of_jobs = self.theNumberOfJobs
806		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
807	<
807	>
808		common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
809
810		# is there any remainder?
#	Line 565 | Line 820 | class Cmssw(JobType):
820		self.list_of_args = []
821		for i in range(self.total_number_of_jobs):
822		## Since there is no input, any site is good
823	<	# self.jobDestination.append(["Any"])
824	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
570	<	args=''
823	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
824	>	args=[]
825		if (self.firstRun):
826	<	## pythia first run
827	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
828	<	args=args+(str(self.firstRun)+str(i))
575	<	else:
576	<	## no first run
577	<	#self.list_of_args.append([str(i)])
578	<	args=args+str(i)
579	<	if (self.sourceSeed):
580	<	if (self.sourceSeedVtx):
581	<	## pythia + vtx random seed
582	<	#self.list_of_args.append([
583	<	#                          str(self.sourceSeed)+str(i),
584	<	#                          str(self.sourceSeedVtx)+str(i)
585	<	#                          ])
586	<	args=args+str(',')+str(self.sourceSeed)+str(i)+str(',')+str(self.sourceSeedVtx)+str(i)
587	<	else:
588	<	## only pythia random seed
589	<	#self.list_of_args.append([(str(self.sourceSeed)+str(i))])
590	<	args=args +str(',')+str(self.sourceSeed)+str(i)
591	<	else:
592	<	## no random seed
593	<	if str(args)=='': args=args+(str(self.firstRun)+str(i))
594	<	arguments=args.split(',')
595	<	if len(arguments)==3:self.list_of_args.append([str(arguments[0]),str(arguments[1]),str(arguments[2])])
596	<	elif len(arguments)==2:self.list_of_args.append([str(arguments[0]),str(arguments[1])])
597	<	else :self.list_of_args.append([str(arguments[0])])
598	<
599	<	#   print self.list_of_args
826	>	## pythia first run
827	>	args.append(str(self.firstRun)+str(i))
828	>	self.list_of_args.append(args)
829
830		return
831
832
833	<	def jobSplittingForScript(self):#CarlosDaniele
833	>	def jobSplittingForScript(self):
834		"""
835		Perform job splitting based on number of job
836		"""
#	Line 617 | Line 846 | class Cmssw(JobType):
846		# argument is seed number.$i
847		self.list_of_args = []
848		for i in range(self.total_number_of_jobs):
620	–	## Since there is no input, any site is good
621	–	# self.jobDestination.append(["Any"])
849		self.jobDestination.append([""])
623	–	## no random seed
850		self.list_of_args.append([str(i)])
851		return
852
853	<	def split(self, jobParams):
854	<
629	<	common.jobDB.load()
630	<	#### Fabio
853	>	def split(self, jobParams,firstJobID):
854	>
855		njobs = self.total_number_of_jobs
856		arglist = self.list_of_args
857		# create the empty structure
858		for i in range(njobs):
859		jobParams.append("")
636	–
637	–	for job in range(njobs):
638	–	jobParams[job] = arglist[job]
639	–	# print str(arglist[job])
640	–	# print jobParams[job]
641	–	common.jobDB.setArguments(job, jobParams[job])
642	–	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
643	–	common.jobDB.setDestination(job, self.jobDestination[job])
860
861	<	common.jobDB.save()
861	>	listID=[]
862	>	listField=[]
863	>	for id in range(njobs):
864	>	job = id + int(firstJobID)
865	>	jobParams[id] = arglist[id]
866	>	listID.append(job+1)
867	>	job_ToSave ={}
868	>	concString = ' '
869	>	argu=''
870	>	if len(jobParams[id]):
871	>	argu +=   concString.join(jobParams[id] )
872	>	job_ToSave['arguments']= str(job+1)+' '+argu
873	>	job_ToSave['dlsDestination']= self.jobDestination[id]
874	>	listField.append(job_ToSave)
875	>	msg="Job "+str(job)+" Arguments:   "+str(job+1)+" "+argu+"\n"  \
876	>	+"                     Destination: "+str(self.jobDestination[id])
877	>	common.logger.debug(5,msg)
878	>	common._db.updateJob_(listID,listField)
879	>	self.argsList = (len(jobParams[0])+1)
880	>
881		return
882	<
648	<	def getJobTypeArguments(self, nj, sched):
649	<	result = ''
650	<	for i in common.jobDB.arguments(nj):
651	<	result=result+str(i)+" "
652	<	return result
653	<
882	>
883		def numberOfJobs(self):
655	–	# Fabio
884		return self.total_number_of_jobs
885
886		def getTarBall(self, exe):
887		"""
888		Return the TarBall with lib and exe
889		"""
890	<
663	<	# if it exist, just return it
664	<	#
665	<	# Marco. Let's start to use relative path for Boss XML files
666	<	#
667	<	self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
890	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
891		if os.path.exists(self.tgzNameWithPath):
892		return self.tgzNameWithPath
893
#	Line 677 | Line 900 | class Cmssw(JobType):
900
901		# First of all declare the user Scram area
902		swArea = self.scram.getSWArea_()
680	–	#print "swArea = ", swArea
681	–	# swVersion = self.scram.getSWVersion()
682	–	# print "swVersion = ", swVersion
903		swReleaseTop = self.scram.getReleaseTop_()
904	<	#print "swReleaseTop = ", swReleaseTop
685	<
904	>
905		## check if working area is release top
906		if swReleaseTop == '' or swArea == swReleaseTop:
907	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
908		return
909
910		import tarfile
911		try: # create tar ball
912		tar = tarfile.open(self.tgzNameWithPath, "w:gz")
913		## First find the executable
914	<	if (executable != ''):
914	>	if (self.executable != ''):
915		exeWithPath = self.scram.findFile_(executable)
916		if ( not exeWithPath ):
917		raise CrabException('User executable '+executable+' not found')
918	<
918	>
919		## then check if it's private or not
920		if exeWithPath.find(swReleaseTop) == -1:
921		# the exe is private, so we must ship
#	Line 704 | Line 924 | class Cmssw(JobType):
924		# distinguish case when script is in user project area or given by full path somewhere else
925		if exeWithPath.find(path) >= 0 :
926		exe = string.replace(exeWithPath, path,'')
927	<	tar.add(path+exe,os.path.basename(executable))
927	>	tar.add(path+exe,exe)
928		else :
929		tar.add(exeWithPath,os.path.basename(executable))
930		pass
931		else:
932		# the exe is from release, we'll find it on WN
933		pass
934	<
934	>
935		## Now get the libraries: only those in local working area
936		libDir = 'lib'
937		lib = swArea+'/' +libDir
938		common.logger.debug(5,"lib "+lib+" to be tarred")
939		if os.path.exists(lib):
940		tar.add(lib,libDir)
941	<
941	>
942		## Now check if module dir is present
943		moduleDir = 'module'
944		module = swArea + '/' + moduleDir
#	Line 726 | Line 946 | class Cmssw(JobType):
946		tar.add(module,moduleDir)
947
948		## Now check if any data dir(s) is present
949	<	swAreaLen=len(swArea)
950	<	for root, dirs, files in os.walk(swArea):
951	<	if "data" in dirs:
952	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
953	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
954	<
955	<	## Add ProdAgent dir to tar
956	<	paDir = 'ProdAgentApi'
957	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
958	<	if os.path.isdir(pa):
959	<	tar.add(pa,paDir)
960	<
949	>	self.dataExist = False
950	>	todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
951	>	while len(todo_list):
952	>	entry, name = todo_list.pop()
953	>	if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
954	>	continue
955	>	if os.path.isdir(swArea+"/src/"+entry):
956	>	entryPath = entry + '/'
957	>	todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
958	>	if name == 'data':
959	>	self.dataExist=True
960	>	common.logger.debug(5,"data "+entry+" to be tarred")
961	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
962	>	pass
963	>	pass
964	>
965	>	### CMSSW ParameterSet
966	>	if not self.pset is None:
967	>	cfg_file = common.work_space.jobDir()+self.configFilename()
968	>	tar.add(cfg_file,self.configFilename())
969	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
970	>
971	>
972	>	## Add ProdCommon dir to tar
973	>	prodcommonDir = './'
974	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
975	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
976	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage']
977	>	for file in neededStuff:
978	>	tar.add(prodcommonPath+file,prodcommonDir+file)
979	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
980	>
981	>	##### ML stuff
982	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
983	>	path=os.environ['CRABDIR'] + '/python/'
984	>	for file in ML_file_list:
985	>	tar.add(path+file,file)
986	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
987	>
988	>	##### Utils
989	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
990	>	for file in Utils_file_list:
991	>	tar.add(path+file,file)
992		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
993	+
994	+	##### AdditionalFiles
995	+	tar.dereference=True
996	+	for file in self.additional_inbox_files:
997	+	tar.add(file,string.split(file,'/')[-1])
998	+	tar.dereference=False
999	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
1000	+
1001		tar.close()
1002	<	except :
1003	<	raise CrabException('Could not create tar-ball')
1002	>	except IOError, exc:
1003	>	common.logger.write(str(exc))
1004	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1005	>	except tarfile.TarError, exc:
1006	>	common.logger.write(str(exc))
1007	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
1008
1009		## check for tarball size
1010		tarballinfo = os.stat(self.tgzNameWithPath)
1011		if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
1012	<	raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
1012	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) \
1013	>	+'MB input sandbox limit \n'
1014	>	msg += '      and not supported by the direct GRID submission system.\n'
1015	>	msg += '      Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
1016	>	msg += '      For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#CRABSERVER_for_Users'
1017	>	raise CrabException(msg)
1018
1019		## create tar-ball with ML stuff
1020	<	self.MLtgzfile =  common.work_space.pathForTgz()+'share/MLfiles.tgz'
1021	<	try:
754	<	tar = tarfile.open(self.MLtgzfile, "w:gz")
755	<	path=os.environ['CRABDIR'] + '/python/'
756	<	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
757	<	tar.add(path+file,file)
758	<	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
759	<	tar.close()
760	<	except :
761	<	raise CrabException('Could not create ML files tar-ball')
762	<
763	<	return
764	<
765	<	def wsSetupEnvironment(self, nj):
1020	>
1021	>	def wsSetupEnvironment(self, nj=0):
1022		"""
1023		Returns part of a job script which prepares
1024		the execution environment for the job 'nj'.
1025		"""
1026	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1027	+	psetName = 'pset.py'
1028	+	else:
1029	+	psetName = 'pset.cfg'
1030		# Prepare JobType-independent part
1031	<	txt = ''
1032	<
1033	<	## OLI_Daniele at this level  middleware already known
774	<
775	<	txt += 'if [ $middleware == LCG ]; then \n'
1031	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
1032	>	txt += 'echo ">>> setup environment"\n'
1033	>	txt += 'if [ $middleware == LCG ]; then \n'
1034		txt += self.wsSetupCMSLCGEnvironment_()
1035		txt += 'elif [ $middleware == OSG ]; then\n'
1036		txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
1037	<	txt += '    echo "Created working directory: $WORKING_DIR"\n'
1038	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
1039	<	txt += '        echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1040	<	txt += '        echo "JOB_EXIT_STATUS = 10016"\n'
783	<	txt += '        echo "JobExitCode=10016" | tee -a $RUNTIME_AREA/$repo\n'
784	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
785	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
786	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
787	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
788	<	txt += '        exit 1\n'
1037	>	txt += '    if [ ! $? == 0 ] ;then\n'
1038	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
1039	>	txt += '        job_exit_code=10016\n'
1040	>	txt += '        func_exit\n'
1041		txt += '    fi\n'
1042	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
1043		txt += '\n'
1044		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
1045		txt += '    cd $WORKING_DIR\n'
1046	<	txt += self.wsSetupCMSOSGEnvironment_()
1046	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
1047	>	txt += self.wsSetupCMSOSGEnvironment_()
1048		txt += 'fi\n'
1049
1050		# Prepare JobType-specific part
1051		scram = self.scram.commandName()
1052		txt += '\n\n'
1053	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
1053	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
1054	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
1055		txt += scram+' project CMSSW '+self.version+'\n'
1056		txt += 'status=$?\n'
1057		txt += 'if [ $status != 0 ] ; then\n'
1058	<	txt += '   echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
1059	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
1060	<	txt += '   echo "JobExitCode=10034" | tee -a $RUNTIME_AREA/$repo\n'
806	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
807	<	txt += '   rm -f $RUNTIME_AREA/$repo \n'
808	<	txt += '   echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
809	<	txt += '   echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
810	<	## OLI_Daniele
811	<	txt += '    if [ $middleware == OSG ]; then \n'
812	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
813	<	txt += '        cd $RUNTIME_AREA\n'
814	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
815	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
816	<	txt += '            echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
817	<	txt += '            echo "JOB_EXIT_STATUS = 10018"\n'
818	<	txt += '            echo "JobExitCode=10018" | tee -a $RUNTIME_AREA/$repo\n'
819	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
820	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
821	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
822	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
823	<	txt += '        fi\n'
824	<	txt += '    fi \n'
825	<	txt += '   exit 1 \n'
1058	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
1059	>	txt += '    job_exit_code=10034\n'
1060	>	txt += '    func_exit\n'
1061		txt += 'fi \n'
827	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
828	–	txt += 'export SCRAM_ARCH='+self.executable_arch+'\n'
1062		txt += 'cd '+self.version+'\n'
1063	<	### needed grep for bug in scramv1 ###
1064	<	txt += scram+' runtime -sh\n'
1063	>	txt += 'SOFTWARE_DIR=`pwd`\n'
1064	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1065		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
1066	<	txt += 'echo $PATH\n'
1067	<
1066	>	txt += 'if [ $? != 0 ] ; then\n'
1067	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
1068	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
1069	>	txt += '    job_exit_code=10034\n'
1070	>	txt += '    func_exit\n'
1071	>	txt += 'fi \n'
1072		# Handle the arguments:
1073		txt += "\n"
1074		txt += "## number of arguments (first argument always jobnumber)\n"
1075		txt += "\n"
1076	<	#        txt += "narg=$#\n"
840	<	txt += "if [ $nargs -lt 2 ]\n"
1076	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
1077		txt += "then\n"
1078	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
1079	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
1080	<	txt += '    echo "JobExitCode=50113" | tee -a $RUNTIME_AREA/$repo\n'
845	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
846	<	txt += '    rm -f $RUNTIME_AREA/$repo \n'
847	<	txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
848	<	txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
849	<	## OLI_Daniele
850	<	txt += '    if [ $middleware == OSG ]; then \n'
851	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
852	<	txt += '        cd $RUNTIME_AREA\n'
853	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
854	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
855	<	txt += '            echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
856	<	txt += '            echo "JOB_EXIT_STATUS = 50114"\n'
857	<	txt += '            echo "JobExitCode=50114" | tee -a $RUNTIME_AREA/$repo\n'
858	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
859	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
860	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
861	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
862	<	txt += '        fi\n'
863	<	txt += '    fi \n'
864	<	txt += "    exit 1\n"
1078	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
1079	>	txt += '    job_exit_code=50113\n'
1080	>	txt += "    func_exit\n"
1081		txt += "fi\n"
1082		txt += "\n"
1083
1084		# Prepare job-specific part
1085		job = common.job_list[nj]
1086	<	if self.pset != None: #CarlosDaniele
1086	>	if (self.datasetPath):
1087	>	self.primaryDataset = self.datasetPath.split("/")[1]
1088	>	DataTier = self.datasetPath.split("/")[2]
1089	>	txt += '\n'
1090	>	txt += 'DatasetPath='+self.datasetPath+'\n'
1091	>
1092	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
1093	>	txt += 'DataTier='+DataTier+'\n'
1094	>	txt += 'ApplicationFamily=cmsRun\n'
1095	>
1096	>	else:
1097	>	self.primaryDataset = 'null'
1098	>	txt += 'DatasetPath=MCDataTier\n'
1099	>	txt += 'PrimaryDataset=null\n'
1100	>	txt += 'DataTier=null\n'
1101	>	txt += 'ApplicationFamily=MCDataTier\n'
1102	>	if self.pset != None:
1103		pset = os.path.basename(job.configFilename())
1104		txt += '\n'
1105	+	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
1106		if (self.datasetPath): # standard job
1107	<	#txt += 'InputFiles=$2\n'
1108	<	txt += 'InputFiles=${args[1]}\n'
1109	<	txt += 'MaxEvents=${args[2]}\n'
1110	<	txt += 'SkipEvents=${args[3]}\n'
1107	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
1108	>	if (self.useParent):
1109	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
1110	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
1111	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
1112	>	else:
1113	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
1114	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
1115		txt += 'echo "Inputfiles:<$InputFiles>"\n'
1116	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset_tmp_1.cfg\n'
1116	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
1117		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
881	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" pset_tmp_1.cfg > pset_tmp_2.cfg\n'
1118		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
883	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" pset_tmp_2.cfg > pset.cfg\n'
1119		else:  # pythia like job
1120	<	if (self.sourceSeed):
1121	<	txt += 'FirstRun=${args[1]}\n'
1120	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
1121	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
1122	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
1123	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
1124	>	if (self.firstRun):
1125	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
1126		txt += 'echo "FirstRun: <$FirstRun>"\n'
888	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" $RUNTIME_AREA/'+pset+' > tmp_1.cfg\n'
889	–	else:
890	–	txt += '# Copy untouched pset\n'
891	–	txt += 'cp $RUNTIME_AREA/'+pset+' tmp_1.cfg\n'
892	–	if (self.sourceSeed):
893	–	#                    txt += 'Seed=$2\n'
894	–	txt += 'Seed=${args[2]}\n'
895	–	txt += 'echo "Seed: <$Seed>"\n'
896	–	txt += 'sed "s#\<INPUT\>#$Seed#" tmp_1.cfg > tmp_2.cfg\n'
897	–	if (self.sourceSeedVtx):
898	–	#                        txt += 'VtxSeed=$3\n'
899	–	txt += 'VtxSeed=${args[3]}\n'
900	–	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
901	–	txt += 'sed "s#INPUTVTX#$VtxSeed#" tmp_2.cfg > pset.cfg\n'
902	–	else:
903	–	txt += 'mv tmp_2.cfg pset.cfg\n'
904	–	else:
905	–	txt += 'mv tmp_1.cfg pset.cfg\n'
906	–	# txt += '# Copy untouched pset\n'
907	–	# txt += 'cp $RUNTIME_AREA/'+pset+' pset.cfg\n'
1127
1128	+	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
1129
1130	<	if len(self.additional_inbox_files) > 0:
1131	<	for file in self.additional_inbox_files:
1132	<	relFile = file.split("/")[-1]
913	<	txt += 'if [ -e $RUNTIME_AREA/'+relFile+' ] ; then\n'
914	<	txt += '   cp $RUNTIME_AREA/'+relFile+' .\n'
915	<	txt += '   chmod +x '+relFile+'\n'
916	<	txt += 'fi\n'
917	<	pass
918	<
919	<	if self.pset != None: #CarlosDaniele
920	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
921	<
1130	>
1131	>	if self.pset != None:
1132	>	# FUTURE: Can simply for 2_1_x and higher
1133		txt += '\n'
1134	<	txt += 'echo "***** cat pset.cfg *********"\n'
1135	<	txt += 'cat pset.cfg\n'
1136	<	txt += 'echo "****** end pset.cfg ********"\n'
1134	>	if self.debug_wrapper==True:
1135	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
1136	>	txt += 'cat ' + psetName + '\n'
1137	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
1138	>	txt += '\n'
1139	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1140	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
1141	>	else:
1142	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
1143	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
1144		txt += '\n'
927	–	# txt += 'echo "***** cat pset1.cfg *********"\n'
928	–	# txt += 'cat pset1.cfg\n'
929	–	# txt += 'echo "****** end pset1.cfg ********"\n'
1145		return txt
1146
1147	<	def wsBuildExe(self, nj=0):
1147	>	def wsUntarSoftware(self, nj=0):
1148		"""
1149		Put in the script the commands to build an executable
1150		or a library.
1151		"""
1152
1153	<	txt = ""
1153	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
1154
1155		if os.path.isfile(self.tgzNameWithPath):
1156	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
1156	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
1157		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
1158	+	if  self.debug_wrapper:
1159	+	txt += 'ls -Al \n'
1160		txt += 'untar_status=$? \n'
1161		txt += 'if [ $untar_status -ne 0 ]; then \n'
1162	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
1163	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
1164	<	txt += '   echo "JobExitCode=$untar_status" | tee -a $RUNTIME_AREA/$repo\n'
948	<	txt += '   if [ $middleware == OSG ]; then \n'
949	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
950	<	txt += '       cd $RUNTIME_AREA\n'
951	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
952	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
953	<	txt += '           echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
954	<	txt += '           echo "JOB_EXIT_STATUS = 50999"\n'
955	<	txt += '           echo "JobExitCode=50999" | tee -a $RUNTIME_AREA/$repo\n'
956	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
957	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
958	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
959	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
960	<	txt += '       fi\n'
961	<	txt += '   fi \n'
962	<	txt += '   \n'
963	<	txt += '   exit 1 \n'
1162	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
1163	>	txt += '   job_exit_code=$untar_status\n'
1164	>	txt += '   func_exit\n'
1165		txt += 'else \n'
1166		txt += '   echo "Successful untar" \n'
1167		txt += 'fi \n'
1168		txt += '\n'
1169	<	txt += 'echo "Include ProdAgentApi in PYTHONPATH"\n'
1169	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1170		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1171	<	txt += '   export PYTHONPATH=ProdAgentApi\n'
1171	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1172		txt += 'else\n'
1173	<	txt += '   export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
1173	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1174	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1175		txt += 'fi\n'
1176		txt += '\n'
1177
1178		pass
1179	<
1179	>
1180		return txt
1181
1182	<	def modifySteeringCards(self, nj):
1182	>	def wsBuildExe(self, nj=0):
1183		"""
1184	<	modify the card provided by the user,
1185	<	writing a new card into share dir
1184	>	Put in the script the commands to build an executable
1185	>	or a library.
1186		"""
1187	<
1187	>
1188	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
1189	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
1190	>
1191	>	txt += 'rm -r lib/ module/ \n'
1192	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
1193	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
1194	>	if self.dataExist == True:
1195	>	txt += 'rm -r src/ \n'
1196	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
1197	>	if len(self.additional_inbox_files)>0:
1198	>	for file in self.additional_inbox_files:
1199	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1200	>	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1201	>	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1202	>
1203	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1204	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1205	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
1206	>	txt += 'else\n'
1207	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1208	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1209	>	txt += 'fi\n'
1210	>	txt += '\n'
1211	>
1212	>	return txt
1213	>
1214	>
1215		def executableName(self):
1216	<	if self.scriptExe: #CarlosDaniele
1216	>	if self.scriptExe:
1217		return "sh "
1218		else:
1219		return self.executable
1220
1221		def executableArgs(self):
1222	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1223		if self.scriptExe:#CarlosDaniele
1224		return   self.scriptExe + " $NJob"
1225	<	else:
1226	<	return " -p pset.cfg"
1225	>	else:
1226	>	ex_args = ""
1227	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1228	>	# Framework job report
1229	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1230	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1231	>	# Type of config file
1232	>	if self.CMSSW_major >= 2 :
1233	>	ex_args += " -p pset.py"
1234	>	else:
1235	>	ex_args += " -p pset.cfg"
1236	>	return ex_args
1237
1238		def inputSandbox(self, nj):
1239		"""
1240		Returns a list of filenames to be put in JDL input sandbox.
1241		"""
1242		inp_box = []
1003	–	# # dict added to delete duplicate from input sandbox file list
1004	–	# seen = {}
1005	–	## code
1243		if os.path.isfile(self.tgzNameWithPath):
1244		inp_box.append(self.tgzNameWithPath)
1245	<	if os.path.isfile(self.MLtgzfile):
1009	<	inp_box.append(self.MLtgzfile)
1010	<	## config
1011	<	if not self.pset is None:
1012	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1013	<	## additional input files
1014	<	for file in self.additional_inbox_files:
1015	<	inp_box.append(file)
1245	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
1246		return inp_box
1247
1248		def outputSandbox(self, nj):
#	Line 1023 | Line 1253 | class Cmssw(JobType):
1253
1254		## User Declared output files
1255		for out in (self.output_file+self.output_file_sandbox):
1256	<	n_out = nj + 1
1257	<	out_box.append(self.numberFile_(out,str(n_out)))
1256	>	n_out = nj + 1
1257	>	out_box.append(numberFile(out,str(n_out)))
1258		return out_box
1259
1030	–	def prepareSteeringCards(self):
1031	–	"""
1032	–	Make initial modifications of the user's steering card file.
1033	–	"""
1034	–	return
1260
1261		def wsRenameOutput(self, nj):
1262		"""
1263		Returns part of a job script which renames the produced files.
1264		"""
1265
1266	<	txt = '\n'
1267	<	txt += '# directory content\n'
1268	<	txt += 'ls \n'
1266	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1267	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1268	>	txt += 'echo ">>> current directory content:"\n'
1269	>	if self.debug_wrapper:
1270	>	txt += 'ls -Al\n'
1271	>	txt += '\n'
1272
1273	<	for fileWithSuffix in (self.output_file+self.output_file_sandbox):
1274	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1273	>	for fileWithSuffix in (self.output_file):
1274	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1275		txt += '\n'
1276		txt += '# check output file\n'
1277	<	txt += 'ls '+fileWithSuffix+'\n'
1278	<	txt += 'ls_result=$?\n'
1279	<	txt += 'if [ $ls_result -ne 0 ] ; then\n'
1280	<	txt += '   echo "ERROR: Problem with output file"\n'
1281	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1277	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1278	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1279	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
1280	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1281	>	else:
1282	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1283	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1284	>	txt += 'else\n'
1285	>	txt += '    job_exit_code=60302\n'
1286	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1287	>	if common.scheduler.name().upper() == 'CONDOR_G':
1288		txt += '    if [ $middleware == OSG ]; then \n'
1289		txt += '        echo "prepare dummy output file"\n'
1290		txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1291		txt += '    fi \n'
1058	–	txt += 'else\n'
1059	–	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1292		txt += 'fi\n'
1293	<
1294	<	txt += 'cd $RUNTIME_AREA\n'
1295	<	txt += 'cd $RUNTIME_AREA\n'
1064	<	### OLI_DANIELE
1065	<	txt += 'if [ $middleware == OSG ]; then\n'
1066	<	txt += '    cd $RUNTIME_AREA\n'
1067	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
1068	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
1069	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
1070	<	txt += '        echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1071	<	txt += '        echo "JOB_EXIT_STATUS = 60999"\n'
1072	<	txt += '        echo "JobExitCode=60999" | tee -a $RUNTIME_AREA/$repo\n'
1073	<	txt += '        dumpStatus $RUNTIME_AREA/$repo\n'
1074	<	txt += '        rm -f $RUNTIME_AREA/$repo \n'
1075	<	txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1076	<	txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1077	<	txt += '    fi\n'
1078	<	txt += 'fi\n'
1079	<	txt += '\n'
1080	<
1081	<	file_list = ''
1082	<	## Add to filelist only files to be possibly copied to SE
1083	<	for fileWithSuffix in self.output_file:
1084	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1085	<	file_list=file_list+output_file_num+' '
1086	<	file_list=file_list[:-1]
1087	<	txt += 'file_list="'+file_list+'"\n'
1293	>	file_list = []
1294	>	for fileWithSuffix in (self.output_file):
1295	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$NJob'))
1296
1297	+	txt += 'file_list="'+string.join(file_list,',')+'"\n'
1298	+	txt += '\n'
1299	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1300	+	txt += 'echo ">>> current directory content:"\n'
1301	+	if self.debug_wrapper:
1302	+	txt += 'ls -Al\n'
1303	+	txt += '\n'
1304	+	txt += 'cd $RUNTIME_AREA\n'
1305	+	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
1306		return txt
1307
1091	–	def numberFile_(self, file, txt):
1092	–	"""
1093	–	append _'txt' before last extension of a file
1094	–	"""
1095	–	p = string.split(file,".")
1096	–	# take away last extension
1097	–	name = p[0]
1098	–	for x in p[1:-1]:
1099	–	name=name+"."+x
1100	–	# add "_txt"
1101	–	if len(p)>1:
1102	–	ext = p[len(p)-1]
1103	–	result = name + '_' + txt + "." + ext
1104	–	else:
1105	–	result = name + '_' + txt
1106	–
1107	–	return result
1108	–
1308		def getRequirements(self, nj=[]):
1309		"""
1310	<	return job requirements to add to jdl files
1310	>	return job requirements to add to jdl files
1311		"""
1312		req = ''
1313		if self.version:
1314		req='Member("VO-cms-' + \
1315		self.version + \
1316		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1317	+	if self.executable_arch:
1318	+	req+=' && Member("VO-cms-' + \
1319	+	self.executable_arch + \
1320	+	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1321
1322		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1323	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1324	+	req += ' && other.GlueCEStateStatus == "Production" '
1325
1326		return req
1327
1328		def configFilename(self):
1329		""" return the config filename """
1330	<	return self.name()+'.cfg'
1330	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1331	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1332	>	return self.name()+'.py'
1333	>	else:
1334	>	return self.name()+'.cfg'
1335
1127	–	### OLI_DANIELE
1336		def wsSetupCMSOSGEnvironment_(self):
1337		"""
1338		Returns part of a job script which is prepares
1339		the execution environment and which is common for all CMS jobs.
1340		"""
1341	<	txt = '\n'
1342	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
1343	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1344	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1345	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1346	<	txt += '   elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1341	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1342	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
1343	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1344	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1345	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1346	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1347		txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1348	<	txt += '       source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1349	<	txt += '   else\n'
1350	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1351	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1352	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1353	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1146	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1147	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1148	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1149	<	txt += '       exit 1\n'
1150	<	txt += '\n'
1151	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
1152	<	txt += '       cd $RUNTIME_AREA\n'
1153	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
1154	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
1155	<	txt += '            echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1156	<	txt += '            echo "JOB_EXIT_STATUS = 10017"\n'
1157	<	txt += '            echo "JobExitCode=10017" | tee -a $RUNTIME_AREA/$repo\n'
1158	<	txt += '            dumpStatus $RUNTIME_AREA/$repo\n'
1159	<	txt += '            rm -f $RUNTIME_AREA/$repo \n'
1160	<	txt += '            echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1161	<	txt += '            echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1162	<	txt += '       fi\n'
1163	<	txt += '\n'
1164	<	txt += '       exit 1\n'
1165	<	txt += '   fi\n'
1348	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1349	>	txt += '    else\n'
1350	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1351	>	txt += '        job_exit_code=10020\n'
1352	>	txt += '        func_exit\n'
1353	>	txt += '    fi\n'
1354		txt += '\n'
1355	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1356	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1355	>	txt += '    echo "==> setup cms environment ok"\n'
1356	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1357
1358		return txt
1359	<
1172	<	### OLI_DANIELE
1359	>
1360		def wsSetupCMSLCGEnvironment_(self):
1361		"""
1362		Returns part of a job script which is prepares
1363		the execution environment and which is common for all CMS jobs.
1364		"""
1365	<	txt  = '   \n'
1366	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1367	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1368	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1369	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1370	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1371	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1372	<	txt += '       rm -f $RUNTIME_AREA/$repo \n'
1373	<	txt += '       echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1374	<	txt += '       echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1375	<	txt += '       exit 1\n'
1376	<	txt += '   else\n'
1377	<	txt += '       echo "Sourcing environment... "\n'
1378	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1379	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1380	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1381	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1382	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1383	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1384	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1385	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1386	<	txt += '           exit 1\n'
1387	<	txt += '       fi\n'
1388	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1389	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1390	<	txt += '       result=$?\n'
1391	<	txt += '       if [ $result -ne 0 ]; then\n'
1392	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1393	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1394	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1395	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1396	<	txt += '           rm -f $RUNTIME_AREA/$repo \n'
1397	<	txt += '           echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
1398	<	txt += '           echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
1399	<	txt += '           exit 1\n'
1400	<	txt += '       fi\n'
1401	<	txt += '   fi\n'
1402	<	txt += '   \n'
1403	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1404	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1365	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1366	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1367	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1368	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1369	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1370	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1371	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1372	>	txt += '        job_exit_code=10031\n'
1373	>	txt += '        func_exit\n'
1374	>	txt += '    else\n'
1375	>	txt += '        echo "Sourcing environment... "\n'
1376	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1377	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1378	>	txt += '            job_exit_code=10020\n'
1379	>	txt += '            func_exit\n'
1380	>	txt += '        fi\n'
1381	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1382	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1383	>	txt += '        result=$?\n'
1384	>	txt += '        if [ $result -ne 0 ]; then\n'
1385	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1386	>	txt += '            job_exit_code=10032\n'
1387	>	txt += '            func_exit\n'
1388	>	txt += '        fi\n'
1389	>	txt += '    fi\n'
1390	>	txt += '    \n'
1391	>	txt += '    echo "==> setup cms environment ok"\n'
1392	>	return txt
1393	>
1394	>	def wsModifyReport(self, nj):
1395	>	"""
1396	>	insert the part of the script that modifies the FrameworkJob Report
1397	>	"""
1398	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1399	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1400	>	if (publish_data == 1):
1401	>
1402	>	processedDataset = self.cfg_params['USER.publish_data_name']
1403	>
1404	>	txt += 'if [ $StageOutExitStatus -eq 0 ]; then\n'
1405	>	txt += '    FOR_LFN=$LFNBaseName\n'
1406	>	txt += 'else\n'
1407	>	txt += '    FOR_LFN=/copy_problems/ \n'
1408	>	txt += '    SE=""\n'
1409	>	txt += '    SE_PATH=""\n'
1410	>	txt += 'fi\n'
1411	>
1412	>	txt += 'echo ">>> Modify Job Report:" \n'
1413	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1414	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1415	>	#txt += 'ProcessedDataset=$procDataset \n'
1416	>	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1417	>	txt += 'echo "SE = $SE"\n'
1418	>	txt += 'echo "SE_PATH = $SE_PATH"\n'
1419	>	txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1420	>	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1421	>	args = '$RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' \
1422	>	'$USER-$ProcessedDataset-$PSETHASH $ApplicationFamily '+ \
1423	>	'  $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH'
1424	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1425	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1426	>	txt += 'modifyReport_result=$?\n'
1427	>	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1428	>	txt += '    modifyReport_result=70500\n'
1429	>	txt += '    job_exit_code=$modifyReport_result\n'
1430	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1431	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1432	>	txt += 'else\n'
1433	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1434	>	txt += 'fi\n'
1435	>	return txt
1436	>
1437	>	def wsParseFJR(self):
1438	>	"""
1439	>	Parse the FrameworkJobReport to obtain useful infos
1440	>	"""
1441	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1442	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1443	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1444	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1445	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1446	>	if self.debug_wrapper :
1447	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1448	>	txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1449	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1450	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1451	>	txt += '            cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1452	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1453	>	txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1454	>	txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1455	>	txt += '        else\n'
1456	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1457	>	txt += '        fi\n'
1458	>	txt += '    else\n'
1459	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1460	>	txt += '    fi\n'
1461	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1462	>	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1463	>	txt += '      echo ">>> Executable succeded  $executable_exit_status"\n'
1464	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1465	>	# VERIFY PROCESSED DATA
1466	>	txt += '      echo ">>> Verify list of processed files:"\n'
1467	>	txt += '      echo $InputFiles |tr -d \'\\\\\' |tr \',\' \'\\n\'|tr -d \'"\' > input-files.txt\n'
1468	>	txt += '      python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1469	>	txt += '      cat input-files.txt  | sort | uniq > tmp.txt\n'
1470	>	txt += '      mv tmp.txt input-files.txt\n'
1471	>	txt += '      echo "cat input-files.txt"\n'
1472	>	txt += '      echo "----------------------"\n'
1473	>	txt += '      cat input-files.txt\n'
1474	>	txt += '      cat processed-files.txt | sort | uniq > tmp.txt\n'
1475	>	txt += '      mv tmp.txt processed-files.txt\n'
1476	>	txt += '      echo "----------------------"\n'
1477	>	txt += '      echo "cat processed-files.txt"\n'
1478	>	txt += '      echo "----------------------"\n'
1479	>	txt += '      cat processed-files.txt\n'
1480	>	txt += '      echo "----------------------"\n'
1481	>	txt += '      diff -q input-files.txt processed-files.txt\n'
1482	>	txt += '      fileverify_status=$?\n'
1483	>	txt += '      if [ $fileverify_status -ne 0 ]; then\n'
1484	>	txt += '         executable_exit_status=30001\n'
1485	>	txt += '         echo "ERROR ==> not all input files processed"\n'
1486	>	txt += '         echo "      ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1487	>	txt += '         echo "      ==> diff input-files.txt processed-files.txt"\n'
1488	>	txt += '      fi\n'
1489	>	txt += '    elif [ $executable_exit_status -ne 0 ] || [ $executable_exit_status -ne 50015 ] || [ $executable_exit_status -ne 50017 ];then\n'
1490	>	txt += '      echo ">>> Executable failed  $executable_exit_status"\n'
1491	>	txt += '      echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1492	>	txt += '      echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1493	>	txt += '      job_exit_code=$executable_exit_status\n'
1494	>	txt += '      func_exit\n'
1495	>	txt += '    fi\n'
1496	>	txt += '\n'
1497	>	txt += 'else\n'
1498	>	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1499	>	txt += 'fi\n'
1500	>	txt += '\n'
1501	>	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1502	>	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1503	>	txt += 'job_exit_code=$executable_exit_status\n'
1504	>
1505		return txt
1506
1507		def setParam_(self, param, value):
#	Line 1223 | Line 1510 | class Cmssw(JobType):
1510		def getParams(self):
1511		return self._params
1512
1226	–	def setTaskid_(self):
1227	–	self._taskId = self.cfg_params['taskId']
1228	–
1229	–	def getTaskid(self):
1230	–	return self._taskId
1231	–
1232	–	#######################################################################
1513		def uniquelist(self, old):
1514		"""
1515		remove duplicates from a list
#	Line 1238 | Line 1518 | class Cmssw(JobType):
1518		for e in old:
1519		nd[e]=0
1520		return nd.keys()
1521	+
1522	+	def outList(self):
1523	+	"""
1524	+	check the dimension of the output files
1525	+	"""
1526	+	txt = ''
1527	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1528	+	listOutFiles = []
1529	+	stdout = 'CMSSW_$NJob.stdout'
1530	+	stderr = 'CMSSW_$NJob.stderr'
1531	+	if (self.return_data == 1):
1532	+	for file in (self.output_file+self.output_file_sandbox):
1533	+	listOutFiles.append(numberFile(file, '$NJob'))
1534	+	listOutFiles.append(stdout)
1535	+	listOutFiles.append(stderr)
1536	+	else:
1537	+	for file in (self.output_file_sandbox):
1538	+	listOutFiles.append(numberFile(file, '$NJob'))
1539	+	listOutFiles.append(stdout)
1540	+	listOutFiles.append(stderr)
1541	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1542	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1543	+	txt += 'export filesToCheck\n'
1544	+	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines