[ViewVC] Diff of: cvsroot/COMP/CRAB/python/cms

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs.
Revision 1.236 by spiga, Mon Sep 8 07:42:41 2008 UTC

#	Line 2 \| Line 2 \| from JobType import JobType
2		from crab_logger import Logger
3		from crab_exceptions import *
4		from crab_util import *
5	<	from BlackWhiteListParser import BlackWhiteListParser
5	>	from BlackWhiteListParser import SEBlackWhiteListParser
6		import common
7		import Scram
8	+	from LFNBaseName import *
9
10		import os, string, glob
11
12		class Cmssw(JobType):
13	<	def __init__(self, cfg_params, ncjobs):
13	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
14		JobType.__init__(self, 'CMSSW')
15		common.logger.debug(3,'CMSSW::__init__')
16	+	self.skip_blocks = skip_blocks
17	+
18	+	self.argsList = []
19
20		self._params = {}
21		self.cfg_params = cfg_params
18	–
22		# init BlackWhiteListParser
23	<	self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
23	>	self.blackWhiteListParser = SEBlackWhiteListParser(cfg_params)
24
25	<	try:
26	<	self.MaxTarBallSize = float(self.cfg_params['EDG.maxtarballsize'])
27	<	except KeyError:
28	<	self.MaxTarBallSize = 9.5
25	>	### Temporary patch to automatically skip the ISB size check:
26	>	server=self.cfg_params.get('CRAB.server_name',None)
27	>	size = 9.5
28	>	if server: size = 99999
29	>	### D.S.
30	>	self.MaxTarBallSize = float(self.cfg_params.get('EDG.maxtarballsize',size))
31
32		# number of jobs requested to be created, limit obj splitting
33		self.ncjobs = ncjobs
34
35		log = common.logger
36	<
36	>
37		self.scram = Scram.Scram(cfg_params)
38		self.additional_inbox_files = []
39		self.scriptExe = ''
40		self.executable = ''
41		self.executable_arch = self.scram.getArch()
42		self.tgz_name = 'default.tgz'
38	–	self.additional_tgz_name = 'additional.tgz'
43		self.scriptName = 'CMSSW.sh'
44	<	self.pset = '' #scrip use case Da
45	<	self.datasetPath = '' #scrip use case Da
44	>	self.pset = ''
45	>	self.datasetPath = ''
46
47		# set FJR file name
48		self.fjrFileName = 'crab_fjr.xml'
49
50		self.version = self.scram.getSWVersion()
51	<
52	<	#
53	<	# Try to block creation in case of arch/version mismatch
54	<	#
55	<
56	<	a = string.split(self.version, "_")
57	<
58	<	if int(a[1]) == 1 and (int(a[2]) < 5 and self.executable_arch.find('slc4') == 0):
59	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
60	<	raise CrabException(msg)
57	<	if int(a[1]) == 1 and (int(a[2]) >= 5 and self.executable_arch.find('slc3') == 0):
58	<	msg = "Error: CMS does not support %s with %s architecture"%(self.version, self.executable_arch)
51	>	version_array = self.version.split('_')
52	>	self.CMSSW_major = 0
53	>	self.CMSSW_minor = 0
54	>	self.CMSSW_patch = 0
55	>	try:
56	>	self.CMSSW_major = int(version_array[1])
57	>	self.CMSSW_minor = int(version_array[2])
58	>	self.CMSSW_patch = int(version_array[3])
59	>	except:
60	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
61		raise CrabException(msg)
60	–
61	–	common.taskDB.setDict('codeVersion',self.version)
62	–	self.setParam_('application', self.version)
62
63		### collect Data cards
64
65	<	## get DBS mode
66	<	try:
68	<	self.use_dbs_1 = int(self.cfg_params['CMSSW.use_dbs_1'])
69	<	except KeyError:
70	<	self.use_dbs_1 = 0
71	<
72	<	try:
73	<	tmp = cfg_params['CMSSW.datasetpath']
74	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
75	<	if string.lower(tmp)=='none':
76	<	self.datasetPath = None
77	<	self.selectNoInput = 1
78	<	else:
79	<	self.datasetPath = tmp
80	<	self.selectNoInput = 0
81	<	except KeyError:
82	<	msg = "Error: datasetpath not defined "
65	>	if not cfg_params.has_key('CMSSW.datasetpath'):
66	>	msg = "Error: datasetpath not defined "
67		raise CrabException(msg)
68
69	<	# ML monitoring
70	<	# split dataset path style: /PreProdR3Minbias/SIM/GEN-SIM
71	<	if not self.datasetPath:
72	<	self.setParam_('dataset', 'None')
73	<	self.setParam_('owner', 'None')
69	>	### Temporary: added to remove input file control in the case of PU
70	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
71	>
72	>	tmp = cfg_params['CMSSW.datasetpath']
73	>	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+tmp)
74	>
75	>	if tmp =='':
76	>	msg = "Error: datasetpath not defined "
77	>	raise CrabException(msg)
78	>	elif string.lower(tmp)=='none':
79	>	self.datasetPath = None
80	>	self.selectNoInput = 1
81		else:
82	<	try:
83	<	datasetpath_split = self.datasetPath.split("/")
93	<	# standard style
94	<	self.setParam_('datasetFull', self.datasetPath)
95	<	if self.use_dbs_1 == 1 :
96	<	self.setParam_('dataset', datasetpath_split[1])
97	<	self.setParam_('owner', datasetpath_split[-1])
98	<	else:
99	<	self.setParam_('dataset', datasetpath_split[1])
100	<	self.setParam_('owner', datasetpath_split[2])
101	<	except:
102	<	self.setParam_('dataset', self.datasetPath)
103	<	self.setParam_('owner', self.datasetPath)
104	<
105	<	self.setTaskid_()
106	<	self.setParam_('taskId', self.cfg_params['taskId'])
82	>	self.datasetPath = tmp
83	>	self.selectNoInput = 0
84
85		self.dataTiers = []
86	<
86	>	self.debugWrap = ''
87	>	self.debug_wrapper = cfg_params.get('USER.debug_wrapper',False)
88	>	if self.debug_wrapper: self.debugWrap='--debug'
89		## now the application
90	<	try:
91	<	self.executable = cfg_params['CMSSW.executable']
113	<	self.setParam_('exe', self.executable)
114	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
115	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
116	<	log.debug(3,msg)
117	<	except KeyError:
118	<	self.executable = 'cmsRun'
119	<	self.setParam_('exe', self.executable)
120	<	msg = "User executable not defined. Use cmsRun"
121	<	log.debug(3,msg)
122	<	pass
90	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
91	>	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
92
93	<	try:
125	<	self.pset = cfg_params['CMSSW.pset']
126	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
127	<	if self.pset.lower() != 'none' :
128	<	if (not os.path.exists(self.pset)):
129	<	raise CrabException("User defined PSet file "+self.pset+" does not exist")
130	<	else:
131	<	self.pset = None
132	<	except KeyError:
93	>	if not cfg_params.has_key('CMSSW.pset'):
94		raise CrabException("PSet file missing. Cannot run cmsRun ")
95	+	self.pset = cfg_params['CMSSW.pset']
96	+	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
97	+	if self.pset.lower() != 'none' :
98	+	if (not os.path.exists(self.pset)):
99	+	raise CrabException("User defined PSet file "+self.pset+" does not exist")
100	+	else:
101	+	self.pset = None
102
103		# output files
104		## stuff which must be returned always via sandbox
#	Line 140 \| Line 108 \| class Cmssw(JobType):
108		self.output_file_sandbox.append(self.fjrFileName)
109
110		# other output files to be returned via sandbox or copied to SE
111	<	try:
112	<	self.output_file = []
113	<	tmp = cfg_params['CMSSW.output_file']
114	<	if tmp != '':
115	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
116	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
117	<	for tmp in tmpOutFiles:
118	<	tmp=string.strip(tmp)
151	<	self.output_file.append(tmp)
152	<	pass
153	<	else:
154	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
155	<	pass
156	<	pass
157	<	except KeyError:
158	<	log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
159	<	pass
111	>	outfileflag = False
112	>	self.output_file = []
113	>	tmp = cfg_params.get('CMSSW.output_file',None)
114	>	if tmp :
115	>	self.output_file = [x.strip() for x in tmp.split(',')]
116	>	outfileflag = True #output found
117	>	#else:
118	>	# log.message("No output file defined: only stdout/err and the CRAB Framework Job Report will be available\n")
119
120		# script_exe file as additional file in inputSandbox
121	<	try:
122	<	self.scriptExe = cfg_params['USER.script_exe']
123	<	if self.scriptExe != '':
124	<	if not os.path.isfile(self.scriptExe):
125	<	msg ="ERROR. file "+self.scriptExe+" not found"
126	<	raise CrabException(msg)
168	<	self.additional_inbox_files.append(string.strip(self.scriptExe))
169	<	except KeyError:
170	<	self.scriptExe = ''
121	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
122	>	if self.scriptExe :
123	>	if not os.path.isfile(self.scriptExe):
124	>	msg ="ERROR. file "+self.scriptExe+" not found"
125	>	raise CrabException(msg)
126	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
127
172	–	#CarlosDaniele
128		if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
129	<	msg ="Error. script_exe not defined"
130	<	raise CrabException(msg)
129	>	msg ="Error. script_exe not defined"
130	>	raise CrabException(msg)
131	>
132	>	# use parent files...
133	>	self.useParent = self.cfg_params.get('CMSSW.use_parent',False)
134
135		## additional input files
136	<	try:
136	>	if cfg_params.has_key('USER.additional_input_files'):
137		tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
138		for tmp in tmpAddFiles:
139		tmp = string.strip(tmp)
#	Line 192 \| Line 150 \| class Cmssw(JobType):
150		if not os.path.exists(file):
151		raise CrabException("Additional input file not found: "+file)
152		pass
195	–	# fname = string.split(file, '/')[-1]
196	–	# storedFile = common.work_space.pathForTgz()+'share/'+fname
197	–	# shutil.copyfile(file, storedFile)
153		self.additional_inbox_files.append(string.strip(file))
154		pass
155		pass
156		common.logger.debug(5,"Additional input files: "+str(self.additional_inbox_files))
157	<	except KeyError:
203	<	pass
204	<
205	<	# files per job
206	<	try:
207	<	if (cfg_params['CMSSW.files_per_jobs']):
208	<	raise CrabException("files_per_jobs no longer supported. Quitting.")
209	<	except KeyError:
210	<	pass
157	>	pass
158
159		## Events per job
160	<	try:
160	>	if cfg_params.has_key('CMSSW.events_per_job'):
161		self.eventsPerJob =int( cfg_params['CMSSW.events_per_job'])
162		self.selectEventsPerJob = 1
163	<	except KeyError:
163	>	else:
164		self.eventsPerJob = -1
165		self.selectEventsPerJob = 0
166	<
166	>
167		## number of jobs
168	<	try:
168	>	if cfg_params.has_key('CMSSW.number_of_jobs'):
169		self.theNumberOfJobs =int( cfg_params['CMSSW.number_of_jobs'])
170		self.selectNumberOfJobs = 1
171	<	except KeyError:
171	>	else:
172		self.theNumberOfJobs = 0
173		self.selectNumberOfJobs = 0
174
175	<	try:
175	>	if cfg_params.has_key('CMSSW.total_number_of_events'):
176		self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
177		self.selectTotalNumberEvents = 1
178	<	except KeyError:
178	>	if self.selectNumberOfJobs == 1:
179	>	if (self.total_number_of_events != -1) and int(self.total_number_of_events) < int(self.theNumberOfJobs):
180	>	msg = 'Must specify at least one event per job. total_number_of_events > number_of_jobs '
181	>	raise CrabException(msg)
182	>	else:
183		self.total_number_of_events = 0
184		self.selectTotalNumberEvents = 0
185
186	<	if self.pset != None: #CarlosDaniele
186	>	if self.pset != None:
187		if ( (self.selectTotalNumberEvents + self.selectEventsPerJob + self.selectNumberOfJobs) != 2 ):
188		msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
189		raise CrabException(msg)
#	Line 241 \| Line 192 \| class Cmssw(JobType):
192		msg = 'Must specify number_of_jobs.'
193		raise CrabException(msg)
194
195	<	## source seed for pythia
196	<	try:
197	<	self.sourceSeed = int(cfg_params['CMSSW.pythia_seed'])
198	<	except KeyError:
199	<	self.sourceSeed = None
200	<	common.logger.debug(5,"No seed given")
201	<
202	<	try:
203	<	self.sourceSeedVtx = int(cfg_params['CMSSW.vtx_seed'])
204	<	except KeyError:
205	<	self.sourceSeedVtx = None
206	<	common.logger.debug(5,"No vertex seed given")
207	<
208	<	try:
209	<	self.sourceSeedG4 = int(cfg_params['CMSSW.g4_seed'])
210	<	except KeyError:
211	<	self.sourceSeedG4 = None
212	<	common.logger.debug(5,"No g4 sim hits seed given")
195	>	## New method of dealing with seeds
196	>	self.incrementSeeds = []
197	>	self.preserveSeeds = []
198	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
199	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
200	>	for tmp in tmpList:
201	>	tmp.strip()
202	>	self.preserveSeeds.append(tmp)
203	>	if cfg_params.has_key('CMSSW.increment_seeds'):
204	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
205	>	for tmp in tmpList:
206	>	tmp.strip()
207	>	self.incrementSeeds.append(tmp)
208	>
209	>	## FUTURE: Can remove in CRAB 2.4.0
210	>	self.sourceSeed = cfg_params.get('CMSSW.pythia_seed',None)
211	>	self.sourceSeedVtx = cfg_params.get('CMSSW.vtx_seed',None)
212	>	self.sourceSeedG4 = cfg_params.get('CMSSW.g4_seed',None)
213	>	self.sourceSeedMix = cfg_params.get('CMSSW.mix_seed',None)
214	>	if self.sourceSeed or self.sourceSeedVtx or self.sourceSeedG4 or self.sourceSeedMix:
215	>	msg = 'pythia_seed, vtx_seed, g4_seed, and mix_seed are no longer valid settings. You must use increment_seeds or preserve_seeds'
216	>	raise CrabException(msg)
217
218	<	try:
264	<	self.sourceSeedMix = int(cfg_params['CMSSW.mix_seed'])
265	<	except KeyError:
266	<	self.sourceSeedMix = None
267	<	common.logger.debug(5,"No mix seed given")
218	>	self.firstRun = cfg_params.get('CMSSW.first_run',None)
219
220	<	try:
221	<	self.firstRun = int(cfg_params['CMSSW.first_run'])
222	<	except KeyError:
272	<	self.firstRun = None
273	<	common.logger.debug(5,"No first run given")
274	<	if self.pset != None: #CarlosDaniele
275	<	ver = string.split(self.version,"_")
276	<	if (int(ver[1])>=1 and int(ver[2])>=5):
277	<	import PsetManipulator150 as pp
278	<	else:
279	<	import PsetManipulator as pp
280	<	PsetEdit = pp.PsetManipulator(self.pset) #Daniele Pset
220	>	# Copy/return
221	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
222	>	self.return_data = int(cfg_params.get('USER.return_data',0))
223
224		#DBSDLS-start
225	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
225	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
226		self.maxEvents=0 # max events available ( --> check the requested nb. of evts in Creator.py)
227		self.DBSPaths={} # all dbs paths requested ( --> input to the site local discovery script)
228		self.jobDestination=[] # Site destination(s) for each job (list of lists)
#	Line 289 \| Line 231 \| class Cmssw(JobType):
231		blockSites = {}
232		if self.datasetPath:
233		blockSites = self.DataDiscoveryAndLocation(cfg_params)
234	<	#DBSDLS-end
234	>	#DBSDLS-end
235
294	–	self.tgzNameWithPath = self.getTarBall(self.executable)
295	–
236		## Select Splitting
237	<	if self.selectNoInput:
238	<	if self.pset == None: #CarlosDaniele
237	>	if self.selectNoInput:
238	>	if self.pset == None:
239		self.jobSplittingForScript()
240		else:
241		self.jobSplittingNoInput()
242		else:
243		self.jobSplittingByBlocks(blockSites)
244
245	<	# modify Pset
246	<	if self.pset != None: #CarlosDaniele
247	<	try:
248	<	if (self.datasetPath): # standard job
249	<	# allow to processa a fraction of events in a file
250	<	PsetEdit.inputModule("INPUT")
251	<	PsetEdit.maxEvent("INPUTMAXEVENTS")
252	<	PsetEdit.skipEvent("INPUTSKIPEVENTS")
253	<	else: # pythia like job
245	>	# modify Pset only the first time
246	>	if isNew:
247	>	if self.pset != None:
248	>	import PsetManipulator as pp
249	>	PsetEdit = pp.PsetManipulator(self.pset)
250	>	try:
251	>	# Add FrameworkJobReport to parameter-set, set max events.
252	>	# Reset later for data jobs by writeCFG which does all modifications
253	>	PsetEdit.addCrabFJR(self.fjrFileName) # FUTURE: Job report addition not needed by CMSSW>1.5
254		PsetEdit.maxEvent(self.eventsPerJob)
255	<	if (self.firstRun):
256	<	PsetEdit.pythiaFirstRun("INPUTFIRSTRUN") #First Run
257	<	if (self.sourceSeed) :
258	<	PsetEdit.pythiaSeed("INPUT")
259	<	if (self.sourceSeedVtx) :
260	<	PsetEdit.vtxSeed("INPUTVTX")
261	<	if (self.sourceSeedG4) :
262	<	PsetEdit.g4Seed("INPUTG4")
263	<	if (self.sourceSeedMix) :
264	<	PsetEdit.mixSeed("INPUTMIX")
265	<	# add FrameworkJobReport to parameter-set
266	<	PsetEdit.addCrabFJR(self.fjrFileName)
267	<	PsetEdit.psetWriter(self.configFilename())
268	<	except:
269	<	msg='Error while manipuliating ParameterSet: exiting...'
270	<	raise CrabException(msg)
255	>	PsetEdit.psetWriter(self.configFilename())
256	>	## If present, add TFileService to output files
257	>	if not int(cfg_params.get('CMSSW.skip_TFileService_output',0)):
258	>	tfsOutput = PsetEdit.getTFileService()
259	>	if tfsOutput:
260	>	if tfsOutput in self.output_file:
261	>	common.logger.debug(5,"Output from TFileService "+tfsOutput+" already in output files")
262	>	else:
263	>	outfileflag = True #output found
264	>	self.output_file.append(tfsOutput)
265	>	common.logger.message("Adding "+tfsOutput+" to output files (from TFileService)")
266	>	pass
267	>	pass
268	>	## If present and requested, add PoolOutputModule to output files
269	>	if int(cfg_params.get('CMSSW.get_edm_output',0)):
270	>	edmOutput = PsetEdit.getPoolOutputModule()
271	>	if edmOutput:
272	>	if edmOutput in self.output_file:
273	>	common.logger.debug(5,"Output from PoolOutputModule "+edmOutput+" already in output files")
274	>	else:
275	>	self.output_file.append(edmOutput)
276	>	common.logger.message("Adding "+edmOutput+" to output files (from PoolOutputModule)")
277	>	pass
278	>	pass
279	>	except CrabException:
280	>	msg='Error while manipulating ParameterSet: exiting...'
281	>	raise CrabException(msg)
282	>	## Prepare inputSandbox TarBall (only the first time)
283	>	self.tgzNameWithPath = self.getTarBall(self.executable)
284
285		def DataDiscoveryAndLocation(self, cfg_params):
286
287		import DataDiscovery
335	–	import DataDiscovery_DBS2
288		import DataLocation
289		common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
290
#	Line 341 \| Line 293 \| class Cmssw(JobType):
293		## Contact the DBS
294		common.logger.message("Contacting Data Discovery Services ...")
295		try:
296	<
345	<	if self.use_dbs_1 == 1 :
346	<	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params)
347	<	else :
348	<	self.pubdata=DataDiscovery_DBS2.DataDiscovery_DBS2(datasetPath, cfg_params)
296	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
297		self.pubdata.fetchDBSInfo()
298
299		except DataDiscovery.NotExistingDatasetError, ex :
#	Line 357 \| Line 305 \| class Cmssw(JobType):
305		except DataDiscovery.DataDiscoveryError, ex:
306		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
307		raise CrabException(msg)
360	–	except DataDiscovery_DBS2.NotExistingDatasetError_DBS2, ex :
361	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
362	–	raise CrabException(msg)
363	–	except DataDiscovery_DBS2.NoDataTierinProvenanceError_DBS2, ex :
364	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
365	–	raise CrabException(msg)
366	–	except DataDiscovery_DBS2.DataDiscoveryError_DBS2, ex:
367	–	msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
368	–	raise CrabException(msg)
308
309		self.filesbyblock=self.pubdata.getFiles()
310		self.eventsbyblock=self.pubdata.getEventsPerBlock()
311		self.eventsbyfile=self.pubdata.getEventsPerFile()
312	+	self.parentFiles=self.pubdata.getParent()
313
314		## get max number of events
315	<	self.maxEvents=self.pubdata.getMaxEvents() ## self.maxEvents used in Creator.py
315	>	self.maxEvents=self.pubdata.getMaxEvents()
316
317		## Contact the DLS and build a list of sites hosting the fileblocks
318		try:
#	Line 381 \| Line 321 \| class Cmssw(JobType):
321		except DataLocation.DataLocationError , ex:
322		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
323		raise CrabException(msg)
324	<
324	>
325
326		sites = dataloc.getSites()
327		allSites = []
#	Line 395 \| Line 335 \| class Cmssw(JobType):
335		common.logger.message("Requested dataset: " + datasetPath + " has " + str(self.maxEvents) + " events in " + str(len(self.filesbyblock.keys())) + " blocks.\n")
336
337		return sites
338	<
338	>
339		def jobSplittingByBlocks(self, blockSites):
340		"""
341		Perform job splitting. Jobs run over an integer number of files
#	Line 445 \| Line 385 \| class Cmssw(JobType):
385		totalNumberOfJobs = 999999999
386		else :
387		totalNumberOfJobs = self.ncjobs
448	–
388
389		blocks = blockSites.keys()
390		blockCount = 0
#	Line 465 \| Line 404 \| class Cmssw(JobType):
404		blockCount += 1
405		if block not in jobsOfBlock.keys() :
406		jobsOfBlock[block] = []
407	<
407	>
408		if self.eventsbyblock.has_key(block) :
409		numEventsInBlock = self.eventsbyblock[block]
410		common.logger.debug(5,'Events in Block File '+str(numEventsInBlock))
411	<
411	>
412		files = self.filesbyblock[block]
413		numFilesInBlock = len(files)
414		if (numFilesInBlock <= 0):
#	Line 477 \| Line 416 \| class Cmssw(JobType):
416		fileCount = 0
417
418		# ---- New block => New job ---- #
419	<	parString = "\\{"
419	>	parString = ""
420		# counter for number of events in files currently worked on
421		filesEventCount = 0
422		# flag if next while loop should touch new file
423		newFile = 1
424		# job event counter
425		jobSkipEventCount = 0
426	<
426	>
427		# ---- Iterate over the files in the block until we've met the requested ---- #
428		# ---- total # of events or we've gone over all the files in this block ---- #
429	+	pString=''
430		while ( (eventsRemaining > 0) and (fileCount < numFilesInBlock) and (jobCount < totalNumberOfJobs) ):
431		file = files[fileCount]
432	+	if self.useParent:
433	+	parent = self.parentFiles[file]
434	+	for f in parent :
435	+	pString += '\\\"' + f + '\\\"\,'
436	+	common.logger.debug(6, "File "+str(file)+" has the following parents: "+str(parent))
437	+	common.logger.write("File "+str(file)+" has the following parents: "+str(parent))
438		if newFile :
439		try:
440		numEventsInFile = self.eventsbyfile[file]
#	Line 500 \| Line 446 \| class Cmssw(JobType):
446		newFile = 0
447		except KeyError:
448		common.logger.message("File "+str(file)+" has unknown number of events: skipping")
503	–
449
450	+	eventsPerJobRequested = min(eventsPerJobRequested, eventsRemaining)
451		# if less events in file remain than eventsPerJobRequested
452	<	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested ) :
452	>	if ( filesEventCount - jobSkipEventCount < eventsPerJobRequested):
453		# if last file in block
454		if ( fileCount == numFilesInBlock-1 ) :
455		# end job using last file, use remaining events in block
456		# close job and touch new file
457		fullString = parString[:-2]
458	<	fullString += '\\}'
459	<	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
458	>	if self.useParent:
459	>	fullParentString = pString[:-2]
460	>	list_of_lists.append([fullString,fullParentString,str(-1),str(jobSkipEventCount)])
461	>	else:
462	>	list_of_lists.append([fullString,str(-1),str(jobSkipEventCount)])
463		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(filesEventCount - jobSkipEventCount)+" events (last file in block).")
464		self.jobDestination.append(blockSites[block])
465		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 522 \| Line 471 \| class Cmssw(JobType):
471		eventsRemaining = eventsRemaining - filesEventCount + jobSkipEventCount
472		jobSkipEventCount = 0
473		# reset file
474	<	parString = "\\{"
474	>	pString = ""
475	>	parString = ""
476		filesEventCount = 0
477		newFile = 1
478		fileCount += 1
#	Line 534 \| Line 484 \| class Cmssw(JobType):
484		elif ( filesEventCount - jobSkipEventCount == eventsPerJobRequested ) :
485		# close job and touch new file
486		fullString = parString[:-2]
487	<	fullString += '\\}'
488	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
487	>	if self.useParent:
488	>	fullParentString = pString[:-2]
489	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
490	>	else:
491	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
492		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
493		self.jobDestination.append(blockSites[block])
494		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 546 \| Line 499 \| class Cmssw(JobType):
499		eventsRemaining = eventsRemaining - eventsPerJobRequested
500		jobSkipEventCount = 0
501		# reset file
502	<	parString = "\\{"
502	>	pString = ""
503	>	parString = ""
504		filesEventCount = 0
505		newFile = 1
506		fileCount += 1
507	<
507	>
508		# if more events in file remain than eventsPerJobRequested
509		else :
510		# close job but don't touch new file
511		fullString = parString[:-2]
512	<	fullString += '\\}'
513	<	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
512	>	if self.useParent:
513	>	fullParentString = pString[:-2]
514	>	list_of_lists.append([fullString,fullParentString,str(eventsPerJobRequested),str(jobSkipEventCount)])
515	>	else:
516	>	list_of_lists.append([fullString,str(eventsPerJobRequested),str(jobSkipEventCount)])
517		common.logger.debug(3,"Job "+str(jobCount+1)+" can run over "+str(eventsPerJobRequested)+" events.")
518		self.jobDestination.append(blockSites[block])
519		common.logger.debug(5,"Job "+str(jobCount+1)+" Destination: "+str(self.jobDestination[jobCount]))
#	Line 570 \| Line 527 \| class Cmssw(JobType):
527		jobSkipEventCount = eventsPerJobRequested - (filesEventCount - jobSkipEventCount - self.eventsbyfile[file])
528		# remove all but the last file
529		filesEventCount = self.eventsbyfile[file]
530	<	parString = "\\{"
531	<	parString += '\\\"' + file + '\\\"\,'
530	>	if self.useParent:
531	>	for f in parent : pString += '\\\"' + f + '\\\"\,'
532	>	parString = '\\\"' + file + '\\\"\,'
533		pass # END if
534		pass # END while (iterate over files in the block)
535		pass # END while (iterate over blocks in the dataset)
#	Line 579 \| Line 537 \| class Cmssw(JobType):
537		if (eventsRemaining > 0 and jobCount < totalNumberOfJobs ):
538		common.logger.message("Could not run on all requested events because some blocks not hosted at allowed sites.")
539		common.logger.message(str(jobCount)+" job(s) can run on "+str(totalEventCount)+" events.\n")
540	<
540	>
541		# screen output
542		screenOutput = "List of jobs and available destination sites:\n\n"
543
#	Line 591 \| Line 549 \| class Cmssw(JobType):
549		for block in blocks:
550		if block in jobsOfBlock.keys() :
551		blockCounter += 1
552	<	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
552	>	screenOutput += "Block %5i: jobs %20s: sites: %s\n" % (blockCounter,spanRanges(jobsOfBlock[block]),
553	>	','.join(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)))
554		if len(self.blackWhiteListParser.checkWhiteList(self.blackWhiteListParser.checkBlackList(blockSites[block],block),block)) == 0:
555	<	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
555	>	noSiteBlock.append( spanRanges(jobsOfBlock[block]) )
556		bloskNoSite.append( blockCounter )
557	<
557	>
558		common.logger.message(screenOutput)
559		if len(noSiteBlock) > 0 and len(bloskNoSite) > 0:
560		msg = 'WARNING: No sites are hosting any part of data for block:\n '
#	Line 611 \| Line 570 \| class Cmssw(JobType):
570		for range_jobs in noSiteBlock:
571		msg += str(range_jobs) + virgola
572		msg += '\n will not be submitted and this block of data can not be analyzed!\n'
573	+	if self.cfg_params.has_key('EDG.se_white_list'):
574	+	msg += 'WARNING: SE White List: '+self.cfg_params['EDG.se_white_list']+'\n'
575	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
576	+	msg += 'Please check if the dataset is available at this site!)\n'
577	+	if self.cfg_params.has_key('EDG.ce_white_list'):
578	+	msg += 'WARNING: CE White List: '+self.cfg_params['EDG.ce_white_list']+'\n'
579	+	msg += '(Hint: By whitelisting you force the job to run at this particular site(s).\n'
580	+	msg += 'Please check if the dataset is available at this site!)\n'
581	+
582		common.logger.message(msg)
583
584		self.list_of_args = list_of_lists
#	Line 621 \| Line 589 \| class Cmssw(JobType):
589		Perform job splitting based on number of event per job
590		"""
591		common.logger.debug(5,'Splitting per events')
592	<	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
593	<	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
594	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
592	>
593	>	if (self.selectEventsPerJob):
594	>	common.logger.message('Required '+str(self.eventsPerJob)+' events per job ')
595	>	if (self.selectNumberOfJobs):
596	>	common.logger.message('Required '+str(self.theNumberOfJobs)+' jobs in total ')
597	>	if (self.selectTotalNumberEvents):
598	>	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
599
600		if (self.total_number_of_events < 0):
601		msg='Cannot split jobs per Events with "-1" as total number of events'
#	Line 632 \| Line 604 \| class Cmssw(JobType):
604		if (self.selectEventsPerJob):
605		if (self.selectTotalNumberEvents):
606		self.total_number_of_jobs = int(self.total_number_of_events/self.eventsPerJob)
607	<	elif(self.selectNumberOfJobs) :
607	>	elif(self.selectNumberOfJobs) :
608		self.total_number_of_jobs =self.theNumberOfJobs
609	<	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
609	>	self.total_number_of_events =int(self.theNumberOfJobs*self.eventsPerJob)
610
611		elif (self.selectNumberOfJobs) :
612		self.total_number_of_jobs = self.theNumberOfJobs
613		self.eventsPerJob = int(self.total_number_of_events/self.total_number_of_jobs)
614	<
614	>
615		common.logger.debug(5,'N jobs '+str(self.total_number_of_jobs))
616
617		# is there any remainder?
#	Line 655 \| Line 627 \| class Cmssw(JobType):
627		self.list_of_args = []
628		for i in range(self.total_number_of_jobs):
629		## Since there is no input, any site is good
630	<	# self.jobDestination.append(["Any"])
659	<	self.jobDestination.append([""]) #must be empty to write correctly the xml
630	>	self.jobDestination.append([""]) #must be empty to write correctly the xml
631		args=[]
632		if (self.firstRun):
633	<	## pythia first run
663	<	#self.list_of_args.append([(str(self.firstRun)+str(i))])
633	>	## pythia first run
634		args.append(str(self.firstRun)+str(i))
665	–	else:
666	–	## no first run
667	–	#self.list_of_args.append([str(i)])
668	–	args.append(str(i))
669	–	if (self.sourceSeed):
670	–	args.append(str(self.sourceSeed)+str(i))
671	–	if (self.sourceSeedVtx):
672	–	## + vtx random seed
673	–	args.append(str(self.sourceSeedVtx)+str(i))
674	–	if (self.sourceSeedG4):
675	–	## + G4 random seed
676	–	args.append(str(self.sourceSeedG4)+str(i))
677	–	if (self.sourceSeedMix):
678	–	## + Mix random seed
679	–	args.append(str(self.sourceSeedMix)+str(i))
680	–	pass
681	–	pass
635		self.list_of_args.append(args)
683	–	pass
684	–
685	–	# print self.list_of_args
636
637		return
638
639
640	<	def jobSplittingForScript(self):#CarlosDaniele
640	>	def jobSplittingForScript(self):
641		"""
642		Perform job splitting based on number of job
643		"""
#	Line 703 \| Line 653 \| class Cmssw(JobType):
653		# argument is seed number.$i
654		self.list_of_args = []
655		for i in range(self.total_number_of_jobs):
706	–	## Since there is no input, any site is good
707	–	# self.jobDestination.append(["Any"])
656		self.jobDestination.append([""])
709	–	## no random seed
657		self.list_of_args.append([str(i)])
658		return
659
660	<	def split(self, jobParams):
661	<
715	<	common.jobDB.load()
716	<	#### Fabio
660	>	def split(self, jobParams,firstJobID):
661	>
662		njobs = self.total_number_of_jobs
663		arglist = self.list_of_args
664		# create the empty structure
665		for i in range(njobs):
666		jobParams.append("")
722	–
723	–	for job in range(njobs):
724	–	jobParams[job] = arglist[job]
725	–	# print str(arglist[job])
726	–	# print jobParams[job]
727	–	common.jobDB.setArguments(job, jobParams[job])
728	–	common.logger.debug(5,"Job "+str(job)+" Destination: "+str(self.jobDestination[job]))
729	–	common.jobDB.setDestination(job, self.jobDestination[job])
667
668	<	common.jobDB.save()
668	>	listID=[]
669	>	listField=[]
670	>	for id in range(njobs):
671	>	job = id + int(firstJobID)
672	>	jobParams[id] = arglist[id]
673	>	listID.append(job+1)
674	>	job_ToSave ={}
675	>	concString = ' '
676	>	argu=''
677	>	if len(jobParams[id]):
678	>	argu += concString.join(jobParams[id] )
679	>	job_ToSave['arguments']= str(job+1)+' '+argu
680	>	job_ToSave['dlsDestination']= self.jobDestination[id]
681	>	listField.append(job_ToSave)
682	>	msg="Job "+str(job)+" Arguments: "+str(job+1)+" "+argu+"\n" \
683	>	+" Destination: "+str(self.jobDestination[id])
684	>	common.logger.debug(5,msg)
685	>	common._db.updateJob_(listID,listField)
686	>	self.argsList = (len(jobParams[0])+1)
687	>
688		return
689	<
734	<	def getJobTypeArguments(self, nj, sched):
735	<	result = ''
736	<	for i in common.jobDB.arguments(nj):
737	<	result=result+str(i)+" "
738	<	return result
739	<
689	>
690		def numberOfJobs(self):
741	–	# Fabio
691		return self.total_number_of_jobs
692
693		def getTarBall(self, exe):
694		"""
695		Return the TarBall with lib and exe
696		"""
748	–
749	–	# if it exist, just return it
750	–	#
751	–	# Marco. Let's start to use relative path for Boss XML files
752	–	#
697		self.tgzNameWithPath = common.work_space.pathForTgz()+'share/'+self.tgz_name
698		if os.path.exists(self.tgzNameWithPath):
699		return self.tgzNameWithPath
#	Line 763 \| Line 707 \| class Cmssw(JobType):
707
708		# First of all declare the user Scram area
709		swArea = self.scram.getSWArea_()
766	–	#print "swArea = ", swArea
767	–	# swVersion = self.scram.getSWVersion()
768	–	# print "swVersion = ", swVersion
710		swReleaseTop = self.scram.getReleaseTop_()
711	<	#print "swReleaseTop = ", swReleaseTop
771	<
711	>
712		## check if working area is release top
713		if swReleaseTop == '' or swArea == swReleaseTop:
714	+	common.logger.debug(3,"swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
715		return
716
717		import tarfile
#	Line 781 \| Line 722 \| class Cmssw(JobType):
722		exeWithPath = self.scram.findFile_(executable)
723		if ( not exeWithPath ):
724		raise CrabException('User executable '+executable+' not found')
725	<
725	>
726		## then check if it's private or not
727		if exeWithPath.find(swReleaseTop) == -1:
728		# the exe is private, so we must ship
#	Line 790 \| Line 731 \| class Cmssw(JobType):
731		# distinguish case when script is in user project area or given by full path somewhere else
732		if exeWithPath.find(path) >= 0 :
733		exe = string.replace(exeWithPath, path,'')
734	<	tar.add(path+exe,os.path.basename(executable))
734	>	tar.add(path+exe,exe)
735		else :
736		tar.add(exeWithPath,os.path.basename(executable))
737		pass
738		else:
739		# the exe is from release, we'll find it on WN
740		pass
741	<
741	>
742		## Now get the libraries: only those in local working area
743		libDir = 'lib'
744		lib = swArea+'/' +libDir
745		common.logger.debug(5,"lib "+lib+" to be tarred")
746		if os.path.exists(lib):
747		tar.add(lib,libDir)
748	<
748	>
749		## Now check if module dir is present
750		moduleDir = 'module'
751		module = swArea + '/' + moduleDir
#	Line 812 \| Line 753 \| class Cmssw(JobType):
753		tar.add(module,moduleDir)
754
755		## Now check if any data dir(s) is present
756	<	swAreaLen=len(swArea)
757	<	for root, dirs, files in os.walk(swArea):
758	<	if "data" in dirs:
759	<	common.logger.debug(5,"data "+root+"/data"+" to be tarred")
760	<	tar.add(root+"/data",root[swAreaLen:]+"/data")
761	<
762	<	## Add ProdAgent dir to tar
763	<	paDir = 'ProdAgentApi'
764	<	pa = os.environ['CRABDIR'] + '/' + 'ProdAgentApi'
765	<	if os.path.isdir(pa):
766	<	tar.add(pa,paDir)
767	<
768	<	### FEDE FOR DBS PUBLICATION
769	<	## Add PRODCOMMON dir to tar
770	<	prodcommonDir = 'ProdCommon'
771	<	prodcommonPath = os.environ['CRABDIR'] + '/' + 'ProdCommon'
772	<	if os.path.isdir(prodcommonPath):
773	<	tar.add(prodcommonPath,prodcommonDir)
774	<	#############################
775	<
756	>	self.dataExist = False
757	>	todo_list = [(i, i) for i in os.listdir(swArea+"/src")]
758	>	while len(todo_list):
759	>	entry, name = todo_list.pop()
760	>	if name.startswith('crab_0_') or name.startswith('.') or name == 'CVS':
761	>	continue
762	>	if os.path.isdir(swArea+"/src/"+entry):
763	>	entryPath = entry + '/'
764	>	todo_list += [(entryPath + i, i) for i in os.listdir(swArea+"/src/"+entry)]
765	>	if name == 'data':
766	>	self.dataExist=True
767	>	common.logger.debug(5,"data "+entry+" to be tarred")
768	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
769	>	pass
770	>	pass
771	>
772	>	### CMSSW ParameterSet
773	>	if not self.pset is None:
774	>	cfg_file = common.work_space.jobDir()+self.configFilename()
775	>	tar.add(cfg_file,self.configFilename())
776	>	common.logger.debug(5,"File added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
777	>
778	>
779	>	## Add ProdCommon dir to tar
780	>	prodcommonDir = './'
781	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
782	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools','ProdCommon/Core','ProdCommon/MCPayloads', 'IMProv']
783	>	for file in neededStuff:
784	>	tar.add(prodcommonPath+file,prodcommonDir+file)
785	>	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
786	>
787	>	##### ML stuff
788	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
789	>	path=os.environ['CRABDIR'] + '/python/'
790	>	for file in ML_file_list:
791	>	tar.add(path+file,file)
792		common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
793	+
794	+	##### Utils
795	+	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py']
796	+	for file in Utils_file_list:
797	+	tar.add(path+file,file)
798	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
799	+
800	+	##### AdditionalFiles
801	+	for file in self.additional_inbox_files:
802	+	tar.add(file,string.split(file,'/')[-1])
803	+	common.logger.debug(5,"Files added to "+self.tgzNameWithPath+" : "+str(tar.getnames()))
804	+
805		tar.close()
806	<	except :
807	<	raise CrabException('Could not create tar-ball')
806	>	except IOError:
807	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
808	>	except tarfile.TarError:
809	>	raise CrabException('Could not create tar-ball '+self.tgzNameWithPath)
810
811		## check for tarball size
812		tarballinfo = os.stat(self.tgzNameWithPath)
#	Line 843 \| Line 814 \| class Cmssw(JobType):
814		raise CrabException('Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + str(self.MaxTarBallSize) + ' MB input sandbox limit and not supported by the used GRID submission system. Please make sure that no unnecessary files are in all data directories in your local CMSSW project area as they are automatically packed into the input sandbox.')
815
816		## create tar-ball with ML stuff
846	–	self.MLtgzfile = common.work_space.pathForTgz()+'share/MLfiles.tgz'
847	–	try:
848	–	tar = tarfile.open(self.MLtgzfile, "w:gz")
849	–	path=os.environ['CRABDIR'] + '/python/'
850	–	for file in ['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py', 'parseCrabFjr.py']:
851	–	tar.add(path+file,file)
852	–	common.logger.debug(5,"Files added to "+self.MLtgzfile+" : "+str(tar.getnames()))
853	–	tar.close()
854	–	except :
855	–	raise CrabException('Could not create ML files tar-ball')
856	–
857	–	return
858	–
859	–	def additionalInputFileTgz(self):
860	–	"""
861	–	Put all additional files into a tar ball and return its name
862	–	"""
863	–	import tarfile
864	–	tarName= common.work_space.pathForTgz()+'share/'+self.additional_tgz_name
865	–	tar = tarfile.open(tarName, "w:gz")
866	–	for file in self.additional_inbox_files:
867	–	tar.add(file,string.split(file,'/')[-1])
868	–	common.logger.debug(5,"Files added to "+self.additional_tgz_name+" : "+str(tar.getnames()))
869	–	tar.close()
870	–	return tarName
817
818	<	def wsSetupEnvironment(self, nj):
818	>	def wsSetupEnvironment(self, nj=0):
819		"""
820		Returns part of a job script which prepares
821		the execution environment for the job 'nj'.
822		"""
823	+	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
824	+	psetName = 'pset.py'
825	+	else:
826	+	psetName = 'pset.cfg'
827		# Prepare JobType-independent part
828	<	txt = ''
829	<
830	<	## OLI_Daniele at this level middleware already known
881	<
882	<	txt += 'if [ $middleware == LCG ]; then \n'
883	<	txt += ' echo "### First set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
884	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
885	<	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
828	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
829	>	txt += 'echo ">>> setup environment"\n'
830	>	txt += 'if [ $middleware == LCG ]; then \n'
831		txt += self.wsSetupCMSLCGEnvironment_()
832		txt += 'elif [ $middleware == OSG ]; then\n'
833		txt += ' WORKING_DIR=`/bin/mktemp -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
834	<	txt += ' echo "Created working directory: $WORKING_DIR"\n'
835	<	txt += ' if [ ! -d $WORKING_DIR ] ;then\n'
836	<	txt += ' echo "SET_CMS_ENV 10016 ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
837	<	txt += ' echo "JOB_EXIT_STATUS = 10016"\n'
893	<	txt += ' echo "JobExitCode=10016" \| tee -a $RUNTIME_AREA/$repo\n'
894	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
895	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
896	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
897	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
898	<	txt += ' exit 1\n'
834	>	txt += ' if [ ! $? == 0 ] ;then\n'
835	>	txt += ' echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
836	>	txt += ' job_exit_code=10016\n'
837	>	txt += ' func_exit\n'
838		txt += ' fi\n'
839	+	txt += ' echo ">>> Created working directory: $WORKING_DIR"\n'
840		txt += '\n'
841		txt += ' echo "Change to working directory: $WORKING_DIR"\n'
842		txt += ' cd $WORKING_DIR\n'
843	<	txt += self.wsSetupCMSOSGEnvironment_()
844	<	txt += ' echo "### Set SCRAM ARCH to ' + self.executable_arch + ' ###"\n'
905	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
843	>	txt += ' echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
844	>	txt += self.wsSetupCMSOSGEnvironment_()
845		txt += 'fi\n'
846
847		# Prepare JobType-specific part
848		scram = self.scram.commandName()
849		txt += '\n\n'
850	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
850	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
851	>	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
852		txt += scram+' project CMSSW '+self.version+'\n'
853		txt += 'status=$?\n'
854		txt += 'if [ $status != 0 ] ; then\n'
855	<	txt += ' echo "SET_EXE_ENV 10034 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
856	<	txt += ' echo "JOB_EXIT_STATUS = 10034"\n'
857	<	txt += ' echo "JobExitCode=10034" \| tee -a $RUNTIME_AREA/$repo\n'
918	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
919	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
920	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
921	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
922	<	## OLI_Daniele
923	<	txt += ' if [ $middleware == OSG ]; then \n'
924	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
925	<	txt += ' cd $RUNTIME_AREA\n'
926	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
927	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
928	<	txt += ' echo "SET_CMS_ENV 10018 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after CMSSW CMSSW_0_6_1 not found on `hostname`"\n'
929	<	txt += ' echo "JOB_EXIT_STATUS = 10018"\n'
930	<	txt += ' echo "JobExitCode=10018" \| tee -a $RUNTIME_AREA/$repo\n'
931	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
932	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
933	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
934	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
935	<	txt += ' fi\n'
936	<	txt += ' fi \n'
937	<	txt += ' exit 1 \n'
855	>	txt += ' echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
856	>	txt += ' job_exit_code=10034\n'
857	>	txt += ' func_exit\n'
858		txt += 'fi \n'
939	–	txt += 'echo "CMSSW_VERSION = '+self.version+'"\n'
859		txt += 'cd '+self.version+'\n'
941	–	########## FEDE FOR DBS2 ######################
860		txt += 'SOFTWARE_DIR=`pwd`\n'
861	<	txt += 'echo SOFTWARE_DIR=$SOFTWARE_DIR \n'
944	<	###############################################
945	<	### needed grep for bug in scramv1 ###
946	<	txt += scram+' runtime -sh\n'
861	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
862		txt += 'eval `'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME`\n'
863	<	txt += 'echo $PATH\n'
864	<
863	>	txt += 'if [ $? != 0 ] ; then\n'
864	>	txt += ' echo "ERROR ==> Problem with the command: "\n'
865	>	txt += ' echo "eval \`'+scram+' runtime -sh \| grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
866	>	txt += ' job_exit_code=10034\n'
867	>	txt += ' func_exit\n'
868	>	txt += 'fi \n'
869		# Handle the arguments:
870		txt += "\n"
871		txt += "## number of arguments (first argument always jobnumber)\n"
872		txt += "\n"
873	<	# txt += "narg=$#\n"
955	<	txt += "if [ $nargs -lt 2 ]\n"
873	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
874		txt += "then\n"
875	<	txt += " echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$nargs+ \n"
876	<	txt += ' echo "JOB_EXIT_STATUS = 50113"\n'
877	<	txt += ' echo "JobExitCode=50113" \| tee -a $RUNTIME_AREA/$repo\n'
960	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
961	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
962	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
963	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
964	<	## OLI_Daniele
965	<	txt += ' if [ $middleware == OSG ]; then \n'
966	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
967	<	txt += ' cd $RUNTIME_AREA\n'
968	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
969	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
970	<	txt += ' echo "SET_EXE_ENV 50114 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Too few arguments for CRAB job wrapper"\n'
971	<	txt += ' echo "JOB_EXIT_STATUS = 50114"\n'
972	<	txt += ' echo "JobExitCode=50114" \| tee -a $RUNTIME_AREA/$repo\n'
973	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
974	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
975	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
976	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
977	<	txt += ' fi\n'
978	<	txt += ' fi \n'
979	<	txt += " exit 1\n"
875	>	txt += " echo 'ERROR ==> Too few arguments' +$nargs+ \n"
876	>	txt += ' job_exit_code=50113\n'
877	>	txt += " func_exit\n"
878		txt += "fi\n"
879		txt += "\n"
880
881		# Prepare job-specific part
882		job = common.job_list[nj]
883	<	### FEDE FOR DBS OUTPUT PUBLICATION
986	<	if (self.datasetPath):
883	>	if (self.datasetPath):
884		txt += '\n'
885		txt += 'DatasetPath='+self.datasetPath+'\n'
886
887		datasetpath_split = self.datasetPath.split("/")
888	<
888	>	### FEDE FOR NEW LFN ###
889	>	self.primaryDataset = datasetpath_split[1]
890	>	########################
891		txt += 'PrimaryDataset='+datasetpath_split[1]+'\n'
892		txt += 'DataTier='+datasetpath_split[2]+'\n'
994	–	#txt += 'ProcessedDataset='+datasetpath_split[3]+'\n'
893		txt += 'ApplicationFamily=cmsRun\n'
894
895		else:
896		txt += 'DatasetPath=MCDataTier\n'
897	+	### FEDE FOR NEW LFN ###
898	+	self.primaryDataset = 'null'
899	+	########################
900		txt += 'PrimaryDataset=null\n'
901		txt += 'DataTier=null\n'
1001	–	#txt += 'ProcessedDataset=null\n'
902		txt += 'ApplicationFamily=MCDataTier\n'
903	<	if self.pset != None: #CarlosDaniele
903	>	if self.pset != None:
904		pset = os.path.basename(job.configFilename())
905		txt += '\n'
906		txt += 'cp $RUNTIME_AREA/'+pset+' .\n'
907		if (self.datasetPath): # standard job
908	<	#txt += 'InputFiles=$2\n'
909	<	txt += 'InputFiles=${args[1]}\n'
910	<	txt += 'MaxEvents=${args[2]}\n'
911	<	txt += 'SkipEvents=${args[3]}\n'
908	>	txt += 'InputFiles=${args[1]}; export InputFiles\n'
909	>	if (self.useParent):
910	>	txt += 'ParentFiles=${args[2]}; export ParentFiles\n'
911	>	txt += 'MaxEvents=${args[3]}; export MaxEvents\n'
912	>	txt += 'SkipEvents=${args[4]}; export SkipEvents\n'
913	>	else:
914	>	txt += 'MaxEvents=${args[2]}; export MaxEvents\n'
915	>	txt += 'SkipEvents=${args[3]}; export SkipEvents\n'
916		txt += 'echo "Inputfiles:<$InputFiles>"\n'
917	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
917	>	if (self.useParent): txt += 'echo "ParentFiles:<$ParentFiles>"\n'
918		txt += 'echo "MaxEvents:<$MaxEvents>"\n'
1015	–	txt += 'sed "s#INPUTMAXEVENTS#$MaxEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
919		txt += 'echo "SkipEvents:<$SkipEvents>"\n'
1017	–	txt += 'sed "s#INPUTSKIPEVENTS#$SkipEvents#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
920		else: # pythia like job
921	<	seedIndex=1
921	>	txt += 'PreserveSeeds=' + ','.join(self.preserveSeeds) + '; export PreserveSeeds\n'
922	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
923	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
924	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
925		if (self.firstRun):
926	<	txt += 'FirstRun=${args['+str(seedIndex)+']}\n'
926	>	txt += 'FirstRun=${args[1]}; export FirstRun\n'
927		txt += 'echo "FirstRun: <$FirstRun>"\n'
1023	–	txt += 'sed "s#\<INPUTFIRSTRUN\>#$FirstRun#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1024	–	seedIndex=seedIndex+1
928
929	<	if (self.sourceSeed):
1027	<	txt += 'Seed=${args['+str(seedIndex)+']}\n'
1028	<	txt += 'sed "s#\<INPUT\>#$Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1029	<	seedIndex=seedIndex+1
1030	<	## the following seeds are not always present
1031	<	if (self.sourceSeedVtx):
1032	<	txt += 'VtxSeed=${args['+str(seedIndex)+']}\n'
1033	<	txt += 'echo "VtxSeed: <$VtxSeed>"\n'
1034	<	txt += 'sed "s#\<INPUTVTX\>#$VtxSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1035	<	seedIndex += 1
1036	<	if (self.sourceSeedG4):
1037	<	txt += 'G4Seed=${args['+str(seedIndex)+']}\n'
1038	<	txt += 'echo "G4Seed: <$G4Seed>"\n'
1039	<	txt += 'sed "s#\<INPUTG4\>#$G4Seed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1040	<	seedIndex += 1
1041	<	if (self.sourceSeedMix):
1042	<	txt += 'mixSeed=${args['+str(seedIndex)+']}\n'
1043	<	txt += 'echo "MixSeed: <$mixSeed>"\n'
1044	<	txt += 'sed "s#\<INPUTMIX\>#$mixSeed#" '+pset+' > tmp && mv -f tmp '+pset+'\n'
1045	<	seedIndex += 1
1046	<	pass
1047	<	pass
1048	<	txt += 'mv -f '+pset+' pset.cfg\n'
929	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
930
1050	–	if len(self.additional_inbox_files) > 0:
1051	–	txt += 'if [ -e $RUNTIME_AREA/'+self.additional_tgz_name+' ] ; then\n'
1052	–	txt += ' tar xzvf $RUNTIME_AREA/'+self.additional_tgz_name+'\n'
1053	–	txt += 'fi\n'
1054	–	pass
931
932	<	if self.pset != None: #CarlosDaniele
933	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
1058	<
1059	<	txt += '\n'
1060	<	txt += 'echo "*** cat pset.cfg *******"\n'
1061	<	txt += 'cat pset.cfg\n'
1062	<	txt += 'echo "**** end pset.cfg ******"\n'
932	>	if self.pset != None:
933	>	# FUTURE: Can simply for 2_1_x and higher
934		txt += '\n'
935	<	### FEDE FOR DBS OUTPUT PUBLICATION
936	<	txt += 'PSETHASH=`EdmConfigHash < pset.cfg` \n'
935	>	if self.debug_wrapper==True:
936	>	txt += 'echo "*** cat ' + psetName + ' *******"\n'
937	>	txt += 'cat ' + psetName + '\n'
938	>	txt += 'echo "**** end ' + psetName + ' ******"\n'
939	>	txt += '\n'
940	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
941	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
942	>	else:
943	>	txt += 'PSETHASH=`edmConfigHash < ' + psetName + '` \n'
944		txt += 'echo "PSETHASH = $PSETHASH" \n'
1067	–	##############
945		txt += '\n'
1069	–	# txt += 'echo "*** cat pset1.cfg *******"\n'
1070	–	# txt += 'cat pset1.cfg\n'
1071	–	# txt += 'echo "**** end pset1.cfg ******"\n'
946		return txt
947
948	<	def wsBuildExe(self, nj=0):
948	>	def wsUntarSoftware(self, nj=0):
949		"""
950		Put in the script the commands to build an executable
951		or a library.
952		"""
953
954	<	txt = ""
954	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
955
956		if os.path.isfile(self.tgzNameWithPath):
957	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
957	>	txt += 'echo ">>> tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
958		txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
959	+	if self.debug_wrapper:
960	+	txt += 'ls -Al \n'
961		txt += 'untar_status=$? \n'
962		txt += 'if [ $untar_status -ne 0 ]; then \n'
963	<	txt += ' echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
964	<	txt += ' echo "JOB_EXIT_STATUS = $untar_status" \n'
965	<	txt += ' echo "JobExitCode=$untar_status" \| tee -a $RUNTIME_AREA/$repo\n'
1090	<	txt += ' if [ $middleware == OSG ]; then \n'
1091	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1092	<	txt += ' cd $RUNTIME_AREA\n'
1093	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1094	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1095	<	txt += ' echo "SET_EXE 50999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after Untarring .tgz file failed"\n'
1096	<	txt += ' echo "JOB_EXIT_STATUS = 50999"\n'
1097	<	txt += ' echo "JobExitCode=50999" \| tee -a $RUNTIME_AREA/$repo\n'
1098	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1099	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1100	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1101	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1102	<	txt += ' fi\n'
1103	<	txt += ' fi \n'
1104	<	txt += ' \n'
1105	<	txt += ' exit 1 \n'
963	>	txt += ' echo "ERROR ==> Untarring .tgz file failed"\n'
964	>	txt += ' job_exit_code=$untar_status\n'
965	>	txt += ' func_exit\n'
966		txt += 'else \n'
967		txt += ' echo "Successful untar" \n'
968		txt += 'fi \n'
969		txt += '\n'
970	<	txt += 'echo "Include ProdAgentApi and PRODCOMMON in PYTHONPATH"\n'
970	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
971		txt += 'if [ -z "$PYTHONPATH" ]; then\n'
972	<	#### FEDE FOR DBS OUTPUT PUBLICATION
1113	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon\n'
1114	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon\n'
1115	<	#txt += ' export PYTHONPATH=ProdAgentApi\n'
972	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
973		txt += 'else\n'
974	<	txt += ' export PYTHONPATH=$SOFTWARE_DIR/ProdAgentApi:$SOFTWARE_DIR/ProdCommon:${PYTHONPATH}\n'
1118	<	#txt += ' export PYTHONPATH=`pwd`/ProdAgentApi:`pwd`/ProdCommon:${PYTHONPATH}\n'
1119	<	#txt += ' export PYTHONPATH=ProdAgentApi:${PYTHONPATH}\n'
974	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
975		txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1121	–	###################
976		txt += 'fi\n'
977		txt += '\n'
978
979		pass
980	<
980	>
981		return txt
982
983	<	def modifySteeringCards(self, nj):
983	>	def wsBuildExe(self, nj=0):
984		"""
985	<	modify the card provided by the user,
986	<	writing a new card into share dir
985	>	Put in the script the commands to build an executable
986	>	or a library.
987		"""
988	<
988	>
989	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
990	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
991	>
992	>	txt += 'rm -r lib/ module/ \n'
993	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
994	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
995	>	if self.dataExist == True:
996	>	txt += 'rm -r src/ \n'
997	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
998	>	if len(self.additional_inbox_files)>0:
999	>	for file in self.additional_inbox_files:
1000	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
1001	>	# txt += 'mv $RUNTIME_AREA/ProdCommon/ . \n'
1002	>	# txt += 'mv $RUNTIME_AREA/IMProv/ . \n'
1003	>
1004	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
1005	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
1006	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/\n'
1007	>	txt += 'else\n'
1008	>	txt += ' export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
1009	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
1010	>	txt += 'fi\n'
1011	>	txt += '\n'
1012	>
1013	>	return txt
1014	>
1015	>
1016		def executableName(self):
1017	<	if self.scriptExe: #CarlosDaniele
1017	>	if self.scriptExe:
1018		return "sh "
1019		else:
1020		return self.executable
1021
1022		def executableArgs(self):
1023	+	# FUTURE: This function tests the CMSSW version. Can be simplified as we drop support for old versions
1024		if self.scriptExe:#CarlosDaniele
1025		return self.scriptExe + " $NJob"
1026		else:
1027	<	# if >= CMSSW_1_5_X, add -e
1028	<	version_array = self.scram.getSWVersion().split('_')
1029	<	major = 0
1030	<	minor = 0
1031	<	try:
1032	<	major = int(version_array[1])
1033	<	minor = int(version_array[2])
1034	<	except:
1153	<	msg = "Cannot parse CMSSW version string: " + "_".join(version_array) + " for major and minor release number!"
1154	<	raise CrabException(msg)
1155	<	if major >= 1 and minor >= 5 :
1156	<	return " -e -p pset.cfg"
1027	>	ex_args = ""
1028	>	# FUTURE: This tests the CMSSW version. Can remove code as versions deprecated
1029	>	# Framework job report
1030	>	if (self.CMSSW_major >= 1 and self.CMSSW_minor >= 5) or (self.CMSSW_major >= 2):
1031	>	ex_args += " -j $RUNTIME_AREA/crab_fjr_$NJob.xml"
1032	>	# Type of config file
1033	>	if self.CMSSW_major >= 2 :
1034	>	ex_args += " -p pset.py"
1035		else:
1036	<	return " -p pset.cfg"
1036	>	ex_args += " -p pset.cfg"
1037	>	return ex_args
1038
1039		def inputSandbox(self, nj):
1040		"""
1041		Returns a list of filenames to be put in JDL input sandbox.
1042		"""
1043		inp_box = []
1165	–	# # dict added to delete duplicate from input sandbox file list
1166	–	# seen = {}
1167	–	## code
1044		if os.path.isfile(self.tgzNameWithPath):
1045		inp_box.append(self.tgzNameWithPath)
1046	<	if os.path.isfile(self.MLtgzfile):
1047	<	inp_box.append(self.MLtgzfile)
1172	<	## config
1173	<	if not self.pset is None:
1174	<	inp_box.append(common.work_space.pathForTgz() + 'job/' + self.configFilename())
1175	<	## additional input files
1176	<	tgz = self.additionalInputFileTgz()
1177	<	inp_box.append(tgz)
1046	>	wrapper = os.path.basename(str(common._db.queryTask('scriptName')))
1047	>	inp_box.append(common.work_space.pathForTgz() +'job/'+ wrapper)
1048		return inp_box
1049
1050		def outputSandbox(self, nj):
#	Line 1185 \| Line 1055 \| class Cmssw(JobType):
1055
1056		## User Declared output files
1057		for out in (self.output_file+self.output_file_sandbox):
1058	<	n_out = nj + 1
1059	<	out_box.append(self.numberFile_(out,str(n_out)))
1058	>	n_out = nj + 1
1059	>	out_box.append(numberFile(out,str(n_out)))
1060		return out_box
1061
1192	–	def prepareSteeringCards(self):
1193	–	"""
1194	–	Make initial modifications of the user's steering card file.
1195	–	"""
1196	–	return
1062
1063		def wsRenameOutput(self, nj):
1064		"""
1065		Returns part of a job script which renames the produced files.
1066		"""
1067
1068	<	txt = '\n'
1069	<	txt += '# directory content\n'
1070	<	txt += 'ls \n'
1071	<
1072	<	txt += 'output_exit_status=0\n'
1073	<
1074	<	for fileWithSuffix in (self.output_file_sandbox):
1210	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1211	<	txt += '\n'
1212	<	txt += '# check output file\n'
1213	<	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1214	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1215	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1216	<	txt += 'else\n'
1217	<	txt += ' exit_status=60302\n'
1218	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1219	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1220	<	txt += ' if [ $middleware == OSG ]; then \n'
1221	<	txt += ' echo "prepare dummy output file"\n'
1222	<	txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
1223	<	txt += ' fi \n'
1224	<	txt += 'fi\n'
1225	<
1068	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
1069	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1070	>	txt += 'echo ">>> current directory content:"\n'
1071	>	if self.debug_wrapper:
1072	>	txt += 'ls -Al\n'
1073	>	txt += '\n'
1074	>
1075		for fileWithSuffix in (self.output_file):
1076	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
1076	>	output_file_num = numberFile(fileWithSuffix, '$NJob')
1077		txt += '\n'
1078		txt += '# check output file\n'
1079		txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
1080	<	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA\n'
1081	<	txt += ' cp $RUNTIME_AREA/'+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1080	>	if (self.copy_data == 1): # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
1081	>	txt += ' mv '+fileWithSuffix+' '+output_file_num+'\n'
1082	>	txt += ' ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1083	>	else:
1084	>	txt += ' mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
1085	>	txt += ' ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
1086		txt += 'else\n'
1087	<	txt += ' exit_status=60302\n'
1088	<	txt += ' echo "ERROR: Problem with output file '+fileWithSuffix+'"\n'
1089	<	txt += ' echo "JOB_EXIT_STATUS = $exit_status"\n'
1237	<	txt += ' output_exit_status=$exit_status\n'
1238	<	if common.scheduler.boss_scheduler_name == 'condor_g':
1087	>	txt += ' job_exit_code=60302\n'
1088	>	txt += ' echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
1089	>	if common.scheduler.name().upper() == 'CONDOR_G':
1090		txt += ' if [ $middleware == OSG ]; then \n'
1091		txt += ' echo "prepare dummy output file"\n'
1092		txt += ' echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
#	Line 1243 \| Line 1094 \| class Cmssw(JobType):
1094		txt += 'fi\n'
1095		file_list = []
1096		for fileWithSuffix in (self.output_file):
1097	<	file_list.append(self.numberFile_(fileWithSuffix, '$NJob'))
1098	<
1097	>	file_list.append(numberFile(fileWithSuffix, '$NJob'))
1098	>
1099		txt += 'file_list="'+string.join(file_list,' ')+'"\n'
1100	+	txt += '\n'
1101	+	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
1102	+	txt += 'echo ">>> current directory content:"\n'
1103	+	if self.debug_wrapper:
1104	+	txt += 'ls -Al\n'
1105	+	txt += '\n'
1106		txt += 'cd $RUNTIME_AREA\n'
1107	+	txt += 'echo ">>> current directory (RUNTIME_AREA): $RUNTIME_AREA"\n'
1108		return txt
1109
1252	–	def numberFile_(self, file, txt):
1253	–	"""
1254	–	append _'txt' before last extension of a file
1255	–	"""
1256	–	p = string.split(file,".")
1257	–	# take away last extension
1258	–	name = p[0]
1259	–	for x in p[1:-1]:
1260	–	name=name+"."+x
1261	–	# add "_txt"
1262	–	if len(p)>1:
1263	–	ext = p[len(p)-1]
1264	–	result = name + '_' + txt + "." + ext
1265	–	else:
1266	–	result = name + '_' + txt
1267	–
1268	–	return result
1269	–
1110		def getRequirements(self, nj=[]):
1111		"""
1112	<	return job requirements to add to jdl files
1112	>	return job requirements to add to jdl files
1113		"""
1114		req = ''
1115		if self.version:
1116		req='Member("VO-cms-' + \
1117		self.version + \
1118		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1119	<	## SL add requirement for OS version only if SL4
1280	<	#reSL4 = re.compile( r'slc4' )
1281	<	if self.executable_arch: # and reSL4.search(self.executable_arch):
1119	>	if self.executable_arch:
1120		req+=' && Member("VO-cms-' + \
1121		self.executable_arch + \
1122		'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
1123
1124		req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
1125	+	if ( common.scheduler.name() == "glitecoll" ) or ( common.scheduler.name() == "glite"):
1126	+	req += ' && other.GlueCEStateStatus == "Production" '
1127
1128		return req
1129
1130		def configFilename(self):
1131		""" return the config filename """
1132	<	return self.name()+'.cfg'
1132	>	# FUTURE: Can remove cfg mode for CMSSW >= 2_1_x
1133	>	if (self.CMSSW_major >= 2 and self.CMSSW_minor >= 1) or (self.CMSSW_major >= 3):
1134	>	return self.name()+'.py'
1135	>	else:
1136	>	return self.name()+'.cfg'
1137
1294	–	### OLI_DANIELE
1138		def wsSetupCMSOSGEnvironment_(self):
1139		"""
1140		Returns part of a job script which is prepares
1141		the execution environment and which is common for all CMS jobs.
1142		"""
1143	<	txt = '\n'
1144	<	txt += ' echo "### SETUP CMS OSG ENVIRONMENT ###"\n'
1145	<	txt += ' if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
1146	<	txt += ' # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
1147	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1148	<	txt += ' source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
1306	<	txt += ' elif [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1143	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
1144	>	txt += ' echo ">>> setup CMS OSG environment:"\n'
1145	>	txt += ' echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
1146	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1147	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1148	>	txt += ' if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
1149		txt += ' # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
1150	<	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1151	<	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1152	<	txt += ' else\n'
1153	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1154	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1155	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1314	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1315	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1316	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1317	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1318	<	txt += ' exit 1\n'
1319	<	txt += '\n'
1320	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1321	<	txt += ' cd $RUNTIME_AREA\n'
1322	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1323	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1324	<	txt += ' echo "SET_CMS_ENV 10017 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1325	<	txt += ' echo "JOB_EXIT_STATUS = 10017"\n'
1326	<	txt += ' echo "JobExitCode=10017" \| tee -a $RUNTIME_AREA/$repo\n'
1327	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1328	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1329	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1330	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1331	<	txt += ' fi\n'
1332	<	txt += '\n'
1333	<	txt += ' exit 1\n'
1334	<	txt += ' fi\n'
1150	>	txt += ' source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
1151	>	txt += ' else\n'
1152	>	txt += ' echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
1153	>	txt += ' job_exit_code=10020\n'
1154	>	txt += ' func_exit\n'
1155	>	txt += ' fi\n'
1156		txt += '\n'
1157	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1158	<	txt += ' echo " END SETUP CMS OSG ENVIRONMENT "\n'
1157	>	txt += ' echo "==> setup cms environment ok"\n'
1158	>	txt += ' echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1159
1160		return txt
1161	<
1341	<	### OLI_DANIELE
1161	>
1162		def wsSetupCMSLCGEnvironment_(self):
1163		"""
1164		Returns part of a job script which is prepares
1165		the execution environment and which is common for all CMS jobs.
1166		"""
1167	<	txt = ' \n'
1168	<	txt += ' echo " ### SETUP CMS LCG ENVIRONMENT ### "\n'
1169	<	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1170	<	txt += ' echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1171	<	txt += ' echo "JOB_EXIT_STATUS = 10031" \n'
1172	<	txt += ' echo "JobExitCode=10031" \| tee -a $RUNTIME_AREA/$repo\n'
1173	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1174	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1175	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1176	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1177	<	txt += ' exit 1\n'
1178	<	txt += ' else\n'
1179	<	txt += ' echo "Sourcing environment... "\n'
1180	<	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1181	<	txt += ' echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1182	<	txt += ' echo "JOB_EXIT_STATUS = 10020"\n'
1183	<	txt += ' echo "JobExitCode=10020" \| tee -a $RUNTIME_AREA/$repo\n'
1184	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1185	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1186	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1187	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1188	<	txt += ' exit 1\n'
1189	<	txt += ' fi\n'
1190	<	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1191	<	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1192	<	txt += ' result=$?\n'
1193	<	txt += ' if [ $result -ne 0 ]; then\n'
1374	<	txt += ' echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1375	<	txt += ' echo "JOB_EXIT_STATUS = 10032"\n'
1376	<	txt += ' echo "JobExitCode=10032" \| tee -a $RUNTIME_AREA/$repo\n'
1377	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1378	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1379	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1380	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1381	<	txt += ' exit 1\n'
1382	<	txt += ' fi\n'
1383	<	txt += ' fi\n'
1384	<	txt += ' \n'
1385	<	txt += ' echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1386	<	txt += ' echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1167	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1168	>	txt += ' echo ">>> setup CMS LCG environment:"\n'
1169	>	txt += ' echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1170	>	txt += ' export SCRAM_ARCH='+self.executable_arch+'\n'
1171	>	txt += ' export BUILD_ARCH='+self.executable_arch+'\n'
1172	>	txt += ' if [ ! $VO_CMS_SW_DIR ] ;then\n'
1173	>	txt += ' echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1174	>	txt += ' job_exit_code=10031\n'
1175	>	txt += ' func_exit\n'
1176	>	txt += ' else\n'
1177	>	txt += ' echo "Sourcing environment... "\n'
1178	>	txt += ' if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1179	>	txt += ' echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1180	>	txt += ' job_exit_code=10020\n'
1181	>	txt += ' func_exit\n'
1182	>	txt += ' fi\n'
1183	>	txt += ' echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1184	>	txt += ' source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1185	>	txt += ' result=$?\n'
1186	>	txt += ' if [ $result -ne 0 ]; then\n'
1187	>	txt += ' echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1188	>	txt += ' job_exit_code=10032\n'
1189	>	txt += ' func_exit\n'
1190	>	txt += ' fi\n'
1191	>	txt += ' fi\n'
1192	>	txt += ' \n'
1193	>	txt += ' echo "==> setup cms environment ok"\n'
1194		return txt
1195
1389	–	### FEDE FOR DBS OUTPUT PUBLICATION
1196		def modifyReport(self, nj):
1197		"""
1198	<	insert the part of the script that modifies the FrameworkJob Report
1198	>	insert the part of the script that modifies the FrameworkJob Report
1199		"""
1200	<
1201	<	txt = ''
1202	<	try:
1397	<	publish_data = int(self.cfg_params['USER.publish_data'])
1398	<	except KeyError:
1399	<	publish_data = 0
1400	<	if (publish_data == 1):
1401	<	txt += 'echo "Modify Job Report" \n'
1402	<	#txt += 'chmod a+x $RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1403	<	################ FEDE FOR DBS2 #############################################
1404	<	txt += 'chmod a+x $SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py\n'
1405	<	#############################################################################
1406	<	#try:
1407	<	# publish_data = int(self.cfg_params['USER.publish_data'])
1408	<	#except KeyError:
1409	<	# publish_data = 0
1410	<
1411	<	txt += 'if [ -z "$SE" ]; then\n'
1412	<	txt += ' SE="" \n'
1413	<	txt += 'fi \n'
1414	<	txt += 'if [ -z "$SE_PATH" ]; then\n'
1415	<	txt += ' SE_PATH="" \n'
1416	<	txt += 'fi \n'
1417	<	txt += 'echo "SE = $SE"\n'
1418	<	txt += 'echo "SE_PATH = $SE_PATH"\n'
1419	<
1420	<	#if (publish_data == 1):
1421	<	#processedDataset = self.cfg_params['USER.processed_datasetname']
1200	>	txt = '\n#Written by cms_cmssw::modifyReport\n'
1201	>	publish_data = int(self.cfg_params.get('USER.publish_data',0))
1202	>	if (publish_data == 1):
1203		processedDataset = self.cfg_params['USER.publish_data_name']
1204	<	txt += 'ProcessedDataset='+processedDataset+'\n'
1205	<	#### LFN=/store/user/<user>/processedDataset_PSETHASH
1206	<	txt += 'if [ "$SE_PATH" == "" ]; then\n'
1207	<	#### FEDE: added slash in LFN ##############
1204	>	if (self.primaryDataset == 'null'):
1205	>	self.primaryDataset = processedDataset
1206	>	if (common.scheduler.name().upper() == "CAF" or common.scheduler.name().upper() == "LSF"):
1207	>	### FEDE FOR NEW LFN ###
1208	>	LFNBaseName = LFNBase(self.primaryDataset, processedDataset, LocalUser=True)
1209	>	self.user = getUserName(LocalUser=True)
1210	>	########################
1211	>	else :
1212	>	### FEDE FOR NEW LFN ###
1213	>	LFNBaseName = LFNBase(self.primaryDataset, processedDataset)
1214	>	self.user = getUserName()
1215	>	########################
1216	>
1217	>	txt += 'if [ $copy_exit_status -eq 0 ]; then\n'
1218	>	### FEDE FOR NEW LFN ###
1219	>	#txt += ' FOR_LFN=%s_${PSETHASH}/\n'%(LFNBaseName)
1220	>	txt += ' FOR_LFN=%s/${PSETHASH}/\n'%(LFNBaseName)
1221	>	########################
1222	>	txt += 'else\n'
1223		txt += ' FOR_LFN=/copy_problems/ \n'
1224	<	txt += 'else \n'
1225	<	txt += ' tmp=`echo $SE_PATH \| awk -F \'store\' \'{print$2}\'` \n'
1226	<	##### FEDE TO BE CHANGED, BECAUSE STORE IS HARDCODED!!!! ########
1227	<	txt += ' FOR_LFN=/store$tmp \n'
1228	<	txt += 'fi \n'
1224	>	txt += ' SE=""\n'
1225	>	txt += ' SE_PATH=""\n'
1226	>	txt += 'fi\n'
1227	>
1228	>	txt += 'echo ">>> Modify Job Report:" \n'
1229	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1230	>	txt += 'ProcessedDataset='+processedDataset+'\n'
1231		txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1232	+	txt += 'echo "SE = $SE"\n'
1233	+	txt += 'echo "SE_PATH = $SE_PATH"\n'
1234		txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1235		txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1236	<	#txt += 'echo "$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1237	<	txt += 'echo "$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1238	<	txt += '$SOFTWARE_DIR/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1239	<	#txt += '$RUNTIME_AREA/'+self.version+'/ProdAgentApi/FwkJobRep/ModifyJobReport.py crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier $ProcessedDataset $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1440	<
1236	>	### FEDE FOR NEW LFN ###
1237	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH"\n'
1238	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py $RUNTIME_AREA/crab_fjr_$NJob.xml $NJob $FOR_LFN $PrimaryDataset $DataTier ' + self.user + '-$ProcessedDataset-$PSETHASH $ApplicationFamily $executable $CMSSW_VERSION $PSETHASH $SE $SE_PATH\n'
1239	>	########################
1240		txt += 'modifyReport_result=$?\n'
1442	–	txt += 'echo modifyReport_result = $modifyReport_result\n'
1241		txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1242	<	txt += ' exit_status=1\n'
1243	<	txt += ' echo "ERROR: Problem with ModifyJobReport"\n'
1242	>	txt += ' modifyReport_result=70500\n'
1243	>	txt += ' job_exit_code=$modifyReport_result\n'
1244	>	txt += ' echo "ModifyReportResult=$modifyReport_result" \| tee -a $RUNTIME_AREA/$repo\n'
1245	>	txt += ' echo "WARNING: Problem with ModifyJobReport"\n'
1246		txt += 'else\n'
1247	<	txt += ' mv NewFrameworkJobReport.xml crab_fjr_$NJob.xml\n'
1247	>	txt += ' mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1248		txt += 'fi\n'
1449	–	else:
1450	–	txt += 'echo "no data publication required"\n'
1451	–	#txt += 'ProcessedDataset=no_data_to_publish \n'
1452	–	#### FEDE: added slash in LFN ##############
1453	–	#txt += 'FOR_LFN=/local/ \n'
1454	–	#txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1455	–	#txt += 'echo "FOR_LFN = $FOR_LFN" \n'
1249		return txt
1250
1251	<	def cleanEnv(self):
1252	<	### OLI_DANIELE
1253	<	txt = ''
1254	<	txt += 'if [ $middleware == OSG ]; then\n'
1255	<	txt += ' cd $RUNTIME_AREA\n'
1256	<	txt += ' echo "Remove working directory: $WORKING_DIR"\n'
1257	<	txt += ' /bin/rm -rf $WORKING_DIR\n'
1258	<	txt += ' if [ -d $WORKING_DIR ] ;then\n'
1259	<	txt += ' echo "SET_EXE 60999 ==> OSG $WORKING_DIR could not be deleted on WN `hostname` after cleanup of WN"\n'
1260	<	txt += ' echo "JOB_EXIT_STATUS = 60999"\n'
1261	<	txt += ' echo "JobExitCode=60999" \| tee -a $RUNTIME_AREA/$repo\n'
1262	<	txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
1263	<	txt += ' rm -f $RUNTIME_AREA/$repo \n'
1264	<	txt += ' echo "MonitorJobID=`echo $MonitorJobID`" \| tee -a $RUNTIME_AREA/$repo \n'
1265	<	txt += ' echo "MonitorID=`echo $MonitorID`" \| tee -a $RUNTIME_AREA/$repo\n'
1251	>	def wsParseFJR(self):
1252	>	"""
1253	>	Parse the FrameworkJobReport to obtain useful infos
1254	>	"""
1255	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1256	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1257	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1258	>	txt += ' if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1259	>	txt += ' cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1260	>	if self.debug_wrapper :
1261	>	txt += ' echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1262	>	txt += ' executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1263	>	txt += ' if [ $executable_exit_status -eq 50115 ];then\n'
1264	>	txt += ' echo ">>> crab_fjr.xml contents: "\n'
1265	>	txt += ' cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1266	>	txt += ' echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1267	>	txt += ' elif [ $executable_exit_status -eq -999 ];then\n'
1268	>	txt += ' echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1269	>	txt += ' else\n'
1270	>	txt += ' echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1271	>	txt += ' fi\n'
1272	>	txt += ' else\n'
1273	>	txt += ' echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1274	>	txt += ' fi\n'
1275	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1276	>
1277	>	txt += ' if [ $executable_exit_status -eq 0 ];then\n'
1278	>	txt += ' echo ">>> Executable succeded $executable_exit_status"\n'
1279	>	if (self.datasetPath and not (self.dataset_pu or self.useParent)) :
1280	>	# VERIFY PROCESSED DATA
1281	>	txt += ' echo ">>> Verify list of processed files:"\n'
1282	>	txt += ' echo $InputFiles \|tr -d \'\\\\\' \|tr \',\' \'\\n\'\|tr -d \'"\' > input-files.txt\n'
1283	>	txt += ' python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --lfn > processed-files.txt\n'
1284	>	txt += ' cat input-files.txt \| sort \| uniq > tmp.txt\n'
1285	>	txt += ' mv tmp.txt input-files.txt\n'
1286	>	txt += ' echo "cat input-files.txt"\n'
1287	>	txt += ' echo "----------------------"\n'
1288	>	txt += ' cat input-files.txt\n'
1289	>	txt += ' cat processed-files.txt \| sort \| uniq > tmp.txt\n'
1290	>	txt += ' mv tmp.txt processed-files.txt\n'
1291	>	txt += ' echo "----------------------"\n'
1292	>	txt += ' echo "cat processed-files.txt"\n'
1293	>	txt += ' echo "----------------------"\n'
1294	>	txt += ' cat processed-files.txt\n'
1295	>	txt += ' echo "----------------------"\n'
1296	>	txt += ' diff -q input-files.txt processed-files.txt\n'
1297	>	txt += ' fileverify_status=$?\n'
1298	>	txt += ' if [ $fileverify_status -ne 0 ]; then\n'
1299	>	txt += ' executable_exit_status=30001\n'
1300	>	txt += ' echo "ERROR ==> not all input files processed"\n'
1301	>	txt += ' echo " ==> list of processed files from crab_fjr.xml differs from list in pset.cfg"\n'
1302	>	txt += ' echo " ==> diff input-files.txt processed-files.txt"\n'
1303	>	txt += ' fi\n'
1304	>	txt += ' elif [ $executable_exit_status -ne 0 ] \|\| [ $executable_exit_status -ne 50015 ] \|\| [ $executable_exit_status -ne 50017 ];then\n'
1305	>	txt += ' echo ">>> Executable failed $executable_exit_status"\n'
1306	>	txt += ' func_exit\n'
1307		txt += ' fi\n'
1308	+	txt += '\n'
1309	+	txt += 'else\n'
1310	+	txt += ' echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1311		txt += 'fi\n'
1312		txt += '\n'
1313	+	txt += 'echo "ExeExitCode=$executable_exit_status" \| tee -a $RUNTIME_AREA/$repo\n'
1314	+	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1315	+	txt += 'job_exit_code=$executable_exit_status\n'
1316	+
1317		return txt
1318
1319		def setParam_(self, param, value):
#	Line 1481 \| Line 1322 \| class Cmssw(JobType):
1322		def getParams(self):
1323		return self._params
1324
1484	–	def setTaskid_(self):
1485	–	self._taskId = self.cfg_params['taskId']
1486	–
1487	–	def getTaskid(self):
1488	–	return self._taskId
1489	–
1325		def uniquelist(self, old):
1326		"""
1327		remove duplicates from a list
#	Line 1496 \| Line 1331 \| class Cmssw(JobType):
1331		nd[e]=0
1332		return nd.keys()
1333
1334	<
1500	<	def checkOut(self, limit):
1334	>	def outList(self):
1335		"""
1336		check the dimension of the output files
1337		"""
1338	<	txt = 'echo "*****************************************"\n'
1339	<	txt += 'echo " Starting output sandbox limit check "\n'
1506	<	txt += 'echo "*****************************************"\n'
1507	<	allOutFiles = ""
1338	>	txt = ''
1339	>	txt += 'echo ">>> list of expected files on output sandbox"\n'
1340		listOutFiles = []
1341	<	for fileOut in (self.output_file+self.output_file_sandbox):
1342	<	if fileOut.find('crab_fjr') == -1:
1343	<	allOutFiles = allOutFiles + " " + self.numberFile_(fileOut, '$NJob')
1344	<	listOutFiles.append(self.numberFile_(fileOut, '$NJob'))
1345	<	txt += 'echo "OUTPUT files: '+str(allOutFiles)+'";\n'
1346	<	txt += 'ls -gGhrta;\n'
1347	<	txt += 'sum=0;\n'
1348	<	txt += 'for file in '+str(allOutFiles)+' ; do\n'
1349	<	txt += ' if [ -e $file ]; then\n'
1350	<	txt += ' tt=`ls -gGrta $file \| awk \'{ print $3 }\'`\n'
1351	<	txt += ' sum=`expr $sum + $tt`\n'
1352	<	txt += ' else\n'
1353	<	txt += ' echo "WARNING: output file $file not found!"\n'
1354	<	txt += ' fi\n'
1355	<	txt += 'done\n'
1524	<	txt += 'echo "Total Output dimension: $sum";\n'
1525	<	txt += 'limit='+str(limit)+';\n'
1526	<	txt += 'echo "OUTPUT FILES LIMIT SET TO: $limit";\n'
1527	<	txt += 'if [ $limit -lt $sum ]; then\n'
1528	<	txt += ' echo "WARNING: output files have to big size - something will be lost;"\n'
1529	<	txt += ' echo " checking the output file sizes..."\n'
1530	<	"""
1531	<	txt += ' dim=0;\n'
1532	<	txt += ' exclude=0;\n'
1533	<	txt += ' for files in '+str(allOutFiles)+' ; do\n'
1534	<	txt += ' sumTemp=0;\n'
1535	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1536	<	txt += ' if [ $file != $file2 ]; then\n'
1537	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1538	<	txt += ' sumTemp=`expr $sumTemp + $tt`;\n'
1539	<	txt += ' fi\n'
1540	<	txt += ' done\n'
1541	<	txt += ' if [ $sumTemp -lt $limit ]; then\n'
1542	<	txt += ' if [ $dim -lt $sumTemp ]; then\n'
1543	<	txt += ' dim=$sumTemp;\n'
1544	<	txt += ' exclude=$file;\n'
1545	<	txt += ' fi\n'
1546	<	txt += ' fi\n'
1547	<	txt += ' done\n'
1548	<	txt += ' echo "Dimension calculated: $dim"; echo "File to exclude: $exclude";\n'
1549	<	"""
1550	<	txt += ' tot=0;\n'
1551	<	txt += ' for file2 in '+str(allOutFiles)+' ; do\n'
1552	<	txt += ' tt=`ls -gGrta $file2 \| awk \'{ print $3 }\';`\n'
1553	<	txt += ' tot=`expr $tot + $tt`;\n'
1554	<	txt += ' if [ $limit -lt $tot ]; then\n'
1555	<	txt += ' tot=`expr $tot - $tt`;\n'
1556	<	txt += ' fileLast=$file;\n'
1557	<	txt += ' break;\n'
1558	<	txt += ' fi\n'
1559	<	txt += ' done\n'
1560	<	txt += ' echo "Dimension calculated: $tot"; echo "First file to exclude: $file";\n'
1561	<	txt += ' flag=0;\n'
1562	<	txt += ' for filess in '+str(allOutFiles)+' ; do\n'
1563	<	txt += ' if [ $fileLast = $filess ]; then\n'
1564	<	txt += ' flag=1;\n'
1565	<	txt += ' fi\n'
1566	<	txt += ' if [ $flag -eq 1 ]; then\n'
1567	<	txt += ' rm -f $filess;\n'
1568	<	txt += ' fi\n'
1569	<	txt += ' done\n'
1570	<	txt += ' ls -agGhrt;\n'
1571	<	txt += ' echo "WARNING: output files are too big in dimension: can not put in the output_sandbox.";\n'
1572	<	txt += ' echo "JOB_EXIT_STATUS = 70000";\n'
1573	<	txt += ' exit_status=70000;\n'
1574	<	txt += 'else'
1575	<	txt += ' echo "Total Output dimension $sum is fine.";\n'
1576	<	txt += 'fi\n'
1577	<	txt += 'echo "*****************************************"\n'
1578	<	txt += 'echo "* Ending output sandbox limit check *"\n'
1579	<	txt += 'echo "*****************************************"\n'
1341	>	stdout = 'CMSSW_$NJob.stdout'
1342	>	stderr = 'CMSSW_$NJob.stderr'
1343	>	if (self.return_data == 1):
1344	>	for file in (self.output_file+self.output_file_sandbox):
1345	>	listOutFiles.append(numberFile(file, '$NJob'))
1346	>	listOutFiles.append(stdout)
1347	>	listOutFiles.append(stderr)
1348	>	else:
1349	>	for file in (self.output_file_sandbox):
1350	>	listOutFiles.append(numberFile(file, '$NJob'))
1351	>	listOutFiles.append(stdout)
1352	>	listOutFiles.append(stderr)
1353	>	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1354	>	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1355	>	txt += 'export filesToCheck\n'
1356		return txt

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing COMP/CRAB/python/cms_cmssw.py (file contents): Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs. Revision 1.236 by spiga, Mon Sep 8 07:42:41 2008 UTC

Diff Legend

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.128 by fanzago, Thu Oct 11 16:23:44 2007 UTC vs.
Revision 1.236 by spiga, Mon Sep 8 07:42:41 2008 UTC