ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/cms_cmssw.py

Comparing COMP/CRAB/python/cms_cmssw.py (file contents):
Revision 1.4 by afanfani, Tue May 30 00:48:37 2006 UTC vs.
Revision 1.365 by spiga, Tue Nov 9 21:10:07 2010 UTC

#	Line 1 | Line 1
1	+
2	+	__revision__ = "$Id$"
3	+	__version__ = "$Revision$"
4	+
5		from JobType import JobType
2	–	from crab_logger import Logger
6		from crab_exceptions import *
7		from crab_util import *
8		import common
9	<	import PsetManipulator
7	<
8	<	import DBSInfo_EDM
9	<	#from DataDiscovery_EDM import DataDiscovery_EDM
10	<	import DataDiscovery_EDM
11	<	#from DataLocation_EDM import DataLocation_EDM
12	<	import DataLocation_EDM
9	>	import re
10		import Scram
11	<
12	<	import os, string, re
11	>	from Splitter import JobSplitter
12	>	from Downloader import Downloader
13	>	try:
14	>	import json
15	>	except:
16	>	import simplejson as json
17	>
18	>	from IMProv.IMProvNode import IMProvNode
19	>	from IMProv.IMProvLoader import loadIMProvFile
20	>	import os, string, glob
21	>	from xml.dom import pulldom
22
23		class Cmssw(JobType):
24	<	def __init__(self, cfg_params):
24	>	def __init__(self, cfg_params, ncjobs,skip_blocks, isNew):
25		JobType.__init__(self, 'CMSSW')
26	<	common.logger.debug(3,'CMSSW::__init__')
27	<
28	<	self.analisys_common_info = {}
29	<	# Marco.
26	>	common.logger.debug('CMSSW::__init__')
27	>	self.skip_blocks = skip_blocks
28	>	self.argsList = 2
29	>	self.NumEvents=0
30		self._params = {}
31		self.cfg_params = cfg_params
32	+	### FEDE FOR MULTI ###
33	+	self.var_filter=''
34	+
35	+	### Temporary patch to automatically skip the ISB size check:
36	+	self.server = self.cfg_params.get('CRAB.server_name',None) or \
37	+	self.cfg_params.get('CRAB.use_server',0)
38	+	self.local  = common.scheduler.name().upper() in ['LSF','CAF','CONDOR','SGE','PBS']
39	+	size = 9.5
40	+	if self.server :
41	+	size = 1000
42	+	elif self.local:
43	+	size = 9999999
44	+	self.MaxTarBallSize = float(self.cfg_params.get('GRID.maxtarballsize',size))
45	+
46	+	# number of jobs requested to be created, limit obj splitting
47	+	self.ncjobs = ncjobs
48
27	–	log = common.logger
28	–
49		self.scram = Scram.Scram(cfg_params)
30	–	scramArea = ''
50		self.additional_inbox_files = []
51		self.scriptExe = ''
52		self.executable = ''
53	+	self.executable_arch = self.scram.getArch()
54		self.tgz_name = 'default.tgz'
55	<
55	>	self.scriptName = 'CMSSW.sh'
56	>	self.pset = ''
57	>	self.datasetPath = ''
58	>
59	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
60	>	# set FJR file name
61	>	self.fjrFileName = 'crab_fjr.xml'
62
63		self.version = self.scram.getSWVersion()
64	<	common.analisys_common_info['sw_version'] = self.version
65	<	### FEDE
66	<	common.analisys_common_info['copy_input_data'] = 0
67	<	common.analisys_common_info['events_management'] = 1
64	>	common.logger.log(10-1,"CMSSW version is: "+str(self.version))
65	>	version_array = self.version.split('_')
66	>	self.CMSSW_major = 0
67	>	self.CMSSW_minor = 0
68	>	self.CMSSW_patch = 0
69	>	try:
70	>	self.CMSSW_major = int(version_array[1])
71	>	self.CMSSW_minor = int(version_array[2])
72	>	self.CMSSW_patch = int(version_array[3])
73	>	except:
74	>	msg = "Cannot parse CMSSW version string: " + self.version + " for major and minor release number!"
75	>	raise CrabException(msg)
76
77	+	if self.CMSSW_major < 2 or (self.CMSSW_major == 2 and self.CMSSW_minor < 1):
78	+	msg = "CRAB supports CMSSW >= 2_1_x only. Use an older CRAB version."
79	+	raise CrabException(msg)
80	+	"""
81	+	As CMSSW versions are dropped we can drop more code:
82	+	2.x dropped: drop check for lumi range setting
83	+	"""
84	+	self.checkCMSSWVersion()
85		### collect Data cards
86	<	try:
87	<	#   self.owner = cfg_params['CMSSW.owner']
88	<	#   log.debug(6, "CMSSW::CMSSW(): owner = "+self.owner)
89	<	#   self.dataset = cfg_params['CMSSW.dataset']
90	<	self.datasetPath = cfg_params['CMSSW.datasetpath']
91	<	log.debug(6, "CMSSW::CMSSW(): datasetPath = "+self.datasetPath)
92	<	except KeyError:
93	<	#    msg = "Error: owner and/or dataset not defined "
94	<	msg = "Error: datasetpath not defined "
95	<	raise CrabException(msg)
96	<	self.dataTiers = []
97	<	#       try:
98	<	#           tmpDataTiers = string.split(cfg_params['CMSSW.data_tier'],',')
99	<	#           for tmp in tmpDataTiers:
100	<	#               tmp=string.strip(tmp)
101	<	#               self.dataTiers.append(tmp)
102	<	#               pass
103	<	#           pass
104	<	#       except KeyError:
105	<	#           pass
106	<	#       log.debug(6, "Cmssw::Cmssw(): dataTiers = "+str(self.dataTiers))
86	>
87	>	### Temporary: added to remove input file control in the case of PU
88	>	self.dataset_pu = cfg_params.get('CMSSW.dataset_pu', None)
89	>
90	>	tmp =  cfg_params['CMSSW.datasetpath']
91	>	common.logger.log(10-1, "CMSSW::CMSSW(): datasetPath = "+tmp)
92	>
93	>	if tmp =='':
94	>	msg = "Error: datasetpath not defined "
95	>	raise CrabException(msg)
96	>	elif string.lower(tmp)=='none':
97	>	self.datasetPath = None
98	>	self.selectNoInput = 1
99	>	self.primaryDataset = 'null'
100	>	else:
101	>	self.datasetPath = tmp
102	>	self.selectNoInput = 0
103	>	ll = len(self.datasetPath.split("/"))
104	>	if (ll < 4):
105	>	msg = 'Your datasetpath has a invalid format ' + self.datasetPath + '\n'
106	>	msg += 'Expected a path in format /PRIMARY/PROCESSED/TIER1-TIER2 or /PRIMARY/PROCESSED/TIER/METHOD for ADS'
107	>	raise CrabException(msg)
108	>	self.primaryDataset = self.datasetPath.split("/")[1]
109	>	self.dataTier = self.datasetPath.split("/")[2]
110	>
111	>	# Analysis dataset is primary/processed/tier/definition
112	>	self.ads = False
113	>	if self.datasetPath:
114	>	self.ads = len(self.datasetPath.split("/")) > 4
115	>	self.lumiMask = self.cfg_params.get('CMSSW.lumi_mask',None)
116	>	self.lumiParams = self.cfg_params.get('CMSSW.total_number_of_lumis',None) or \
117	>	self.cfg_params.get('CMSSW.lumis_per_job',None)
118	>
119	>	# FUTURE: Can remove this check
120	>	if self.ads and self.CMSSW_major < 3:
121	>	common.logger.info('Warning: Analysis dataset support is incomplete in CMSSW 2_x.')
122	>	common.logger.info('  Only file level, not lumi level, granularity is supported.')
123	>
124	>	self.debugWrap=''
125	>	self.debug_wrapper = int(cfg_params.get('USER.debug_wrapper',0))
126	>	if self.debug_wrapper == 1: self.debugWrap='--debug'
127
128		## now the application
129	<	try:
130	<	self.executable = cfg_params['CMSSW.executable']
131	<	log.debug(6, "CMSSW::CMSSW(): executable = "+self.executable)
132	<	msg = "Default executable cmsRun overridden. Switch to " + self.executable
71	<	log.debug(3,msg)
72	<	except KeyError:
73	<	self.executable = 'cmsRun'
74	<	msg = "User executable not defined. Use cmsRun"
75	<	log.debug(3,msg)
76	<	pass
129	>	self.managedGenerators = ['madgraph', 'comphep', 'lhe']
130	>	self.generator = cfg_params.get('CMSSW.generator','pythia').lower()
131	>	self.executable = cfg_params.get('CMSSW.executable','cmsRun')
132	>	common.logger.log(10-1, "CMSSW::CMSSW(): executable = "+self.executable)
133
134	<	try:
135	<	self.pset = cfg_params['CMSSW.pset']
136	<	log.debug(6, "Cmssw::Cmssw(): PSet file = "+self.pset)
134	>	if not cfg_params.has_key('CMSSW.pset'):
135	>	raise CrabException("PSet file missing. Cannot run cmsRun ")
136	>	self.pset = cfg_params['CMSSW.pset']
137	>	common.logger.log(10-1, "Cmssw::Cmssw(): PSet file = "+self.pset)
138	>	if self.pset.lower() != 'none' :
139		if (not os.path.exists(self.pset)):
140		raise CrabException("User defined PSet file "+self.pset+" does not exist")
141	<	except KeyError:
142	<	raise CrabException("PSet file missing. Cannot run cmsRun ")
141	>	else:
142	>	self.pset = None
143
144		# output files
145	<	try:
146	<	self.output_file = []
145	>	## stuff which must be returned always via sandbox
146	>	self.output_file_sandbox = []
147
148	<	tmp = cfg_params['CMSSW.output_file']
149	<	if tmp != '':
92	<	tmpOutFiles = string.split(cfg_params['CMSSW.output_file'],',')
93	<	log.debug(7, 'cmssw::cmssw(): output files '+str(tmpOutFiles))
94	<	for tmp in tmpOutFiles:
95	<	tmp=string.strip(tmp)
96	<	self.output_file.append(tmp)
97	<	pass
98	<	else:
99	<	log.message("No output file defined: only stdout/err will be available")
100	<	pass
101	<	pass
102	<	except KeyError:
103	<	log.message("No output file defined: only stdout/err will be available")
104	<	pass
148	>	# add fjr report by default via sandbox
149	>	self.output_file_sandbox.append(self.fjrFileName)
150
151	<	# script_exe file as additional file in inputSandbox
152	<	try:
153	<	self.scriptExe = cfg_params['USER.script_exe']
154	<	self.additional_inbox_files.append(self.scriptExe)
155	<	except KeyError:
156	<	pass
157	<	if self.scriptExe != '':
158	<	if os.path.isfile(self.scriptExe):
159	<	pass
160	<	else:
161	<	log.message("WARNING. file "+self.scriptExe+" not found")
162	<	sys.exit()
163	<
164	<	## additional input files
120	<	try:
121	<	tmpAddFiles = string.split(cfg_params['CMSSW.additional_input_files'],',')
122	<	for tmp in tmpAddFiles:
123	<	if not os.path.exists(tmp):
124	<	raise CrabException("Additional input file not found: "+tmp)
125	<	tmp=string.strip(tmp)
126	<	self.additional_inbox_files.append(tmp)
127	<	pass
128	<	pass
129	<	except KeyError:
130	<	pass
151	>	# other output files to be returned via sandbox or copied to SE
152	>	outfileflag = False
153	>	self.output_file = []
154	>	tmp = cfg_params.get('CMSSW.output_file',None)
155	>	if tmp :
156	>	self.output_file = [x.strip() for x in tmp.split(',')]
157	>	outfileflag = True #output found
158	>
159	>	self.scriptExe = cfg_params.get('USER.script_exe',None)
160	>	if self.scriptExe :
161	>	if not os.path.isfile(self.scriptExe):
162	>	msg ="ERROR. file "+self.scriptExe+" not found"
163	>	raise CrabException(msg)
164	>	self.additional_inbox_files.append(string.strip(self.scriptExe))
165
166	<	try:
167	<	self.filesPerJob = int(cfg_params['CMSSW.files_per_jobs']) #Daniele
134	<	except KeyError:
135	<	self.filesPerJob = 1
166	>	self.AdditionalArgs = cfg_params.get('USER.script_arguments',None)
167	>	if self.AdditionalArgs : self.AdditionalArgs = string.replace(self.AdditionalArgs,',',' ')
168
169	<	## Max event   will be total_number_of_events ???  Daniele
170	<	try:
139	<	self.maxEv = cfg_params['CMSSW.event_per_job']
140	<	except KeyError:
141	<	self.maxEv = "-1"
142	<	##
143	<	try:
144	<	self.total_number_of_events = int(cfg_params['CMSSW.total_number_of_events'])
145	<	except KeyError:
146	<	msg = 'Must define total_number_of_events'
169	>	if self.datasetPath == None and self.pset == None and self.scriptExe == '' :
170	>	msg ="Error. script_exe  not defined"
171		raise CrabException(msg)
148	–
149	–	CEBlackList = []
150	–	try:
151	–	tmpBad = string.split(cfg_params['EDG.ce_black_list'],',')
152	–	for tmp in tmpBad:
153	–	tmp=string.strip(tmp)
154	–	CEBlackList.append(tmp)
155	–	except KeyError:
156	–	pass
172
173	<	self.reCEBlackList=[]
174	<	for bad in CEBlackList:
160	<	self.reCEBlackList.append(re.compile( bad ))
173	>	# use parent files...
174	>	self.useParent = int(self.cfg_params.get('CMSSW.use_parent',0))
175
176	<	common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
177	<
178	<	CEWhiteList = []
179	<	try:
180	<	tmpGood = string.split(cfg_params['EDG.ce_white_list'],',')
181	<	for tmp in tmpGood:
182	<	tmp=string.strip(tmp)
183	<	CEWhiteList.append(tmp)
184	<	except KeyError:
176	>	## additional input files
177	>	if cfg_params.has_key('USER.additional_input_files'):
178	>	tmpAddFiles = string.split(cfg_params['USER.additional_input_files'],',')
179	>	for tmp in tmpAddFiles:
180	>	tmp = string.strip(tmp)
181	>	dirname = ''
182	>	if not tmp[0]=="/": dirname = "."
183	>	files = []
184	>	if string.find(tmp,"*")>-1:
185	>	files = glob.glob(os.path.join(dirname, tmp))
186	>	if len(files)==0:
187	>	raise CrabException("No additional input file found with this pattern: "+tmp)
188	>	else:
189	>	files.append(tmp)
190	>	for file in files:
191	>	if not os.path.exists(file):
192	>	raise CrabException("Additional input file not found: "+file)
193	>	pass
194	>	self.additional_inbox_files.append(string.strip(file))
195	>	pass
196		pass
197	+	common.logger.debug("Additional input files: "+str(self.additional_inbox_files))
198	+	pass
199
173	–	#print 'CEWhiteList: ',CEWhiteList
174	–	self.reCEWhiteList=[]
175	–	for Good in CEWhiteList:
176	–	self.reCEWhiteList.append(re.compile( Good ))
177	–
178	–	common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
200
201	<	self.PsetEdit = PsetManipulator.PsetManipulator(self.pset) #Daniele Pset
201	>	## New method of dealing with seeds
202	>	self.incrementSeeds = []
203	>	self.preserveSeeds = []
204	>	if cfg_params.has_key('CMSSW.preserve_seeds'):
205	>	tmpList = cfg_params['CMSSW.preserve_seeds'].split(',')
206	>	for tmp in tmpList:
207	>	tmp.strip()
208	>	self.preserveSeeds.append(tmp)
209	>	if cfg_params.has_key('CMSSW.increment_seeds'):
210	>	tmpList = cfg_params['CMSSW.increment_seeds'].split(',')
211	>	for tmp in tmpList:
212	>	tmp.strip()
213	>	self.incrementSeeds.append(tmp)
214	>
215	>	# Copy/return/publish
216	>	self.copy_data = int(cfg_params.get('USER.copy_data',0))
217	>	self.return_data = int(cfg_params.get('USER.return_data',0))
218	>	self.publish_data = int(cfg_params.get('USER.publish_data',0))
219	>	if (self.publish_data == 1):
220	>	if not cfg_params.has_key('USER.publish_data_name'):
221	>	raise CrabException('Cannot publish output data, because you did not specify USER.publish_data_name parameter in the crab.cfg file')
222	>	else:
223	>	self.processedDataset = cfg_params['USER.publish_data_name']
224
225	+	self.conf = {}
226	+	self.conf['pubdata'] = None
227	+	# number of jobs requested to be created, limit obj splitting DD
228		#DBSDLS-start
229	<	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
229	>	## Initialize the variables that are extracted from DBS/DLS and needed in other places of the code
230		self.maxEvents=0  # max events available   ( --> check the requested nb. of evts in Creator.py)
231		self.DBSPaths={}  # all dbs paths requested ( --> input to the site local discovery script)
232	+	self.jobDestination=[]  # Site destination(s) for each job (list of lists)
233		## Perform the data location and discovery (based on DBS/DLS)
234	<	self.DataDiscoveryAndLocation(cfg_params)
235	<	#DBSDLS-end
234	>	## SL: Don't if NONE is specified as input (pythia use case)
235	>	blockSites = {}
236	>	#wmbs
237	>	self.automation = int(self.cfg_params.get('WMBS.automation',0))
238	>	if self.automation == 0:
239	>	if self.datasetPath:
240	>	blockSites = self.DataDiscoveryAndLocation(cfg_params)
241	>	#DBSDLS-end
242	>	self.conf['blockSites']=blockSites
243	>
244	>	## Select Splitting
245	>	splitByRun = int(cfg_params.get('CMSSW.split_by_run',0))
246	>
247	>	if self.selectNoInput:
248	>	if self.pset == None:
249	>	self.algo = 'ForScript'
250	>	else:
251	>	self.algo = 'NoInput'
252	>	self.conf['managedGenerators']=self.managedGenerators
253	>	self.conf['generator']=self.generator
254	>	elif self.ads or self.lumiMask or self.lumiParams:
255	>	self.algo = 'LumiBased'
256	>	if splitByRun:
257	>	msg = "Cannot combine split by run with lumi_mask, ADS, " \
258	>	"or lumis_per_job. Use split by lumi mode instead."
259	>	raise CrabException(msg)
260	>
261	>	elif splitByRun ==1:
262	>	self.algo = 'RunBased'
263	>	else:
264	>	self.algo = 'EventBased'
265	>	common.logger.debug("Job splitting method: %s" % self.algo)
266	>
267	>	splitter = JobSplitter(self.cfg_params,self.conf)
268	>	self.dict = splitter.Algos()[self.algo]()
269	>
270	>	self.argsFile= '%s/arguments.xml'%common.work_space.shareDir()
271	>	self.rootArgsFilename= 'arguments'
272	>	# modify Pset only the first time
273	>	if isNew:
274	>	if self.pset != None: self.ModifyPset()
275	>
276	>	## Prepare inputSandbox TarBall (only the first time)
277	>	self.tarNameWithPath = self.getTarBall(self.executable)
278	>
279	>
280	>	def ModifyPset(self):
281	>	import PsetManipulator as pp
282	>
283	>	# If pycfg_params set, fake out the config script
284	>	# to make it think it was called with those args
285	>	pycfg_params = self.cfg_params.get('CMSSW.pycfg_params',None)
286	>	if pycfg_params:
287	>	trueArgv = sys.argv
288	>	sys.argv = [self.pset]
289	>	sys.argv.extend(pycfg_params.split(' '))
290	>	PsetEdit = pp.PsetManipulator(self.pset)
291	>	if pycfg_params: # Restore original sys.argv
292	>	sys.argv = trueArgv
293	>
294	>	try:
295	>	# Add FrameworkJobReport to parameter-set, set max events.
296	>	# Reset later for data jobs by writeCFG which does all modifications
297	>	PsetEdit.maxEvent(1)
298	>	PsetEdit.skipEvent(0)
299	>	PsetEdit.psetWriter(self.configFilename())
300	>	## If present, add TFileService to output files
301	>	if not int(self.cfg_params.get('CMSSW.skip_tfileservice_output',0)):
302	>	tfsOutput = PsetEdit.getTFileService()
303	>	if tfsOutput:
304	>	if tfsOutput in self.output_file:
305	>	common.logger.debug("Output from TFileService "+tfsOutput+" already in output files")
306	>	else:
307	>	outfileflag = True #output found
308	>	self.output_file.append(tfsOutput)
309	>	common.logger.info("Adding "+tfsOutput+" (from TFileService) to list of output files")
310	>	pass
311	>	pass
312
313	<	self.tgzNameWithPath = self.getTarBall(self.executable)
313	>	# If requested, add PoolOutputModule to output files
314	>	### FEDE FOR MULTI ###
315	>	#edmOutput = PsetEdit.getPoolOutputModule()
316	>	edmOutputDict = PsetEdit.getPoolOutputModule()
317	>	common.logger.debug("(test) edmOutputDict = "+str(edmOutputDict))
318	>	filter_dict = {}
319	>	for key in edmOutputDict.keys():
320	>	filter_dict[key]=edmOutputDict[key]['dataset']
321	>	common.logger.debug("(test) filter_dict for multi =  "+str(filter_dict))
322	>
323	>	#### in CMSSW.sh: export var_filter
324	>
325	>	self.var_filter = json.dumps(filter_dict)
326	>	common.logger.debug("(test) var_filter for multi =  "+self.var_filter)
327	>
328	>	edmOutput = edmOutputDict.keys()
329	>	if int(self.cfg_params.get('CMSSW.get_edm_output',0)):
330	>	if edmOutput:
331	>	for outputFile in edmOutput:
332	>	if outputFile in self.output_file:
333	>	common.logger.debug("Output from PoolOutputModule "+outputFile+" already in output files")
334	>	else:
335	>	self.output_file.append(outputFile)
336	>	common.logger.info("Adding "+outputFile+" (from PoolOutputModule) to list of output files")
337	>	# not requested, check anyhow to avoid accidental T2 overload
338	>	else:
339	>	if edmOutput:
340	>	missedFiles = []
341	>	for outputFile in edmOutput:
342	>	if outputFile not in self.output_file:
343	>	missedFiles.append(outputFile)
344	>	if missedFiles:
345	>	msg  = "ERROR: PoolOutputModule(s) are present in your ParameteSet %s \n"%self.pset
346	>	msg += "    but the file(s) produced ( %s ) are not in the list of output files\n" % ', '.join(missedFiles)
347	>	msg += "WARNING: please remove them. If you want to keep them, add the file(s) to output_files or use CMSSW.get_edm_output = 1\n"
348	>	if int(self.cfg_params.get('CMSSW.ignore_edm_output',0)):
349	>	msg += "    CMSSW.ignore_edm_output==1 : Hope you know what you are doing...\n"
350	>	common.logger.info(msg)
351	>	else :
352	>	raise CrabException(msg)
353	>
354	>	if (PsetEdit.getBadFilesSetting()):
355	>	msg = "WARNING: You have set skipBadFiles to True. This will continue processing on some errors and you may not be notified."
356	>	common.logger.info(msg)
357	>
358	>	except CrabException, msg:
359	>	common.logger.info(str(msg))
360	>	msg='Error while manipulating ParameterSet (see previous message, if any): exiting...'
361	>	raise CrabException(msg)
362	>
363	>	valid = re.compile('^[\w\.\-]+$')
364	>	for fileName in self.output_file:
365	>	if not valid.match(fileName):
366	>	msg = "The file %s may only contain alphanumeric characters and -, _, ." % fileName
367	>	raise CrabException(msg)
368
192	–	self.jobSplitting()  #Daniele job Splitting
193	–	self.PsetEdit.maxEvent(self.maxEv) #Daniele
194	–	self.PsetEdit.inputModule("INPUT") #Daniele
195	–	self.PsetEdit.psetWriter(self.configFilename())
369
370		def DataDiscoveryAndLocation(self, cfg_params):
371
372	<	common.logger.debug(10,"CMSSW::DataDiscoveryAndLocation()")
372	>	import DataDiscovery
373	>	import DataLocation
374	>	common.logger.log(10-1,"CMSSW::DataDiscoveryAndLocation()")
375
201	–	#datasetPath = "/"+self.owner+"/"+self.dataTiers[0]+"/"+self.dataset
202	–
376		datasetPath=self.datasetPath
377
205	–	## TODO
206	–	dataTiersList = ""
207	–	dataTiers = dataTiersList.split(',')
208	–
378		## Contact the DBS
379	+	common.logger.info("Contacting Data Discovery Services ...")
380		try:
381	<	self.pubdata=DataDiscovery_EDM.DataDiscovery_EDM(datasetPath, dataTiers, cfg_params)
381	>	self.pubdata=DataDiscovery.DataDiscovery(datasetPath, cfg_params,self.skip_blocks)
382		self.pubdata.fetchDBSInfo()
383
384	<	except DataDiscovery_EDM.NotExistingDatasetError, ex :
384	>	except DataDiscovery.NotExistingDatasetError, ex :
385		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
386		raise CrabException(msg)
387	<
218	<	except DataDiscovery_EDM.NoDataTierinProvenanceError, ex :
387	>	except DataDiscovery.NoDataTierinProvenanceError, ex :
388		msg = 'ERROR ***: failed Data Discovery in DBS : %s'%ex.getErrorMessage()
389		raise CrabException(msg)
390	<	except DataDiscovery_EDM.DataDiscoveryError, ex:
391	<	msg = 'ERROR ***: failed Data Discovery in DBS  %s'%ex.getErrorMessage()
390	>	except DataDiscovery.DataDiscoveryError, ex:
391	>	msg = 'ERROR ***: failed Data Discovery in DBS :  %s'%ex.getErrorMessage()
392		raise CrabException(msg)
393
394	<	## get list of all required data in the form of dbs paths  (dbs path = /dataset/datatier/owner)
395	<	## self.DBSPaths=self.pubdata.getDBSPaths()
227	<	common.logger.message("Required data are :"+self.datasetPath)
228	<
229	<	filesbyblock=self.pubdata.getFiles()
230	<	self.AllInputFiles=filesbyblock.values()
231	<	self.files = self.AllInputFiles
232	<
233	<	## TEMP
234	<	#    self.filesTmp = filesbyblock.values()
235	<	#    self.files = []
236	<	#    locPath='rfio:cmsbose2.bo.infn.it:/flatfiles/SE00/cms/fanfani/ProdTest/'
237	<	#    locPath=''
238	<	#    tmp = []
239	<	#    for file in self.filesTmp[0]:
240	<	#        tmp.append(locPath+file)
241	<	#    self.files.append(tmp)
242	<	## END TEMP
394	>	self.filesbyblock=self.pubdata.getFiles()
395	>	self.conf['pubdata']=self.pubdata
396
397		## get max number of events
398	<	#common.logger.debug(10,"number of events for primary fileblocks %i"%self.pubdata.getMaxEvents())
246	<	self.maxEvents=self.pubdata.getMaxEvents() ##  self.maxEvents used in Creator.py
247	<	common.logger.message("\nThe number of available events is %s"%self.maxEvents)
398	>	self.maxEvents=self.pubdata.getMaxEvents()
399
400		## Contact the DLS and build a list of sites hosting the fileblocks
401		try:
402	<	dataloc=DataLocation_EDM.DataLocation_EDM(filesbyblock.keys(),cfg_params)
403	<	dataloc.fetchDLSInfo()
402	>	dataloc=DataLocation.DataLocation(self.filesbyblock.keys(),cfg_params)
403	>	dataloc.fetchDLSInfo()
404
405	<	except DataLocation_EDM.DataLocationError , ex:
405	>	except DataLocation.DataLocationError , ex:
406		msg = 'ERROR ***: failed Data Location in DLS \n %s '%ex.getErrorMessage()
407		raise CrabException(msg)
408	<
409	<	allsites=dataloc.getSites()
410	<	common.logger.debug(5,"sites are %s"%allsites)
411	<	sites=self.checkBlackList(allsites)
412	<	common.logger.debug(5,"sites are (after black list) %s"%sites)
413	<	sites=self.checkWhiteList(sites)
414	<	common.logger.debug(5,"sites are (after white list) %s"%sites)
408	>
409	>
410	>	unsorted_sites = dataloc.getSites()
411	>	sites = self.filesbyblock.fromkeys(self.filesbyblock,'')
412	>	for lfn in self.filesbyblock.keys():
413	>	if unsorted_sites.has_key(lfn):
414	>	sites[lfn]=unsorted_sites[lfn]
415	>	else:
416	>	sites[lfn]=[]
417
418		if len(sites)==0:
419	<	msg = 'No sites hosting all the needed data! Exiting... '
419	>	msg = 'ERROR ***: no location for any of the blocks of this dataset: \n\t %s \n'%datasetPath
420	>	msg += "\tMaybe the dataset is located only at T1's (or at T0), where analysis jobs are not allowed\n"
421	>	msg += "\tPlease check DataDiscovery page https://cmsweb.cern.ch/dbs_discovery/\n"
422		raise CrabException(msg)
423
424	<	common.logger.message("List of Sites hosting the data : "+str(sites))
425	<	common.logger.debug(6, "List of Sites: "+str(sites))
426	<	common.analisys_common_info['sites']=sites    ## used in SchedulerEdg.py in createSchScript
427	<	return
428	<
429	<	def jobSplitting(self):
430	<	"""
431	<	first implemntation for job splitting
432	<	"""
433	<	#  print 'eventi totali '+str(self.maxEvents)
434	<	#  print 'eventi totali richiesti dallo user '+str(self.total_number_of_events)
435	<	#print 'files per job '+str(self.filesPerJob)
281	<	common.logger.message('Required '+str(self.filesPerJob)+' files per job ')
282	<	common.logger.message('Required '+str(self.total_number_of_events)+' events in total ')
283	<
284	<	## TODO: SL need to have (from DBS) a detailed list of how many events per each file
285	<	n_tot_files = (len(self.files[0]))
286	<	## SL: this is wrong if the files have different number of events
287	<	evPerFile = int(self.maxEvents)/n_tot_files
288	<
289	<	common.logger.debug(5,'Events per File '+str(evPerFile))
290	<
291	<	## if asked to process all events, do it
292	<	if self.total_number_of_events == -1:
293	<	self.total_number_of_events=self.maxEvents
294	<	self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
295	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for all available events '+str(self.total_number_of_events)+' events')
296	<
424	>	allSites = []
425	>	listSites = sites.values()
426	>	for listSite in listSites:
427	>	for oneSite in listSite:
428	>	allSites.append(oneSite)
429	>	[allSites.append(it) for it in allSites if not allSites.count(it)]
430	>
431	>
432	>	# screen output
433	>	if self.ads or self.lumiMask:
434	>	common.logger.info("Requested (A)DS %s has %s block(s)." %
435	>	(datasetPath, len(self.filesbyblock.keys())))
436		else:
437	<	self.total_number_of_files = int(self.total_number_of_events/evPerFile)
438	<	## SL: if ask for less event than what is computed to be available on a
439	<	##     file, process the first file anyhow.
301	<	if self.total_number_of_files == 0:
302	<	self.total_number_of_files = self.total_number_of_files + 1
303	<
304	<	common.logger.debug(5,'N files  '+str(self.total_number_of_files))
305	<
306	<	check = 0
307	<
308	<	## Compute the number of jobs
309	<	#self.total_number_of_jobs = int(n_tot_files)*1/int(self.filesPerJob)
310	<	self.total_number_of_jobs = int(self.total_number_of_files/self.filesPerJob)
311	<	common.logger.debug(5,'N jobs  '+str(self.total_number_of_jobs))
312	<
313	<	## is there any remainder?
314	<	check = int(self.total_number_of_files) - (int(self.total_number_of_jobs)*self.filesPerJob)
315	<
316	<	common.logger.debug(5,'Check  '+str(check))
317	<
318	<	if check > 0:
319	<	self.total_number_of_jobs =  self.total_number_of_jobs + 1
320	<	common.logger.message('Warning: last job will be created with '+str(check)+' files')
437	>	common.logger.info("Requested dataset: " + datasetPath + \
438	>	" has " + str(self.maxEvents) + " events in " + \
439	>	str(len(self.filesbyblock.keys())) + " blocks.\n")
440
441	<	common.logger.message(str(self.total_number_of_jobs)+' jobs will be created for a total of '+str((self.total_number_of_jobs-1)*self.filesPerJob*evPerFile + check*evPerFile)+' events')
323	<	pass
441	>	return sites
442
325	–	list_of_lists = []
326	–	for i in xrange(0, int(n_tot_files), self.filesPerJob):
327	–	list_of_lists.append(self.files[0][i: i+self.filesPerJob])
443
444	<	self.list_of_files = list_of_lists
445	<
446	<	return
444	>	def split(self, jobParams,firstJobID):
445	>
446	>	jobParams = self.dict['args']
447	>	njobs = self.dict['njobs']
448	>	self.jobDestination = self.dict['jobDestination']
449	>
450	>	if njobs == 0:
451	>	raise CrabException("Asked to split zero jobs: aborting")
452	>	if not self.server and not self.local and njobs > 500:
453	>	raise CrabException("The CRAB client will not submit more than 500 jobs. You must use the server mode.")
454
333	–	def split(self, jobParams):
334	–
335	–	common.jobDB.load()
336	–	#### Fabio
337	–	njobs = self.total_number_of_jobs
338	–	filelist = self.list_of_files
455		# create the empty structure
456		for i in range(njobs):
457		jobParams.append("")
342	–
343	–	for job in range(njobs):
344	–	jobParams[job] = filelist[job]
345	–	common.jobDB.setArguments(job, jobParams[job])
458
459	<	common.jobDB.save()
459	>	listID=[]
460	>	listField=[]
461	>	listDictions=[]
462	>	exist= os.path.exists(self.argsFile)
463	>	for id in range(njobs):
464	>	job = id + int(firstJobID)
465	>	listID.append(job+1)
466	>	job_ToSave ={}
467	>	concString = ' '
468	>	argu=''
469	>	str_argu = str(job+1)
470	>	if len(jobParams[id]):
471	>	argu = {'JobID': job+1}
472	>	for i in range(len(jobParams[id])):
473	>	argu[self.dict['params'][i]]=jobParams[id][i]
474	>	if len(jobParams[id])==1: self.NumEvents = jobParams[id][i]
475	>	# just for debug
476	>	str_argu += concString.join(jobParams[id])
477	>	if argu != '': listDictions.append(argu)
478	>	job_ToSave['arguments']= '%d %d'%( (job+1), 0)
479	>	job_ToSave['dlsDestination']= self.jobDestination[id]
480	>	listField.append(job_ToSave)
481	>	from ProdCommon.SiteDB.CmsSiteMapper import CmsSEMap
482	>	cms_se = CmsSEMap()
483	>	msg="Job  %s  Arguments:  %s\n"%(str(job+1),str_argu)
484	>	msg+="\t  Destination: %s "%(str(self.jobDestination[id]))
485	>	SEDestination = [cms_se[dest] for dest in self.jobDestination[id]]
486	>	msg+="\t  CMSDestination: %s "%(str(SEDestination))
487	>	common.logger.log(10-1,msg)
488	>	# write xml
489	>	if len(listDictions):
490	>	if exist==False: self.CreateXML()
491	>	self.addEntry(listDictions)
492	>	common._db.updateJob_(listID,listField)
493		return
349	–
350	–	def getJobTypeArguments(self, nj, sched):
351	–	params = common.jobDB.arguments(nj)
352	–	#print params
353	–	parString = "\\{"
354	–
355	–	for i in range(len(params) - 1):
356	–	parString += '\\\"' + params[i] + '\\\"\,'
357	–
358	–	parString += '\\\"' + params[len(params) - 1] + '\\\"\\}'
359	–	return parString
360	–
361	–	def numberOfJobs(self):
362	–	# Fabio
363	–
364	–	return self.total_number_of_jobs
365	–
494
495	+	def CreateXML(self):
496	+	"""
497	+	"""
498	+	result = IMProvNode( self.rootArgsFilename )
499	+	outfile = file( self.argsFile, 'w').write(str(result))
500	+	return
501
502	<	def checkBlackList(self, allSites):
503	<	if len(self.reCEBlackList)==0: return allSites
504	<	sites = []
371	<	for site in allSites:
372	<	common.logger.debug(10,'Site '+site)
373	<	good=1
374	<	for re in self.reCEBlackList:
375	<	if re.search(site):
376	<	common.logger.message('CE in black list, skipping site '+site)
377	<	good=0
378	<	pass
379	<	if good: sites.append(site)
380	<	if len(sites) == 0:
381	<	common.logger.debug(3,"No sites found after BlackList")
382	<	return sites
502	>	def addEntry(self, listDictions):
503	>	"""
504	>	_addEntry_
505
506	<	def checkWhiteList(self, allSites):
506	>	add an entry to the xml file
507	>	"""
508	>	## load xml
509	>	improvDoc = loadIMProvFile(self.argsFile)
510	>	entrname= 'Job'
511	>	for dictions in listDictions:
512	>	report = IMProvNode(entrname , None, **dictions)
513	>	improvDoc.addNode(report)
514	>	outfile = file( self.argsFile, 'w').write(str(improvDoc))
515	>	return
516
517	<	if len(self.reCEWhiteList)==0: return allSites
518	<	sites = []
519	<	for site in allSites:
520	<	good=0
390	<	for re in self.reCEWhiteList:
391	<	if re.search(site):
392	<	common.logger.debug(5,'CE in white list, adding site '+site)
393	<	good=1
394	<	if not good: continue
395	<	sites.append(site)
396	<	if len(sites) == 0:
397	<	common.logger.message("No sites found after WhiteList\n")
517	>	def numberOfJobs(self):
518	>	#wmbs
519	>	if self.automation==0:
520	>	return self.dict['njobs']
521		else:
522	<	common.logger.debug(5,"Selected sites via WhiteList are "+str(sites)+"\n")
400	<	return sites
522	>	return None
523
524		def getTarBall(self, exe):
525		"""
526		Return the TarBall with lib and exe
527		"""
528	<
407	<	# if it exist, just return it
408	<	self.tgzNameWithPath = common.work_space.shareDir()+self.tgz_name
528	>	self.tgzNameWithPath = common.work_space.pathForTgz()+self.tgz_name
529		if os.path.exists(self.tgzNameWithPath):
530		return self.tgzNameWithPath
531
#	Line 418 | Line 538 | class Cmssw(JobType):
538
539		# First of all declare the user Scram area
540		swArea = self.scram.getSWArea_()
421	–	#print "swArea = ", swArea
422	–	swVersion = self.scram.getSWVersion()
423	–	#print "swVersion = ", swVersion
541		swReleaseTop = self.scram.getReleaseTop_()
542	<	#print "swReleaseTop = ", swReleaseTop
426	<
542	>
543		## check if working area is release top
544		if swReleaseTop == '' or swArea == swReleaseTop:
545	+	common.logger.debug("swArea = "+swArea+" swReleaseTop ="+swReleaseTop)
546		return
547
548	<	filesToBeTarred = []
549	<	## First find the executable
550	<	if (self.executable != ''):
551	<	exeWithPath = self.scram.findFile_(executable)
552	<	#           print exeWithPath
553	<	if ( not exeWithPath ):
554	<	raise CrabException('User executable '+executable+' not found')
555	<
556	<	## then check if it's private or not
557	<	if exeWithPath.find(swReleaseTop) == -1:
558	<	# the exe is private, so we must ship
559	<	common.logger.debug(5,"Exe "+exeWithPath+" to be tarred")
560	<	path = swArea+'/'
561	<	exe = string.replace(exeWithPath, path,'')
562	<	filesToBeTarred.append(exe)
548	>	import tarfile
549	>	try: # create tar ball
550	>	tar = tarfile.open(self.tgzNameWithPath, "w:gz")
551	>	## First find the executable
552	>	if (self.executable != ''):
553	>	exeWithPath = self.scram.findFile_(executable)
554	>	if ( not exeWithPath ):
555	>	raise CrabException('User executable '+executable+' not found')
556	>
557	>	## then check if it's private or not
558	>	if exeWithPath.find(swReleaseTop) == -1:
559	>	# the exe is private, so we must ship
560	>	common.logger.debug("Exe "+exeWithPath+" to be tarred")
561	>	path = swArea+'/'
562	>	# distinguish case when script is in user project area or given by full path somewhere else
563	>	if exeWithPath.find(path) >= 0 :
564	>	exe = string.replace(exeWithPath, path,'')
565	>	tar.add(path+exe,exe)
566	>	else :
567	>	tar.add(exeWithPath,os.path.basename(executable))
568	>	pass
569	>	else:
570	>	# the exe is from release, we'll find it on WN
571	>	pass
572	>
573	>	## Now get the libraries: only those in local working area
574	>	tar.dereference=True
575	>	libDir = 'lib'
576	>	lib = swArea+'/' +libDir
577	>	common.logger.debug("lib "+lib+" to be tarred")
578	>	if os.path.exists(lib):
579	>	tar.add(lib,libDir)
580	>
581	>	## Now check if module dir is present
582	>	moduleDir = 'module'
583	>	module = swArea + '/' + moduleDir
584	>	if os.path.isdir(module):
585	>	tar.add(module,moduleDir)
586	>	tar.dereference=False
587	>
588	>	## Now check if any data dir(s) is present
589	>	self.dataExist = False
590	>	todo_list = [(i, i) for i in  os.listdir(swArea+"/src")]
591	>	while len(todo_list):
592	>	entry, name = todo_list.pop()
593	>	if name.startswith('crab_0_') or  name.startswith('.') or name == 'CVS':
594	>	continue
595	>	if os.path.isdir(swArea+"/src/"+entry):
596	>	entryPath = entry + '/'
597	>	todo_list += [(entryPath + i, i) for i in  os.listdir(swArea+"/src/"+entry)]
598	>	if name == 'data':
599	>	self.dataExist=True
600	>	common.logger.debug("data "+entry+" to be tarred")
601	>	tar.add(swArea+"/src/"+entry,"src/"+entry)
602	>	pass
603		pass
604	<	else:
605	<	# the exe is from release, we'll find it on WN
604	>
605	>	### CMSSW ParameterSet
606	>	if not self.pset is None:
607	>	cfg_file = common.work_space.jobDir()+self.configFilename()
608	>	pickleFile = common.work_space.jobDir()+self.configFilename() + '.pkl'
609	>	tar.add(cfg_file,self.configFilename())
610	>	tar.add(pickleFile,self.configFilename() + '.pkl')
611	>
612	>	try:
613	>	crab_cfg_file = common.work_space.shareDir()+'/crab.cfg'
614	>	tar.add(crab_cfg_file,'crab.cfg')
615	>	except:
616		pass
617	<
618	<	## Now get the libraries: only those in local working area
619	<	libDir = 'lib'
620	<	lib = swArea+'/' +libDir
621	<	common.logger.debug(5,"lib "+lib+" to be tarred")
622	<	if os.path.exists(lib):
623	<	filesToBeTarred.append(libDir)
624	<
625	<	## Now check if module dir is present
626	<	moduleDir = 'module'
627	<	if os.path.isdir(swArea+'/'+moduleDir):
628	<	filesToBeTarred.append(moduleDir)
629	<
630	<	## Now check if the Data dir is present
631	<	dataDir = 'src/Data/'
632	<	if os.path.isdir(swArea+'/'+dataDir):
633	<	filesToBeTarred.append(dataDir)
634	<
635	<	## Create the tar-ball
636	<	if len(filesToBeTarred)>0:
637	<	cwd = os.getcwd()
638	<	os.chdir(swArea)
639	<	tarcmd = 'tar zcvf ' + self.tgzNameWithPath + ' '
640	<	for line in filesToBeTarred:
641	<	tarcmd = tarcmd + line + ' '
642	<	cout = runCommand(tarcmd)
643	<	if not cout:
644	<	raise CrabException('Could not create tar-ball')
645	<	os.chdir(cwd)
646	<	else:
647	<	common.logger.debug(5,"No files to be to be tarred")
648	<
649	<	return
650	<
651	<	def wsSetupEnvironment(self, nj):
617	>
618	>	## Add ProdCommon dir to tar
619	>	prodcommonDir = './'
620	>	prodcommonPath = os.environ['CRABDIR'] + '/' + 'external/'
621	>	neededStuff = ['ProdCommon/__init__.py','ProdCommon/FwkJobRep', 'ProdCommon/CMSConfigTools', \
622	>	'ProdCommon/Core', 'ProdCommon/MCPayloads', 'IMProv', 'ProdCommon/Storage', \
623	>	'WMCore/__init__.py','WMCore/Algorithms']
624	>	for file in neededStuff:
625	>	tar.add(prodcommonPath+file,prodcommonDir+file)
626	>
627	>	##### ML stuff
628	>	ML_file_list=['report.py', 'DashboardAPI.py', 'Logger.py', 'ProcInfo.py', 'apmon.py']
629	>	path=os.environ['CRABDIR'] + '/python/'
630	>	for file in ML_file_list:
631	>	tar.add(path+file,file)
632	>
633	>	##### Utils
634	>	Utils_file_list=['parseCrabFjr.py','writeCfg.py', 'fillCrabFjr.py','cmscp.py']
635	>	for file in Utils_file_list:
636	>	tar.add(path+file,file)
637	>
638	>	##### AdditionalFiles
639	>	tar.dereference=True
640	>	for file in self.additional_inbox_files:
641	>	tar.add(file,string.split(file,'/')[-1])
642	>	tar.dereference=False
643	>	common.logger.log(10-1,"Files in "+self.tgzNameWithPath+" : "+str(tar.getnames()))
644	>
645	>	tar.close()
646	>	except IOError, exc:
647	>	msg = 'Could not create tar-ball %s \n'%self.tgzNameWithPath
648	>	msg += str(exc)
649	>	raise CrabException(msg)
650	>	except tarfile.TarError, exc:
651	>	msg = 'Could not create tar-ball %s \n'%self.tgzNameWithPath
652	>	msg += str(exc)
653	>	raise CrabException(msg)
654	>
655	>	tarballinfo = os.stat(self.tgzNameWithPath)
656	>	if ( tarballinfo.st_size > self.MaxTarBallSize*1024*1024 ) :
657	>	if not self.server:
658	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' + \
659	>	str(self.MaxTarBallSize) +'MB input sandbox limit \n'
660	>	msg += '      and not supported by the direct GRID submission system.\n'
661	>	msg += '      Please use the CRAB server mode by setting server_name=<NAME> in section [CRAB] of your crab.cfg.\n'
662	>	msg += '      For further infos please see https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabServerForUsers#Server_available_for_users'
663	>	else:
664	>	msg  = 'Input sandbox size of ' + str(float(tarballinfo.st_size)/1024.0/1024.0) + ' MB is larger than the allowed ' +  \
665	>	str(self.MaxTarBallSize) +'MB input sandbox limit in the server.'
666	>	raise CrabException(msg)
667	>
668	>	## create tar-ball with ML stuff
669	>
670	>	def wsSetupEnvironment(self, nj=0):
671		"""
672		Returns part of a job script which prepares
673		the execution environment for the job 'nj'.
674		"""
675	<	# Prepare JobType-independent part
490	<	txt = ''
491	<
492	<	## OLI_Daniele at this level  middleware already known
675	>	psetName = 'pset.py'
676
677	<	txt += 'if [ $middleware == LCG ]; then \n'
677	>	# Prepare JobType-independent part
678	>	txt = '\n#Written by cms_cmssw::wsSetupEnvironment\n'
679	>	txt += 'echo ">>> setup environment"\n'
680	>	txt += 'echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
681	>	txt += 'export SCRAM_ARCH=' + self.executable_arch + '\n'
682	>	txt += 'echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
683	>	txt += 'if [ $middleware == LCG ] || [ $middleware == CAF ] || [ $middleware == LSF ]; then \n'
684		txt += self.wsSetupCMSLCGEnvironment_()
685		txt += 'elif [ $middleware == OSG ]; then\n'
686	<	txt += '    time=`date -u +"%s"`\n'
687	<	txt += '    WORKING_DIR=$OSG_WN_TMP/cms_$time\n'
688	<	txt += '    echo "Creating working directory: $WORKING_DIR"\n'
689	<	txt += '    /bin/mkdir -p $WORKING_DIR\n'
690	<	txt += '    if [ ! -d $WORKING_DIR ] ;then\n'
502	<	txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be created on on WN `hostname`"\n'
503	<
504	<	txt += '        echo "JOB_EXIT_STATUS = 1"\n'
505	<	txt += '        exit 1\n'
686	>	txt += '    WORKING_DIR=`/bin/mktemp  -d $OSG_WN_TMP/cms_XXXXXXXXXXXX`\n'
687	>	txt += '    if [ ! $? == 0 ] ;then\n'
688	>	txt += '        echo "ERROR ==> OSG $WORKING_DIR could not be created on WN `hostname`"\n'
689	>	txt += '        job_exit_code=10016\n'
690	>	txt += '        func_exit\n'
691		txt += '    fi\n'
692	+	txt += '    echo ">>> Created working directory: $WORKING_DIR"\n'
693		txt += '\n'
694		txt += '    echo "Change to working directory: $WORKING_DIR"\n'
695		txt += '    cd $WORKING_DIR\n'
696	<	txt += self.wsSetupCMSOSGEnvironment_()
696	>	txt += '    echo ">>> current directory (WORKING_DIR): $WORKING_DIR"\n'
697	>	txt += self.wsSetupCMSOSGEnvironment_()
698	>	#Setup SGE Environment
699	>	txt += 'elif [ $middleware == SGE ]; then\n'
700	>	txt += self.wsSetupCMSLCGEnvironment_()
701	>
702	>	txt += 'elif [ $middleware == ARC ]; then\n'
703	>	txt += self.wsSetupCMSLCGEnvironment_()
704	>
705	>	#Setup PBS Environment
706	>	txt += 'elif [ $middleware == PBS ]; then\n'
707	>	txt += self.wsSetupCMSLCGEnvironment_()
708	>
709		txt += 'fi\n'
710
711		# Prepare JobType-specific part
712		scram = self.scram.commandName()
713		txt += '\n\n'
714	<	txt += 'echo "### SPECIFIC JOB SETUP ENVIRONMENT ###"\n'
714	>	txt += 'echo ">>> specific cmssw setup environment:"\n'
715	>	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
716		txt += scram+' project CMSSW '+self.version+'\n'
717		txt += 'status=$?\n'
718		txt += 'if [ $status != 0 ] ; then\n'
719	<	txt += '   echo "SET_EXE_ENV 1 ==>ERROR CMSSW '+self.version+' not found on `hostname`" \n'
720	<	txt += '   echo "JOB_EXIT_STATUS = 10034"\n'
721	<	txt += '   echo "SanityCheckCode = 10034" | tee -a $RUNTIME_AREA/$repo\n'
523	<	txt += '   dumpStatus $RUNTIME_AREA/$repo\n'
524	<	## OLI_Daniele
525	<	txt += '    if [ $middleware == OSG ]; then \n'
526	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
527	<	txt += '        cd $RUNTIME_AREA\n'
528	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
529	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
530	<	txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
531	<	txt += '        fi\n'
532	<	txt += '    fi \n'
533	<	txt += '   exit 1 \n'
719	>	txt += '    echo "ERROR ==> CMSSW '+self.version+' not found on `hostname`" \n'
720	>	txt += '    job_exit_code=10034\n'
721	>	txt += '    func_exit\n'
722		txt += 'fi \n'
535	–	txt += 'echo "CMSSW_VERSION =  '+self.version+'"\n'
723		txt += 'cd '+self.version+'\n'
724	<	### needed grep for bug in scramv1 ###
724	>	txt += 'SOFTWARE_DIR=`pwd`; export SOFTWARE_DIR\n'
725	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
726		txt += 'eval `'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME`\n'
727	<
727	>	txt += 'if [ $? != 0 ] ; then\n'
728	>	txt += '    echo "ERROR ==> Problem with the command: "\n'
729	>	txt += '    echo "eval \`'+scram+' runtime -sh | grep -v SCRAMRT_LSB_JOBNAME \` at `hostname`"\n'
730	>	txt += '    job_exit_code=10034\n'
731	>	txt += '    func_exit\n'
732	>	txt += 'fi \n'
733		# Handle the arguments:
734		txt += "\n"
735	<	txt += "## ARGUMNETS: $1 Job Number\n"
543	<	# txt += "## ARGUMNETS: $2 First Event for this job\n"
544	<	# txt += "## ARGUMNETS: $3 Max Event for this job\n"
735	>	txt += "## number of arguments (first argument always jobnumber, the second is the resubmission number)\n"
736		txt += "\n"
737	<	txt += "narg=$#\n"
547	<	txt += "if [ $narg -lt 2 ]\n"
737	>	txt += "if [ $nargs -lt "+str(self.argsList)+" ]\n"
738		txt += "then\n"
739	<	txt += "    echo 'SET_EXE_ENV 1 ==> ERROR Too few arguments' +$narg+ \n"
740	<	txt += '    echo "JOB_EXIT_STATUS = 50113"\n'
741	<	txt += '    echo "SanityCheckCode = 50113" | tee -a $RUNTIME_AREA/$repo\n'
552	<	txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
553	<	## OLI_Daniele
554	<	txt += '    if [ $middleware == OSG ]; then \n'
555	<	txt += '        echo "Remove working directory: $WORKING_DIR"\n'
556	<	txt += '        cd $RUNTIME_AREA\n'
557	<	txt += '        /bin/rm -rf $WORKING_DIR\n'
558	<	txt += '        if [ -d $WORKING_DIR ] ;then\n'
559	<	txt += '            echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
560	<	txt += '        fi\n'
561	<	txt += '    fi \n'
562	<	txt += "    exit 1\n"
739	>	txt += "    echo 'ERROR ==> Too few arguments' +$nargs+ \n"
740	>	txt += '    job_exit_code=50113\n'
741	>	txt += "    func_exit\n"
742		txt += "fi\n"
743		txt += "\n"
565	–	txt += "NJob=$1\n"
566	–	txt += "InputFiles=$2\n"
567	–	txt += "echo \"<$InputFiles>\"\n"
568	–	# txt += "Args = ` cat $2 |  sed -e \'s/\\\\//g\' -e \'s/\"/\\x27/g\' `"
569	–
570	–	### OLI_DANIELE
571	–	txt += 'if [ $middleware == LCG ]; then \n'
572	–	txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
573	–	txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
574	–	txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo\n'
575	–	txt += 'elif [ $middleware == OSG ]; then\n'
576	–
577	–	# OLI: added monitoring for dashbord, use hash of crab.cfg
578	–	if common.scheduler.boss_scheduler_name == 'condor_g':
579	–	# create hash of cfg file
580	–	hash = makeCksum(common.work_space.cfgFileName())
581	–	txt += '    echo "MonitorJobID=`echo ${NJob}_'+hash+'_$GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
582	–	txt += '    echo "SyncGridJobId=`echo $GLOBUS_GRAM_JOB_CONTACT`" | tee -a $RUNTIME_AREA/$repo\n'
583	–	txt += '    echo "SyncCE=`echo $hostname`" | tee -a $RUNTIME_AREA/$repo\n'
584	–	else :
585	–	txt += '    echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
586	–	txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
587	–	txt += '    echo "SyncCE=`$EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo\n'
588	–
589	–	txt += 'fi\n'
590	–	txt += 'dumpStatus $RUNTIME_AREA/$repo\n'
744
745		# Prepare job-specific part
746		job = common.job_list[nj]
747	<	pset = os.path.basename(job.configFilename())
748	<	txt += '\n'
749	<	#txt += 'echo sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' \n'
597	<	txt += 'sed "s#{\'INPUT\'}#$InputFiles#" $RUNTIME_AREA/'+pset+' > pset.cfg\n'
598	<	#txt += 'sed "s#{\'INPUT\'}#${InputFiles}#" $RUNTIME_AREA/'+pset+' > pset1.cfg\n'
747	>	if (self.datasetPath):
748	>	txt += '\n'
749	>	txt += 'DatasetPath='+self.datasetPath+'\n'
750
751	<	if len(self.additional_inbox_files) > 0:
752	<	for file in self.additional_inbox_files:
753	<	txt += 'if [ -e $RUNTIME_AREA/'+file+' ] ; then\n'
603	<	txt += '   cp $RUNTIME_AREA/'+file+' .\n'
604	<	txt += '   chmod +x '+file+'\n'
605	<	txt += 'fi\n'
606	<	pass
751	>	txt += 'PrimaryDataset='+self.primaryDataset +'\n'
752	>	txt += 'DataTier='+self.dataTier+'\n'
753	>	txt += 'ApplicationFamily=cmsRun\n'
754
755	<	txt += 'echo "### END JOB SETUP ENVIRONMENT ###"\n\n'
755	>	else:
756	>	txt += 'DatasetPath=MCDataTier\n'
757	>	txt += 'PrimaryDataset=null\n'
758	>	txt += 'DataTier=null\n'
759	>	txt += 'ApplicationFamily=MCDataTier\n'
760	>	if self.pset != None:
761	>	pset = os.path.basename(job.configFilename())
762	>	pkl  = os.path.basename(job.configFilename()) + '.pkl'
763	>	txt += '\n'
764	>	txt += 'cp  $RUNTIME_AREA/'+pset+' .\n'
765	>	txt += 'cp  $RUNTIME_AREA/'+pkl+' .\n'
766
767	<	txt += '\n'
768	<	txt += 'echo "***** cat pset.cfg *********"\n'
769	<	txt += 'cat pset.cfg\n'
770	<	txt += 'echo "****** end pset.cfg ********"\n'
771	<	txt += '\n'
772	<	# txt += 'echo "***** cat pset1.cfg *********"\n'
773	<	# txt += 'cat pset1.cfg\n'
774	<	# txt += 'echo "****** end pset1.cfg ********"\n'
767	>	txt += 'PreserveSeeds='  + ','.join(self.preserveSeeds)  + '; export PreserveSeeds\n'
768	>	txt += 'IncrementSeeds=' + ','.join(self.incrementSeeds) + '; export IncrementSeeds\n'
769	>	txt += 'echo "PreserveSeeds: <$PreserveSeeds>"\n'
770	>	txt += 'echo "IncrementSeeds:<$IncrementSeeds>"\n'
771	>
772	>	txt += 'mv -f ' + pset + ' ' + psetName + '\n'
773	>	if self.var_filter:
774	>	#print "self.var_filter = ",self.var_filter
775	>	txt += "export var_filter="+"'"+self.var_filter+"'\n"
776	>	txt += 'echo $var_filter'
777	>	else:
778	>	txt += '\n'
779	>	if self.AdditionalArgs: txt += 'export AdditionalArgs=\"%s\"\n'%(self.AdditionalArgs)
780	>	if int(self.NumEvents) != 0: txt += 'export MaxEvents=%s\n'%str(self.NumEvents)
781		return txt
782
783	<	def wsBuildExe(self, nj):
783	>	def wsUntarSoftware(self, nj=0):
784		"""
785		Put in the script the commands to build an executable
786		or a library.
787		"""
788
789	<	txt = ""
789	>	txt = '\n#Written by cms_cmssw::wsUntarSoftware\n'
790
791		if os.path.isfile(self.tgzNameWithPath):
792	<	txt += 'echo "tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'"\n'
793	<	txt += 'tar xzvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
792	>	txt += 'echo ">>> tar xzf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+' :" \n'
793	>	if  self.debug_wrapper==1 :
794	>	txt += 'tar zxvf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
795	>	txt += 'ls -Al \n'
796	>	else:
797	>	txt += 'tar zxf $RUNTIME_AREA/'+os.path.basename(self.tgzNameWithPath)+'\n'
798		txt += 'untar_status=$? \n'
799		txt += 'if [ $untar_status -ne 0 ]; then \n'
800	<	txt += '   echo "SET_EXE 1 ==> ERROR Untarring .tgz file failed"\n'
801	<	txt += '   echo "JOB_EXIT_STATUS = $untar_status" \n'
802	<	txt += '   echo "SanityCheckCode = $untar_status" | tee -a $repo\n'
636	<	txt += '   if [ $middleware == OSG ]; then \n'
637	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
638	<	txt += '       cd $RUNTIME_AREA\n'
639	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
640	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
641	<	txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
642	<	txt += '       fi\n'
643	<	txt += '   fi \n'
644	<	txt += '   \n'
645	<	txt += '   exit $untar_status \n'
800	>	txt += '   echo "ERROR ==> Untarring .tgz file failed"\n'
801	>	txt += '   job_exit_code=$untar_status\n'
802	>	txt += '   func_exit\n'
803		txt += 'else \n'
804		txt += '   echo "Successful untar" \n'
805		txt += 'fi \n'
806	+	txt += '\n'
807	+	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
808	+	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
809	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
810	+	txt += 'else\n'
811	+	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
812	+	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
813	+	txt += 'fi\n'
814	+	txt += '\n'
815	+
816		pass
817	<
817	>
818		return txt
819
820	<	def modifySteeringCards(self, nj):
820	>	def wsBuildExe(self, nj=0):
821		"""
822	<	modify the card provided by the user,
823	<	writing a new card into share dir
822	>	Put in the script the commands to build an executable
823	>	or a library.
824		"""
825	<
825	>
826	>	txt = '\n#Written by cms_cmssw::wsBuildExe\n'
827	>	txt += 'echo ">>> moving CMSSW software directories in `pwd`" \n'
828	>
829	>	txt += 'rm -r lib/ module/ \n'
830	>	txt += 'mv $RUNTIME_AREA/lib/ . \n'
831	>	txt += 'mv $RUNTIME_AREA/module/ . \n'
832	>	if self.dataExist == True:
833	>	txt += 'rm -r src/ \n'
834	>	txt += 'mv $RUNTIME_AREA/src/ . \n'
835	>	if len(self.additional_inbox_files)>0:
836	>	for file in self.additional_inbox_files:
837	>	txt += 'mv $RUNTIME_AREA/'+os.path.basename(file)+' . \n'
838	>
839	>	txt += 'echo ">>> Include $RUNTIME_AREA in PYTHONPATH:"\n'
840	>	txt += 'if [ -z "$PYTHONPATH" ]; then\n'
841	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/\n'
842	>	txt += 'else\n'
843	>	txt += '   export PYTHONPATH=$RUNTIME_AREA/:${PYTHONPATH}\n'
844	>	txt += 'echo "PYTHONPATH=$PYTHONPATH"\n'
845	>	txt += 'fi\n'
846	>	txt += '\n'
847	>
848	>	if self.pset != None:
849	>	psetName = 'pset.py'
850	>
851	>	txt += '\n'
852	>	if self.debug_wrapper == 1:
853	>	txt += 'echo "***** cat ' + psetName + ' *********"\n'
854	>	txt += 'cat ' + psetName + '\n'
855	>	txt += 'echo "****** end ' + psetName + ' ********"\n'
856	>	txt += '\n'
857	>	txt += 'echo "***********************" \n'
858	>	txt += 'which edmConfigHash \n'
859	>	txt += 'echo "***********************" \n'
860	>	txt += 'edmConfigHash ' + psetName + ' \n'
861	>	txt += 'PSETHASH=`edmConfigHash ' + psetName + '` \n'
862	>	txt += 'echo "PSETHASH = $PSETHASH" \n'
863	>	#### FEDE temporary fix for noEdm files #####
864	>	txt += 'if [ -z "$PSETHASH" ]; then \n'
865	>	txt += '   export PSETHASH=null\n'
866	>	txt += 'fi \n'
867	>	#############################################
868	>	txt += '\n'
869	>	return txt
870	>
871	>
872		def executableName(self):
873	<	return self.executable
873	>	if self.scriptExe:
874	>	return "sh "
875	>	else:
876	>	return self.executable
877
878		def executableArgs(self):
879	<	return " -p pset.cfg"
879	>	if self.scriptExe:
880	>	return self.scriptExe + " $NJob $AdditionalArgs"
881	>	else:
882	>	return " -j $RUNTIME_AREA/crab_fjr_$NJob.xml -p pset.py"
883
884		def inputSandbox(self, nj):
885		"""
886		Returns a list of filenames to be put in JDL input sandbox.
887		"""
888		inp_box = []
670	–	# dict added to delete duplicate from input sandbox file list
671	–	seen = {}
672	–	## code
889		if os.path.isfile(self.tgzNameWithPath):
890		inp_box.append(self.tgzNameWithPath)
891	<	## config
892	<	inp_box.append(common.job_list[nj].configFilename())
893	<	## additional input files
678	<	#for file in self.additional_inbox_files:
679	<	#    inp_box.append(common.work_space.cwdDir()+file)
891	>	if os.path.isfile(self.argsFile):
892	>	inp_box.append(self.argsFile)
893	>	inp_box.append(common.work_space.jobDir() + self.scriptName)
894		return inp_box
895
896		def outputSandbox(self, nj):
#	Line 685 | Line 899 | class Cmssw(JobType):
899		"""
900		out_box = []
901
688	–	stdout=common.job_list[nj].stdout()
689	–	stderr=common.job_list[nj].stderr()
690	–
902		## User Declared output files
903	<	for out in self.output_file:
904	<	n_out = nj + 1
905	<	out_box.append(self.numberFile_(out,str(n_out)))
903	>	for out in (self.output_file+self.output_file_sandbox):
904	>	n_out = nj + 1
905	>	out_box.append(numberFile(out,str(n_out)))
906		return out_box
696	–	return []
907
698	–	def prepareSteeringCards(self):
699	–	"""
700	–	Make initial modifications of the user's steering card file.
701	–	"""
702	–	return
908
909		def wsRenameOutput(self, nj):
910		"""
911		Returns part of a job script which renames the produced files.
912		"""
913
914	<	txt = '\n'
915	<	file_list = ''
916	<	check = len(self.output_file)
917	<	i = 0
918	<	for fileWithSuffix in self.output_file:
919	<	i= i + 1
920	<	output_file_num = self.numberFile_(fileWithSuffix, '$NJob')
921	<	file_list=file_list+output_file_num+''
922	<	txt += '\n'
718	<	txt += 'ls \n'
914	>	txt = '\n#Written by cms_cmssw::wsRenameOutput\n'
915	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
916	>	txt += 'echo ">>> current directory content:"\n'
917	>	if self.debug_wrapper==1:
918	>	txt += 'ls -Al\n'
919	>	txt += '\n'
920	>
921	>	for fileWithSuffix in (self.output_file):
922	>	output_file_num = numberFile(fileWithSuffix, '$OutUniqueID')
923		txt += '\n'
924	<	txt += 'ls '+fileWithSuffix+'\n'
925	<	txt += 'exe_result=$?\n'
926	<	txt += 'if [ $exe_result -ne 0 ] ; then\n'
927	<	txt += '   echo "ERROR: No output file to manage"\n'
928	<	### OLI_DANIELE
929	<	txt += '    if [ $middleware == OSG ]; then \n'
930	<	txt += '        echo "prepare dummy output file"\n'
931	<	txt += '        cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
728	<	txt += '    fi \n'
924	>	txt += '# check output file\n'
925	>	txt += 'if [ -e ./'+fileWithSuffix+' ] ; then\n'
926	>	if (self.copy_data == 1):  # For OSG nodes, file is in $WORKING_DIR, should not be moved to $RUNTIME_AREA
927	>	txt += '    mv '+fileWithSuffix+' '+output_file_num+'\n'
928	>	txt += '    ln -s `pwd`/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
929	>	else:
930	>	txt += '    mv '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
931	>	txt += '    ln -s $RUNTIME_AREA/'+output_file_num+' $RUNTIME_AREA/'+fileWithSuffix+'\n'
932		txt += 'else\n'
933	<	txt += '   cp '+fileWithSuffix+' $RUNTIME_AREA/'+output_file_num+'\n'
933	>	txt += '    job_exit_code=60302\n'
934	>	txt += '    echo "WARNING: Output file '+fileWithSuffix+' not found"\n'
935	>	if common.scheduler.name().upper() == 'CONDOR_G':
936	>	txt += '    if [ $middleware == OSG ]; then \n'
937	>	txt += '        echo "prepare dummy output file"\n'
938	>	txt += '        echo "Processing of job output failed" > $RUNTIME_AREA/'+output_file_num+'\n'
939	>	txt += '    fi \n'
940		txt += 'fi\n'
941	<	if i == check:
942	<	txt += 'cd $RUNTIME_AREA\n'
943	<	pass
944	<	pass
945	<
946	<	file_list=file_list[:-1]
947	<	txt += 'file_list="'+file_list+'"\n'
948	<	### OLI_DANIELE
949	<	txt += 'if [ $middleware == OSG ]; then\n'
950	<	txt += '    cd $RUNTIME_AREA\n'
742	<	txt += '    echo "Remove working directory: $WORKING_DIR"\n'
743	<	txt += '    /bin/rm -rf $WORKING_DIR\n'
744	<	txt += '    if [ -d $WORKING_DIR ] ;then\n'
745	<	txt += '        echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
746	<	txt += '    fi\n'
747	<	txt += 'fi\n'
941	>	file_list = []
942	>	for fileWithSuffix in (self.output_file):
943	>	file_list.append(numberFile('$SOFTWARE_DIR/'+fileWithSuffix, '$OutUniqueID'))
944	>
945	>	txt += 'file_list="'+string.join(file_list,',')+'"\n'
946	>	txt += '\n'
947	>	txt += 'echo ">>> current directory (SOFTWARE_DIR): $SOFTWARE_DIR" \n'
948	>	txt += 'echo ">>> current directory content:"\n'
949	>	if self.debug_wrapper==1:
950	>	txt += 'ls -Al\n'
951		txt += '\n'
952	+	txt += 'cd $RUNTIME_AREA\n'
953	+	txt += 'echo ">>> current directory (RUNTIME_AREA):  $RUNTIME_AREA"\n'
954		return txt
955
956	<	def numberFile_(self, file, txt):
956	>	def getRequirements(self, nj=[]):
957		"""
958	<	append _'txt' before last extension of a file
754	<	"""
755	<	p = string.split(file,".")
756	<	# take away last extension
757	<	name = p[0]
758	<	for x in p[1:-1]:
759	<	name=name+"."+x
760	<	# add "_txt"
761	<	if len(p)>1:
762	<	ext = p[len(p)-1]
763	<	#result = name + '_' + str(txt) + "." + ext
764	<	result = name + '_' + txt + "." + ext
765	<	else:
766	<	#result = name + '_' + str(txt)
767	<	result = name + '_' + txt
768	<
769	<	return result
770	<
771	<	def getRequirements(self):
772	<	"""
773	<	return job requirements to add to jdl files
958	>	return job requirements to add to jdl files
959		"""
960		req = ''
961	<	if common.analisys_common_info['sites']:
962	<	if common.analisys_common_info['sw_version']:
963	<	req='Member("VO-cms-' + \
964	<	common.analisys_common_info['sw_version'] + \
965	<	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
966	<	if len(common.analisys_common_info['sites'])>0:
967	<	req = req + ' && ('
968	<	for i in range(len(common.analisys_common_info['sites'])):
969	<	req = req + 'other.GlueCEInfoHostName == "' \
970	<	+ common.analisys_common_info['sites'][i] + '"'
971	<	if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
972	<	req = req + ' || '
973	<	req = req + ')'
974	<	#print "req = ", req
961	>	if self.version:
962	>	req='Member("VO-cms-' + \
963	>	self.version + \
964	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
965	>	if self.executable_arch:
966	>	req+=' && Member("VO-cms-' + \
967	>	self.executable_arch + \
968	>	'", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
969	>
970	>	req = req + ' && (other.GlueHostNetworkAdapterOutboundIP)'
971	>	if ( common.scheduler.name() in ["glite"] ):
972	>	## 25-Jun-2009 SL: patch to use Cream enabled WMS
973	>	if ( self.cfg_params.get('GRID.use_cream',None) ):
974	>	req += ' && (other.GlueCEStateStatus == "Production" || other.GlueCEStateStatus == "Special")'
975	>	else:
976	>	req += ' && other.GlueCEStateStatus == "Production" '
977	>
978		return req
979
980		def configFilename(self):
981		""" return the config filename """
982	<	return self.name()+'.cfg'
982	>	return self.name()+'.py'
983
796	–	### OLI_DANIELE
984		def wsSetupCMSOSGEnvironment_(self):
985		"""
986		Returns part of a job script which is prepares
987		the execution environment and which is common for all CMS jobs.
988		"""
989	<	txt = '\n'
990	<	txt += '   echo "### SETUP CMS OSG  ENVIRONMENT ###"\n'
991	<	txt += '   if [ -f $GRID3_APP_DIR/cmssoft/cmsset_default.sh ] ;then\n'
992	<	txt += '      # Use $GRID3_APP_DIR/cmssoft/cmsset_default.sh to setup cms software\n'
993	<	txt += '       source $GRID3_APP_DIR/cmssoft/cmsset_default.sh '+self.version+'\n'
994	<	txt += '   elif [ -f $OSG_APP/cmssoft/cmsset_default.sh ] ;then\n'
995	<	txt += '      # Use $OSG_APP/cmssoft/cmsset_default.sh to setup cms software\n'
996	<	txt += '       source $OSG_APP/cmssoft/cmsset_default.sh '+self.version+'\n'
997	<	txt += '   else\n'
998	<	txt += '       echo "SET_CMS_ENV 10020 ==> ERROR $GRID3_APP_DIR/cmssoft/cmsset_default.sh and $OSG_APP/cmssoft/cmsset_default.sh file not found"\n'
999	<	txt += '       echo "JOB_EXIT_STATUS = 10020"\n'
1000	<	txt += '       echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1001	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
815	<	txt += '       exit\n'
816	<	txt += '\n'
817	<	txt += '       echo "Remove working directory: $WORKING_DIR"\n'
818	<	txt += '       cd $RUNTIME_AREA\n'
819	<	txt += '       /bin/rm -rf $WORKING_DIR\n'
820	<	txt += '       if [ -d $WORKING_DIR ] ;then\n'
821	<	txt += '           echo "OSG WORKING DIR ==> $WORKING_DIR could not be deleted on on WN `hostname`"\n'
822	<	txt += '       fi\n'
823	<	txt += '\n'
824	<	txt += '       exit\n'
825	<	txt += '   fi\n'
989	>	txt = '\n#Written by cms_cmssw::wsSetupCMSOSGEnvironment_\n'
990	>	txt += '    echo ">>> setup CMS OSG environment:"\n'
991	>	txt += '    echo "set SCRAM ARCH to ' + self.executable_arch + '"\n'
992	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
993	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
994	>	txt += '    if [ -f $OSG_APP/cmssoft/cms/cmsset_default.sh ] ;then\n'
995	>	txt += '      # Use $OSG_APP/cmssoft/cms/cmsset_default.sh to setup cms software\n'
996	>	txt += '        source $OSG_APP/cmssoft/cms/cmsset_default.sh '+self.version+'\n'
997	>	txt += '    else\n'
998	>	txt += '        echo "ERROR ==> $OSG_APP/cmssoft/cms/cmsset_default.sh file not found"\n'
999	>	txt += '        job_exit_code=10020\n'
1000	>	txt += '        func_exit\n'
1001	>	txt += '    fi\n'
1002		txt += '\n'
1003	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1004	<	txt += '   echo " END SETUP CMS OSG  ENVIRONMENT "\n'
1003	>	txt += '    echo "==> setup cms environment ok"\n'
1004	>	txt += '    echo "SCRAM_ARCH = $SCRAM_ARCH"\n'
1005
1006		return txt
1007	<
832	<	### OLI_DANIELE
1007	>
1008		def wsSetupCMSLCGEnvironment_(self):
1009		"""
1010		Returns part of a job script which is prepares
1011		the execution environment and which is common for all CMS jobs.
1012		"""
1013	<	txt  = '   \n'
1014	<	txt += '   echo " ### SETUP CMS LCG  ENVIRONMENT ### "\n'
1015	<	txt += '      echo "JOB_EXIT_STATUS = 0"\n'
1016	<	txt += '   if [ ! $VO_CMS_SW_DIR ] ;then\n'
1017	<	txt += '       echo "SET_CMS_ENV 10031 ==> ERROR CMS software dir not found on WN `hostname`"\n'
1018	<	txt += '       echo "JOB_EXIT_STATUS = 10031" \n'
1019	<	txt += '       echo "JobExitCode=10031" | tee -a $RUNTIME_AREA/$repo\n'
1020	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1021	<	txt += '       exit\n'
1022	<	txt += '   else\n'
1023	<	txt += '       echo "Sourcing environment... "\n'
1024	<	txt += '       if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1025	<	txt += '           echo "SET_CMS_ENV 10020 ==> ERROR cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1026	<	txt += '           echo "JOB_EXIT_STATUS = 10020"\n'
1027	<	txt += '           echo "JobExitCode=10020" | tee -a $RUNTIME_AREA/$repo\n'
1028	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1029	<	txt += '           exit\n'
1030	<	txt += '       fi\n'
1031	<	txt += '       echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1032	<	txt += '       source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1033	<	txt += '       result=$?\n'
1034	<	txt += '       if [ $result -ne 0 ]; then\n'
1035	<	txt += '           echo "SET_CMS_ENV 10032 ==> ERROR problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1036	<	txt += '           echo "JOB_EXIT_STATUS = 10032"\n'
1037	<	txt += '           echo "JobExitCode=10032" | tee -a $RUNTIME_AREA/$repo\n'
1038	<	txt += '           dumpStatus $RUNTIME_AREA/$repo\n'
1039	<	txt += '           exit\n'
1040	<	txt += '       fi\n'
1041	<	txt += '   fi\n'
1042	<	txt += '   \n'
1043	<	txt += '   string=`cat /etc/redhat-release`\n'
1044	<	txt += '   echo $string\n'
1045	<	txt += '   if [[ $string = *alhalla* ]]; then\n'
1046	<	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1047	<	txt += '   elif [[ $string = *Enterprise* ]] || [[ $string = *cientific* ]]; then\n'
1048	<	txt += '       export SCRAM_ARCH=slc3_ia32_gcc323\n'
1049	<	txt += '       echo "SCRAM_ARCH= $SCRAM_ARCH"\n'
1050	<	txt += '   else\n'
1051	<	txt += '       echo "SET_CMS_ENV 1 ==> ERROR OS unknown, LCG environment not initialized"\n'
1052	<	txt += '       echo "JOB_EXIT_STATUS = 10033"\n'
1053	<	txt += '       echo "JobExitCode=10033" | tee -a $RUNTIME_AREA/$repo\n'
1054	<	txt += '       dumpStatus $RUNTIME_AREA/$repo\n'
1055	<	txt += '       exit\n'
1056	<	txt += '   fi\n'
1057	<	txt += '   echo "SET_CMS_ENV 0 ==> setup cms environment ok"\n'
1058	<	txt += '   echo "### END SETUP CMS LCG ENVIRONMENT ###"\n'
1013	>	txt = '\n#Written by cms_cmssw::wsSetupCMSLCGEnvironment_\n'
1014	>	txt += '    echo ">>> setup CMS LCG environment:"\n'
1015	>	txt += '    echo "set SCRAM ARCH and BUILD_ARCH to ' + self.executable_arch + ' ###"\n'
1016	>	txt += '    export SCRAM_ARCH='+self.executable_arch+'\n'
1017	>	txt += '    export BUILD_ARCH='+self.executable_arch+'\n'
1018	>	txt += '    if [ ! $VO_CMS_SW_DIR ] ;then\n'
1019	>	txt += '        echo "ERROR ==> CMS software dir not found on WN `hostname`"\n'
1020	>	txt += '        job_exit_code=10031\n'
1021	>	txt += '        func_exit\n'
1022	>	txt += '    else\n'
1023	>	txt += '        echo "Sourcing environment... "\n'
1024	>	txt += '        if [ ! -s $VO_CMS_SW_DIR/cmsset_default.sh ] ;then\n'
1025	>	txt += '            echo "ERROR ==> cmsset_default.sh file not found into dir $VO_CMS_SW_DIR"\n'
1026	>	txt += '            job_exit_code=10020\n'
1027	>	txt += '            func_exit\n'
1028	>	txt += '        fi\n'
1029	>	txt += '        echo "sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1030	>	txt += '        source $VO_CMS_SW_DIR/cmsset_default.sh\n'
1031	>	txt += '        result=$?\n'
1032	>	txt += '        if [ $result -ne 0 ]; then\n'
1033	>	txt += '            echo "ERROR ==> problem sourcing $VO_CMS_SW_DIR/cmsset_default.sh"\n'
1034	>	txt += '            job_exit_code=10032\n'
1035	>	txt += '            func_exit\n'
1036	>	txt += '        fi\n'
1037	>	txt += '    fi\n'
1038	>	txt += '    \n'
1039	>	txt += '    echo "==> setup cms environment ok"\n'
1040	>	return txt
1041	>
1042	>	def wsModifyReport(self, nj):
1043	>	"""
1044	>	insert the part of the script that modifies the FrameworkJob Report
1045	>	"""
1046	>
1047	>	txt = ''
1048	>	if (self.copy_data == 1):
1049	>	txt = '\n#Written by cms_cmssw::wsModifyReport\n'
1050	>
1051	>	txt += 'echo ">>> Modify Job Report:" \n'
1052	>	txt += 'chmod a+x $RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py\n'
1053	>	txt += 'echo "CMSSW_VERSION = $CMSSW_VERSION"\n\n'
1054	>
1055	>	args = 'fjr $RUNTIME_AREA/crab_fjr_$NJob.xml json $RUNTIME_AREA/resultCopyFile n_job $OutUniqueID PrimaryDataset $PrimaryDataset  ApplicationFamily $ApplicationFamily ApplicationName $executable cmssw_version $CMSSW_VERSION psethash $PSETHASH'
1056	>
1057	>	if (self.publish_data == 1):
1058	>	txt += 'ProcessedDataset='+self.processedDataset+'\n'
1059	>	txt += 'echo "ProcessedDataset = $ProcessedDataset"\n'
1060	>	args += ' UserProcessedDataset $USER-$ProcessedDataset-$PSETHASH'
1061	>
1062	>	txt += 'echo "$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'"\n'
1063	>	txt += '$RUNTIME_AREA/ProdCommon/FwkJobRep/ModifyJobReport.py '+str(args)+'\n'
1064	>	txt += 'modifyReport_result=$?\n'
1065	>	txt += 'if [ $modifyReport_result -ne 0 ]; then\n'
1066	>	txt += '    modifyReport_result=70500\n'
1067	>	txt += '    job_exit_code=$modifyReport_result\n'
1068	>	txt += '    echo "ModifyReportResult=$modifyReport_result" | tee -a $RUNTIME_AREA/$repo\n'
1069	>	txt += '    echo "WARNING: Problem with ModifyJobReport"\n'
1070	>	txt += 'else\n'
1071	>	txt += '    mv NewFrameworkJobReport.xml $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1072	>	txt += 'fi\n'
1073	>	return txt
1074	>
1075	>	def wsParseFJR(self):
1076	>	"""
1077	>	Parse the FrameworkJobReport to obtain useful infos
1078	>	"""
1079	>	txt = '\n#Written by cms_cmssw::wsParseFJR\n'
1080	>	txt += 'echo ">>> Parse FrameworkJobReport crab_fjr.xml"\n'
1081	>	txt += 'if [ -s $RUNTIME_AREA/crab_fjr_$NJob.xml ]; then\n'
1082	>	txt += '    if [ -s $RUNTIME_AREA/parseCrabFjr.py ]; then\n'
1083	>	txt += '        cmd_out=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --dashboard $MonitorID,$MonitorJobID '+self.debugWrap+'`\n'
1084	>	if self.debug_wrapper==1 :
1085	>	txt += '        echo "Result of parsing the FrameworkJobReport crab_fjr.xml: $cmd_out"\n'
1086	>	txt += '        executable_exit_status=`python $RUNTIME_AREA/parseCrabFjr.py --input $RUNTIME_AREA/crab_fjr_$NJob.xml --exitcode`\n'
1087	>	txt += '        if [ $executable_exit_status -eq 50115 ];then\n'
1088	>	txt += '            echo ">>> crab_fjr.xml contents: "\n'
1089	>	txt += '            cat $RUNTIME_AREA/crab_fjr_$NJob.xml\n'
1090	>	txt += '            echo "Wrong FrameworkJobReport --> does not contain useful info. ExitStatus: $executable_exit_status"\n'
1091	>	txt += '        elif [ $executable_exit_status -eq -999 ];then\n'
1092	>	txt += '            echo "ExitStatus from FrameworkJobReport not available. not available. Using exit code of executable from command line."\n'
1093	>	txt += '        else\n'
1094	>	txt += '            echo "Extracted ExitStatus from FrameworkJobReport parsing output: $executable_exit_status"\n'
1095	>	txt += '        fi\n'
1096	>	txt += '    else\n'
1097	>	txt += '        echo "CRAB python script to parse CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1098	>	txt += '    fi\n'
1099	>	#### Patch to check input data reading for CMSSW16x Hopefully we-ll remove it asap
1100	>	txt += '    if [ $executable_exit_status -eq 0 ];then\n'
1101	>	txt += '        echo ">>> Executable succeded  $executable_exit_status"\n'
1102	>	txt += '    fi\n'
1103	>	txt += 'else\n'
1104	>	txt += '    echo "CRAB FrameworkJobReport crab_fjr.xml is not available, using exit code of executable from command line."\n'
1105	>	txt += 'fi\n'
1106	>	txt += '\n'
1107	>	txt += 'if [ $executable_exit_status -ne 0 ];then\n'
1108	>	txt += '    echo ">>> Executable failed  $executable_exit_status"\n'
1109	>	txt += '    echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1110	>	txt += '    echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1111	>	txt += '    job_exit_code=$executable_exit_status\n'
1112	>	txt += '    func_exit\n'
1113	>	txt += 'fi\n\n'
1114	>	txt += 'echo "ExeExitCode=$executable_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
1115	>	txt += 'echo "EXECUTABLE_EXIT_STATUS = $executable_exit_status"\n'
1116	>	txt += 'job_exit_code=$executable_exit_status\n'
1117	>
1118		return txt
1119	+
1120	+	def setParam_(self, param, value):
1121	+	self._params[param] = value
1122	+
1123	+	def getParams(self):
1124	+	return self._params
1125	+
1126	+	def outList(self,list=False):
1127	+	"""
1128	+	check the dimension of the output files
1129	+	"""
1130	+	txt = ''
1131	+	txt += 'echo ">>> list of expected files on output sandbox"\n'
1132	+	listOutFiles = []
1133	+	stdout = 'CMSSW_$NJob.stdout'
1134	+	stderr = 'CMSSW_$NJob.stderr'
1135	+	if len(self.output_file) <= 0:
1136	+	msg ="WARNING: no output files name have been defined!!\n"
1137	+	msg+="\tno output files will be reported back/staged\n"
1138	+	common.logger.info(msg)
1139	+
1140	+	if (self.return_data == 1):
1141	+	for file in (self.output_file):
1142	+	listOutFiles.append(numberFile(file, '$OutUniqueID'))
1143	+	for file in (self.output_file_sandbox):
1144	+	listOutFiles.append(numberFile(file, '$NJob'))
1145	+	listOutFiles.append(stdout)
1146	+	listOutFiles.append(stderr)
1147	+
1148	+	txt += 'echo "output files: '+string.join(listOutFiles,' ')+'"\n'
1149	+	txt += 'filesToCheck="'+string.join(listOutFiles,' ')+'"\n'
1150	+	txt += 'export filesToCheck\n'
1151	+	taskinfo={}
1152	+	taskinfo['outfileBasename'] = self.output_file
1153	+	common._db.updateTask_(taskinfo)
1154	+
1155	+	if list : return self.output_file
1156	+	return txt
1157	+
1158	+	def checkCMSSWVersion(self, url = "https://cmstags.cern.ch/cgi-bin/CmsTC/", fileName = "ReleasesXML"):
1159	+	"""
1160	+	compare current CMSSW release and arch with allowed releases
1161	+	"""
1162	+
1163	+	downloader = Downloader(url)
1164	+	goodRelease = False
1165	+
1166	+	try:
1167	+	result = downloader.config(fileName)
1168	+	except:
1169	+	common.logger.info("ERROR: Problem reading file of allowed CMSSW releases.")
1170	+
1171	+	try:
1172	+	events = pulldom.parseString(result)
1173	+
1174	+	arch     = None
1175	+	release  = None
1176	+	relType  = None
1177	+	relState = None
1178	+	for (event, node) in events:
1179	+	if event == pulldom.START_ELEMENT:
1180	+	if node.tagName == 'architecture':
1181	+	arch = node.attributes.getNamedItem('name').nodeValue
1182	+	if node.tagName == 'project':
1183	+	relType = node.attributes.getNamedItem('type').nodeValue
1184	+	relState = node.attributes.getNamedItem('state').nodeValue
1185	+	if relType == 'Production' and relState == 'Announced':
1186	+	release = node.attributes.getNamedItem('label').nodeValue
1187	+	if self.executable_arch == arch and self.version == release:
1188	+	goodRelease = True
1189	+	return goodRelease
1190	+
1191	+	if not goodRelease:
1192	+	msg = "WARNING: %s on %s is not a supported release. " % \
1193	+	(self.version, self.executable_arch)
1194	+	msg += "Submission may fail."
1195	+	common.logger.info(msg)
1196	+	except:
1197	+	common.logger.info("Problems parsing file of allowed CMSSW releases.")
1198	+
1199	+	return goodRelease
1200	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines