ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.25 by spiga, Tue May 26 10:23:01 2009 UTC vs.
Revision 1.41 by fanzago, Fri Aug 5 15:39:08 2011 UTC

#	Line 5 | Line 5 | from crab_exceptions import *
5		from WorkSpace import *
6		from urlparse import urlparse
7		from LFNBaseName import *
8	+	from crab_util import getUserName
9
10		class PhEDExDatasvcInfo:
11	<	def __init__( self , cfg_params ):
11	>	def __init__( self , cfg_params=None, config=None ):
12
13		## PhEDEx Data Service URL
14	<	url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
14	<	self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
14	>	self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15
16		self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17	<	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17	>	self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
18		self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19
20	+	self.usePhedex = True
21	+	self.sched = common.scheduler.name().upper()
22	+
23	+	if config!=None:
24	+	self.checkConfig(config)
25	+	else:
26	+	self.checkCfgConfig(cfg_params)
27	+
28	+	self.protocol = self.srm_version
29	+
30	+
31	+	def checkConfig(self,config):
32	+	"""
33	+	"""
34	+	self.srm_version = config.get("srm_version",'srmv2')
35	+	self.node = config.get('storage_element',None)
36	+	self.lfn='/store/'
37	+
38	+	def checkCfgConfig(self,cfg_params):
39	+	"""
40	+	"""
41	+	self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
42		self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
43		self.node = cfg_params.get('USER.storage_element',None)
22	–
44
24	–
25	–	self.user_lfn = cfg_params.get("USER.lfn",'')
45		self.publish_data = cfg_params.get("USER.publish_data",0)
46		self.usenamespace = cfg_params.get("USER.usenamespace",0)
47		self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
48		if self.user_remote_dir:
49		if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
31	–	if self.user_lfn:
32	–	msg  = 'Warning: lfn has been deprecated, CRAB will ignore it.\n'
33	–	msg += '\t Please use only user_remote_dir removing lfn from your crab.cfg\n'
34	–	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
35	–	common.logger.info(msg)
50
51		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
52		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
#	Line 43 | Line 57 | class PhEDExDatasvcInfo:
57		if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
58
59		#check if using "private" Storage
46	–	self.usePhedex = True
60		if not self.node :
61		msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
62	<	msg +='\tFor further information please visit : %s'%stage_out_faq
62	>	msg +='\tFor further information please visit : %s'%self.stage_out_faq
63		raise CrabException(msg)
64		if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
65
66		if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
67	<	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
68	<	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
69	<	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
67	>	####### FEDE FOR BUG 73010 ############
68	>	msg = 'Error: you are asking to stage out without using CMS Storage Name convention. In this case you \n'
69	>	msg += '      must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n'
70	>	msg += '      For further information please visit : \n\t%s'%self.stage_out_faq
71	>	task = common._db.getTask()
72	>	add = '\n\n'
73	>	import shutil
74	>	try:
75	>	add += '      Task not correctly created: removing the working_dir ' +  common.work_space._top_dir + ' \n'
76	>	shutil.rmtree(common.work_space._top_dir)
77	>	except OSError:
78	>	add += '      Warning: problems removing the working_dir ' + common.work_space._top_dir + ' \n'
79	>	add += '      Please remove it by hand'
80	>	msg += add
81		raise CrabException(msg)
82	<	self.sched = common.scheduler.name().upper()
59	<	self.protocol = self.srm_version
82	>	#########################################
83
84		self.forced_path = '/store/user/'
85	<	if self.sched in ['CAF','LSF']:
86	<	self.protocol = 'direct'
87	<	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
85	>	if self.sched in ['CAF','LSF','PBS']:
86	>	self.srm_version = 'direct'
87	>	self.SE = {'CAF':'caf.cern.ch', 'LSF':'', 'PBS':''}
88		if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
89
90		if not self.usePhedex:
#	Line 76 | Line 99 | class PhEDExDatasvcInfo:
99
100		#extract the PFN for the given node,LFN,protocol
101		endpoint = self.getStageoutPFN()
102	+	if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
103
104		#extract SE name an SE_PATH (needed for publication)
105		SE, SE_PATH, User = self.splitEndpoint(endpoint)
#	Line 88 | Line 112 | class PhEDExDatasvcInfo:
112		'''
113		SE = ''
114		SE_PATH = ''
115	<	USER = ''
115	>	USER = getUserName()
116		if self.usePhedex:
117		if self.protocol == 'direct':
118		query=endpoint
#	Line 96 | Line 120 | class PhEDExDatasvcInfo:
120		SE = self.SE[self.sched]
121		else:
122		url = 'http://'+endpoint.split('://')[1]
99	–	# python > 2.4
100	–	# SE = urlparse(url).hostname
123		scheme, host, path, params, query, fragment = urlparse(url)
124	<	SE = host.split(':')[0]
124	>	SE = self.getAuthoritativeSE()
125		SE_PATH = endpoint.split(host)[1]
104	–	USER = (query.split('user')[1]).split('/')[1]
126		else:
106	–	#### to test #####
107	–	# url = 'http://'+endpoint.split('://')[1]
108	–	# scheme, host, path, params, query, fragment = urlparse(url)
109	–	# SE = host.split(':')[0]
110	–	# SE_PATH = endpoint.split(host)[1]
127		SE = self.node
128		SE_PATH = self.user_se_path + self.user_remote_dir
129	<	try:
130	<	USER = (self.lfn.split('user')[1]).split('/')[1]
131	<	except:
132	<	pass
133	<
129	>	if self.lfn.find('group') != -1:
130	>	try:
131	>	USER = (self.lfn.split('group')[1]).split('/')[1]
132	>	except:
133	>	pass
134		return SE, SE_PATH, USER
119	–
135
136		def getLFN(self):
137		"""
#	Line 137 | Line 152 | class PhEDExDatasvcInfo:
152		raise CrabException(msg)
153		if self.publish_data_name == '' and int(self.usenamespace) == 1:
154		self.publish_data_name = "DefaultDataset"
155	<	if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
155	>	if int(self.publish_data) == 1:
156	>	if self.sched in ['CAF']: l_User=True
157	>	primaryDataset = self.computePrimaryDataset()
158	>	### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
159	>	### for the publication in order to be able to check the lfn length
160	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'
161	>	elif int(self.usenamespace) == 1:
162		if self.sched in ['CAF']: l_User=True
163		primaryDataset = self.computePrimaryDataset()
164		lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'
165		else:
166		if self.sched in ['CAF','LSF']: l_User=True
167		lfn = LFNBase(self.forced_path,self.user_remote_dir)
168	+	if ( lfn[-1] != '/' ) : lfn = lfn + '/'
169		return lfn
170
171		def computePrimaryDataset(self):
#	Line 156 | Line 178 | class PhEDExDatasvcInfo:
178		primarydataset = self.publish_data_name
179		return primarydataset
180
181	<	def lfn2pfn(self):
181	>	def domPhedex(self,params,datasvc_baseUrl):
182		"""
183		PhEDEx Data Service lfn2pfn call
184
185	<	input:   LFN,node name,protocol
185	>	input:   params,datasvc_baseUrl
186		returns: DOM object with the content of the PhEDEx Data Service call
187		"""
166	–	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
188		params = urllib.urlencode(params)
168	–	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
189		try:
190	<	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
190	>	urlresults = urllib.urlopen(datasvc_baseUrl, params)
191		urlresults = parse(urlresults)
192		except IOError:
193	<	msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
193	>	msg="Unable to access PhEDEx Data Service at %s"%datasvc_baseUrl
194		raise CrabException(msg)
195		except:
196		urlresults = None
#	Line 214 | Line 234 | class PhEDExDatasvcInfo:
234		returns: PFN
235		"""
236		if self.usePhedex:
237	+	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
238	+	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
239		fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
240	<	domlfn2pfn = self.lfn2pfn()
240	>	domlfn2pfn = self.domPhedex(params,datasvc_lfn2pfn)
241		if not domlfn2pfn :
242		msg="Unable to get info from %s"%fullurl
243		raise CrabException(msg)
#	Line 234 | Line 256 | class PhEDExDatasvcInfo:
256		msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
257		raise CrabException(msg)
258		else:
259	<	if self.sched in ['CAF','LSF'] :
259	>	if self.sched in ['CAF','LSF','PBS'] :
260		stageoutpfn = self.user_se_path+self.lfn
261		else:
262		stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
263
264	+	if ( stageoutpfn[-1] != '/' ) : stageoutpfn = stageoutpfn + '/'
265		return stageoutpfn
266
267	+	def getAuthoritativeSE(self):
268	+	"""
269	+	input:   node name
270	+	returns: AuthoritativeSE
271	+	"""
272	+	params = {'node' : self.node }
273	+	datasvc_nodes="%s/nodes"%self.datasvc_url
274	+	fullurl="%s/nodes/?node=%s"%(self.datasvc_url,self.node)
275	+	domnodes = self.domPhedex(params,datasvc_nodes)
276	+
277	+	if not domnodes :
278	+	msg="Unable to get info from %s"%fullurl
279	+	raise CrabException(msg)
280	+
281	+	errormsg = self.parse_error(domnodes)
282	+	if errormsg:
283	+	msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
284	+	raise CrabException(msg)
285	+	result = domnodes.getElementsByTagName('phedex')
286	+	if not result:
287	+	return []
288	+	result = result[0]
289	+	se = None
290	+	node = result.getElementsByTagName('node')
291	+	for m in node:
292	+	se=m.getAttribute("se")
293	+	if se:
294	+	return se
295
296
297		if __name__ == '__main__':

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines