ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.3 by spiga, Sun Sep 21 10:53:10 2008 UTC vs.
Revision 1.36 by fanzago, Thu Mar 18 17:22:45 2010 UTC

#	Line 2 | Line 2 | from Actor import *
2		import urllib
3		from xml.dom.minidom import parse
4		from crab_exceptions import *
5	–	from crab_logger import Logger
5		from WorkSpace import *
6		from urlparse import urlparse
7		from LFNBaseName import *
8	+	from crab_util import getUserName
9
10		class PhEDExDatasvcInfo:
11	<	def __init__( self , cfg_params ):
11	>	def __init__( self , cfg_params=None, config=None ):
12
13		## PhEDEx Data Service URL
14	<	url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15	<	self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
14	>	self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15
16	<	self.FacOps_savannah = 'https://savannah.cern.ch/projects/cmscompinfrasup/'
16	>	self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17	>	self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
18	>	self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19
20	+	self.usePhedex = True
21	+	self.sched = common.scheduler.name().upper()
22	+
23	+	if config!=None:
24	+	self.checkConfig(config)
25	+	else:
26	+	self.checkCfgConfig(cfg_params)
27	+
28	+	self.protocol = self.srm_version
29	+
30	+
31	+	def checkConfig(self,config):
32	+	"""
33	+	"""
34	+	self.srm_version = config.get("srm_version",'srmv2')
35	+	self.node = config.get('storage_element',None)
36	+	self.lfn='/store/'
37	+
38	+	def checkCfgConfig(self,cfg_params):
39	+	"""
40	+	"""
41	+	self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
42		self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
43		self.node = cfg_params.get('USER.storage_element',None)
44	<
44	>
45		self.publish_data = cfg_params.get("USER.publish_data",0)
46		self.usenamespace = cfg_params.get("USER.usenamespace",0)
47	<	self.user_remote_dir = cfg_params.get("USER.remote_dir",'')
47	>	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
48	>	if self.user_remote_dir:
49	>	if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
50	>
51		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
52		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
53
54	<	self.user_lfn = cfg_params.get("USER.lfn",'')
54	>	self.user_port = cfg_params.get("USER.storage_port",'8443')
55		self.user_se_path = cfg_params.get("USER.storage_path",'')
56	<
56	>	if self.user_se_path:
57	>	if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
58	>
59		#check if using "private" Storage
60	<	self.usePhedex = True
61	<	if (self.node.find('T1_') + self.node.find('T2_')) == -2: self.usePhedex = False
62	<	if not self.usePhedex and ( self.user_lfn == '' or self.user_se_path == '' ):
35	<	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
36	<	msg += '      must specify both lfn and storage_path in the crab.cfg section [USER].\n '
37	<	msg += '      For further information please visit: ADD_TWIKI_LINK'
60	>	if not self.node :
61	>	msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
62	>	msg +='\tFor further information please visit : %s'%self.stage_out_faq
63		raise CrabException(msg)
64	<	self.sched = common.scheduler.name().upper()
64	>	if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
65
66	<	self.protocol = self.srm_version
67	<	if self.sched in ['CAF','LSF']:self.protocol = 'direct'
66	>	if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
67	>	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
68	>	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
69	>	msg += '\t For further information please visit : \n\t%s'%self.stage_out_faq
70	>	raise CrabException(msg)
71
72	+	self.forced_path = '/store/user/'
73	+	if self.sched in ['CAF','LSF','PBS']:
74	+	self.srm_version = 'direct'
75	+	self.SE = {'CAF':'caf.cern.ch', 'LSF':'', 'PBS':''}
76	+	if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
77	+
78	+	if not self.usePhedex:
79	+	self.forced_path = self.user_remote_dir
80		return
81
82		def getEndpoint(self):
#	Line 51 | Line 87 | class PhEDExDatasvcInfo:
87
88		#extract the PFN for the given node,LFN,protocol
89		endpoint = self.getStageoutPFN()
90	+	if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91
92		#extract SE name an SE_PATH (needed for publication)
93		SE, SE_PATH, User = self.splitEndpoint(endpoint)
#	Line 68 | Line 105 | class PhEDExDatasvcInfo:
105		if self.protocol == 'direct':
106		query=endpoint
107		SE_PATH = endpoint
108	+	SE = self.SE[self.sched]
109		else:
110		url = 'http://'+endpoint.split('://')[1]
73	–	# python > 2.4
74	–	# SE = urlparse(url).hostname
111		scheme, host, path, params, query, fragment = urlparse(url)
112	<	SE = host.split(':')[0]
112	>	SE = self.getAuthoritativeSE()
113		SE_PATH = endpoint.split(host)[1]
114	<	USER = (query.split('user')[1]).split('/')[1]
114	>	### fede ###
115	>	#USER = (query.split('user')[1]).split('/')[1]
116		else:
117	+	#### to test #####
118	+	# url = 'http://'+endpoint.split('://')[1]
119	+	# scheme, host, path, params, query, fragment = urlparse(url)
120	+	# SE = host.split(':')[0]
121	+	# SE_PATH = endpoint.split(host)[1]
122		SE = self.node
123	<	SE_PATH = self.user_se_path + self.user_lfn
123	>	SE_PATH = self.user_se_path + self.user_remote_dir
124	>	### fede ###
125	>	#if self.lfn.find('user'):
126	>	#    try:
127	>	#        USER = (self.lfn.split('user')[1]).split('/')[1]
128	>	#    except:
129	>	#        pass
130	>	### fede ###
131	>	if self.lfn.find('group') != -1:
132		try:
133	<	USER = (self.lfn.split('user')[1]).split('/')[1]
133	>	USER = (self.lfn.split('group')[1]).split('/')[1]
134		except:
135		pass
136	<
136	>	else:
137	>	USER = getUserName()
138		return SE, SE_PATH, USER
139
140
#	Line 98 | Line 149 | class PhEDExDatasvcInfo:
149		## check if storage_name is a T2 (siteDB query)
150		## if yes :match self.user_lfn with LFNBaseName...
151		##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
152	<	lfn = self.user_lfn
152	>	lfn = self.user_remote_dir
153		return lfn
154		if self.publish_data_name == '' and int(self.publish_data) == 1:
155	<	msg = "Eeror. The [USER] section does not have 'publish_data_name'"
155	>	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
156	>	msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
157		raise CrabException(msg)
158		if self.publish_data_name == '' and int(self.usenamespace) == 1:
159		self.publish_data_name = "DefaultDataset"
160	<	if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
160	>	if int(self.publish_data) == 1:
161	>	if self.sched in ['CAF']: l_User=True
162	>	primaryDataset = self.computePrimaryDataset()
163	>	### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
164	>	### for the publication in order to be able to check the lfn length
165	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'
166	>	elif int(self.usenamespace) == 1:
167		if self.sched in ['CAF']: l_User=True
168		primaryDataset = self.computePrimaryDataset()
169	<	lfn = LFNBase(primaryDataset,self.publish_data_name,LocalUser=l_User)  + '/${PSETHASH}/'
169	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'
170		else:
171	<	if self.sched in ['LSF']: l_User=True
172	<	lfn = LFNBase(self.user_remote_dir,LocalUser=l_User)
171	>	if self.sched in ['CAF','LSF']: l_User=True
172	>	lfn = LFNBase(self.forced_path,self.user_remote_dir)
173	>	if ( lfn[-1] != '/' ) : lfn = lfn + '/'
174		return lfn
175
176		def computePrimaryDataset(self):
#	Line 124 | Line 183 | class PhEDExDatasvcInfo:
183		primarydataset = self.publish_data_name
184		return primarydataset
185
186	<	def lfn2pfn(self):
186	>	def domPhedex(self,params,datasvc_baseUrl):
187		"""
188		PhEDEx Data Service lfn2pfn call
189
190	<	input:   LFN,node name,protocol
190	>	input:   params,datasvc_baseUrl
191		returns: DOM object with the content of the PhEDEx Data Service call
192		"""
134	–	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
193		params = urllib.urlencode(params)
136	–	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
137	–	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
194		try:
195	+	urlresults = urllib.urlopen(datasvc_baseUrl, params)
196		urlresults = parse(urlresults)
197	+	except IOError:
198	+	msg="Unable to access PhEDEx Data Service at %s"%datasvc_baseUrl
199	+	raise CrabException(msg)
200		except:
201		urlresults = None
202
#	Line 179 | Line 239 | class PhEDExDatasvcInfo:
239		returns: PFN
240		"""
241		if self.usePhedex:
242	+	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
243	+	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
244		fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
245	<	domlfn2pfn = self.lfn2pfn()
245	>	domlfn2pfn = self.domPhedex(params,datasvc_lfn2pfn)
246		if not domlfn2pfn :
247		msg="Unable to get info from %s"%fullurl
248		raise CrabException(msg)
#	Line 192 | Line 254 | class PhEDExDatasvcInfo:
254
255		stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
256		if not stageoutpfn:
257	<	msg ='Unable to get stageout path for Site %s. Maybe it does not correctly export its TFC. \n'%self.node
258	<	msg+='      Please alert the FacOps group through their savannah %s'%self.FacOps_savannah
257	>	msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
258	>	msg+='      Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
259	>	msg+='      reporting: \n'
260	>	msg+='       Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
261	>	msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
262		raise CrabException(msg)
263		else:
264	<	stageoutpfn = 'srm://'+self.node+':8443'+self.user_se_path+self.lfn
264	>	if self.sched in ['CAF','LSF','PBS'] :
265	>	stageoutpfn = self.user_se_path+self.lfn
266	>	else:
267	>	stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
268
269	+	if ( stageoutpfn[-1] != '/' ) : stageoutpfn = stageoutpfn + '/'
270		return stageoutpfn
271	+
272	+	def getAuthoritativeSE(self):
273	+	"""
274	+	input:   node name
275	+	returns: AuthoritativeSE
276	+	"""
277	+	params = {'node' : self.node }
278	+	datasvc_nodes="%s/nodes"%self.datasvc_url
279	+	fullurl="%s/nodes/?node=%s"%(self.datasvc_url,self.node)
280	+	domnodes = self.domPhedex(params,datasvc_nodes)
281	+
282	+	if not domnodes :
283	+	msg="Unable to get info from %s"%fullurl
284	+	raise CrabException(msg)
285	+
286	+	errormsg = self.parse_error(domnodes)
287	+	if errormsg:
288	+	msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
289	+	raise CrabException(msg)
290	+	result = domnodes.getElementsByTagName('phedex')
291	+	if not result:
292	+	return []
293	+	result = result[0]
294	+	se = None
295	+	node = result.getElementsByTagName('node')
296	+	for m in node:
297	+	se=m.getAttribute("se")
298	+	if se:
299	+	return se
300	+
301	+
302	+	if __name__ == '__main__':
303	+	"""
304	+	Sort of unit testing to check Phedex API for whatever site and/or lfn.
305	+	Usage:
306	+	python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
307	+
308	+	"""
309	+	import getopt,sys
310	+	from crab_util import *
311	+	import common
312	+	klass_name = 'SchedulerGlite'
313	+	klass = importName(klass_name, klass_name)
314	+	common.scheduler = klass()
315	+
316	+	lfn="/store/user/"
317	+	node='T2_IT_Bari'
318	+	valid = ['node=','lfn=']
319	+	try:
320	+	opts, args = getopt.getopt(sys.argv[1:], "", valid)
321	+	except getopt.GetoptError, ex:
322	+	print str(ex)
323	+	sys.exit(1)
324	+	for o, a in opts:
325	+	if o == "--node":
326	+	node = a
327	+	if o == "--lfn":
328	+	lfn = a
329	+
330	+	mycfg_params = { 'USER.storage_element': node }
331	+	dsvc = PhEDExDatasvcInfo(mycfg_params)
332	+	dsvc.lfn = lfn
333	+	print dsvc.getStageoutPFN()
334	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines