ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.29 by spiga, Sat Sep 19 11:11:01 2009 UTC vs.
Revision 1.40 by fanzago, Wed Jul 13 17:08:40 2011 UTC

#	Line 5 | Line 5 | from crab_exceptions import *
5		from WorkSpace import *
6		from urlparse import urlparse
7		from LFNBaseName import *
8	+	from crab_util import getUserName
9
10		class PhEDExDatasvcInfo:
11		def __init__( self , cfg_params=None, config=None ):
#	Line 13 | Line 14 | class PhEDExDatasvcInfo:
14		self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15
16		self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17	<	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17	>	self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
18		self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19
20		self.usePhedex = True
#	Line 58 | Line 59 | class PhEDExDatasvcInfo:
59		#check if using "private" Storage
60		if not self.node :
61		msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
62	<	msg +='\tFor further information please visit : %s'%stage_out_faq
62	>	msg +='\tFor further information please visit : %s'%self.stage_out_faq
63		raise CrabException(msg)
64		if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
65
66		if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
67	<	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
67	>	##### remove
68	>	####### FEDE FOR BUG 73010 ############
69	>	print "--------->>>> self = ", self
70	>	task = common._db.getTask()
71	>	print "task = ", task
72	>	print "common.work_space = ", common.work_space._top_dir
73	>	print "removing common.work_space"
74	>	add = '\n'
75	>	import shutil
76	>	try:
77	>	shutil.rmtree(common.work_space._top_dir)
78	>	except OSError:
79	>	add += '\t Warning: problems removing the dir ' + common.work_space._top_dir + ' \n'
80	>	add += '\t Please remove it by hand'
81	>	#common.work_space.delete()
82	>
83	>	msg = 'Error: you are asking to stage out without using CMS Storage Name convention. In this case you \n'
84		msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
85	<	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
85	>	msg += '\t Otherwise this task can not be correctly created. \n'
86	>	msg += '\t For further information please visit : \n\t%s'%self.stage_out_faq
87	>	msg += add
88		raise CrabException(msg)
89	+	#########################################
90
91		self.forced_path = '/store/user/'
92	<	if self.sched in ['CAF','LSF']:
93	<	self.protocol = 'direct'
94	<	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
92	>	if self.sched in ['CAF','LSF','PBS']:
93	>	self.srm_version = 'direct'
94	>	self.SE = {'CAF':'caf.cern.ch', 'LSF':'', 'PBS':''}
95		if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
96
97		if not self.usePhedex:
#	Line 86 | Line 106 | class PhEDExDatasvcInfo:
106
107		#extract the PFN for the given node,LFN,protocol
108		endpoint = self.getStageoutPFN()
89	–	### FEDE added a check for the final /
109		if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91	–	######################################
110
111		#extract SE name an SE_PATH (needed for publication)
112		SE, SE_PATH, User = self.splitEndpoint(endpoint)
#	Line 101 | Line 119 | class PhEDExDatasvcInfo:
119		'''
120		SE = ''
121		SE_PATH = ''
122	<	USER = ''
122	>	USER = getUserName()
123		if self.usePhedex:
124		if self.protocol == 'direct':
125		query=endpoint
#	Line 109 | Line 127 | class PhEDExDatasvcInfo:
127		SE = self.SE[self.sched]
128		else:
129		url = 'http://'+endpoint.split('://')[1]
112	–	# python > 2.4
113	–	# SE = urlparse(url).hostname
130		scheme, host, path, params, query, fragment = urlparse(url)
131	<	SE = host.split(':')[0]
131	>	SE = self.getAuthoritativeSE()
132		SE_PATH = endpoint.split(host)[1]
117	–	USER = (query.split('user')[1]).split('/')[1]
133		else:
134	<	#### to test #####
134	>	#### to test #####
135		# url = 'http://'+endpoint.split('://')[1]
136		# scheme, host, path, params, query, fragment = urlparse(url)
137		# SE = host.split(':')[0]
138		# SE_PATH = endpoint.split(host)[1]
139		SE = self.node
140		SE_PATH = self.user_se_path + self.user_remote_dir
141	<	try:
142	<	USER = (self.lfn.split('user')[1]).split('/')[1]
143	<	except:
144	<	pass
145	<
141	>	if self.lfn.find('group') != -1:
142	>	try:
143	>	USER = (self.lfn.split('group')[1]).split('/')[1]
144	>	except:
145	>	pass
146		return SE, SE_PATH, USER
132	–
147
148		def getLFN(self):
149		"""
#	Line 163 | Line 177 | class PhEDExDatasvcInfo:
177		else:
178		if self.sched in ['CAF','LSF']: l_User=True
179		lfn = LFNBase(self.forced_path,self.user_remote_dir)
180	+	if ( lfn[-1] != '/' ) : lfn = lfn + '/'
181		return lfn
182
183		def computePrimaryDataset(self):
#	Line 175 | Line 190 | class PhEDExDatasvcInfo:
190		primarydataset = self.publish_data_name
191		return primarydataset
192
193	<	def lfn2pfn(self):
193	>	def domPhedex(self,params,datasvc_baseUrl):
194		"""
195		PhEDEx Data Service lfn2pfn call
196
197	<	input:   LFN,node name,protocol
197	>	input:   params,datasvc_baseUrl
198		returns: DOM object with the content of the PhEDEx Data Service call
199		"""
185	–	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
200		params = urllib.urlencode(params)
187	–	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
201		try:
202	<	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
202	>	urlresults = urllib.urlopen(datasvc_baseUrl, params)
203		urlresults = parse(urlresults)
204		except IOError:
205	<	msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
205	>	msg="Unable to access PhEDEx Data Service at %s"%datasvc_baseUrl
206		raise CrabException(msg)
207		except:
208		urlresults = None
#	Line 233 | Line 246 | class PhEDExDatasvcInfo:
246		returns: PFN
247		"""
248		if self.usePhedex:
249	+	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
250	+	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
251		fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
252	<	domlfn2pfn = self.lfn2pfn()
252	>	domlfn2pfn = self.domPhedex(params,datasvc_lfn2pfn)
253		if not domlfn2pfn :
254		msg="Unable to get info from %s"%fullurl
255		raise CrabException(msg)
#	Line 253 | Line 268 | class PhEDExDatasvcInfo:
268		msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
269		raise CrabException(msg)
270		else:
271	<	if self.sched in ['CAF','LSF'] :
271	>	if self.sched in ['CAF','LSF','PBS'] :
272		stageoutpfn = self.user_se_path+self.lfn
273		else:
274		stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
275
276	+	if ( stageoutpfn[-1] != '/' ) : stageoutpfn = stageoutpfn + '/'
277		return stageoutpfn
278
279	+	def getAuthoritativeSE(self):
280	+	"""
281	+	input:   node name
282	+	returns: AuthoritativeSE
283	+	"""
284	+	params = {'node' : self.node }
285	+	datasvc_nodes="%s/nodes"%self.datasvc_url
286	+	fullurl="%s/nodes/?node=%s"%(self.datasvc_url,self.node)
287	+	domnodes = self.domPhedex(params,datasvc_nodes)
288	+
289	+	if not domnodes :
290	+	msg="Unable to get info from %s"%fullurl
291	+	raise CrabException(msg)
292	+
293	+	errormsg = self.parse_error(domnodes)
294	+	if errormsg:
295	+	msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
296	+	raise CrabException(msg)
297	+	result = domnodes.getElementsByTagName('phedex')
298	+	if not result:
299	+	return []
300	+	result = result[0]
301	+	se = None
302	+	node = result.getElementsByTagName('node')
303	+	for m in node:
304	+	se=m.getAttribute("se")
305	+	if se:
306	+	return se
307
308
309		if __name__ == '__main__':

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines