ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.3 by spiga, Sun Sep 21 10:53:10 2008 UTC vs.
Revision 1.26.2.5 by spiga, Thu Oct 8 21:22:21 2009 UTC

#	Line 2 | Line 2 | from Actor import *
2		import urllib
3		from xml.dom.minidom import parse
4		from crab_exceptions import *
5	–	from crab_logger import Logger
5		from WorkSpace import *
6		from urlparse import urlparse
7		from LFNBaseName import *
8
9		class PhEDExDatasvcInfo:
10	<	def __init__( self , cfg_params ):
10	>	def __init__( self , cfg_params=None, config=None ):
11
12		## PhEDEx Data Service URL
13	<	url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15	<	self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
13	>	self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
14
15	<	self.FacOps_savannah = 'https://savannah.cern.ch/projects/cmscompinfrasup/'
15	>	self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
16	>	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17	>	self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
18
19	+	self.usePhedex = True
20	+	self.sched = common.scheduler.name().upper()
21	+
22	+	if config!=None:
23	+	self.checkConfig(config)
24	+	else:
25	+	self.checkCfgConfig(cfg_params)
26	+
27	+	self.protocol = self.srm_version
28	+
29	+
30	+	def checkConfig(self,config):
31	+	"""
32	+	"""
33	+	self.srm_version = config.get("srm_version",'srmv2')
34	+	self.node = config.get('storage_element',None)
35	+	self.lfn='/store/'
36	+
37	+	def checkCfgConfig(self,cfg_params):
38	+	"""
39	+	"""
40	+	self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
41		self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
42		self.node = cfg_params.get('USER.storage_element',None)
43	<
43	>
44		self.publish_data = cfg_params.get("USER.publish_data",0)
45		self.usenamespace = cfg_params.get("USER.usenamespace",0)
46	<	self.user_remote_dir = cfg_params.get("USER.remote_dir",'')
46	>	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
47	>	if self.user_remote_dir:
48	>	if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
49	>
50		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
51		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
52
53	<	self.user_lfn = cfg_params.get("USER.lfn",'')
53	>	self.user_port = cfg_params.get("USER.storage_port",'8443')
54		self.user_se_path = cfg_params.get("USER.storage_path",'')
55	<
55	>	if self.user_se_path:
56	>	if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
57	>
58		#check if using "private" Storage
59	<	self.usePhedex = True
60	<	if (self.node.find('T1_') + self.node.find('T2_')) == -2: self.usePhedex = False
61	<	if not self.usePhedex and ( self.user_lfn == '' or self.user_se_path == '' ):
35	<	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
36	<	msg += '      must specify both lfn and storage_path in the crab.cfg section [USER].\n '
37	<	msg += '      For further information please visit: ADD_TWIKI_LINK'
59	>	if not self.node :
60	>	msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
61	>	msg +='\tFor further information please visit : %s'%stage_out_faq
62		raise CrabException(msg)
63	<	self.sched = common.scheduler.name().upper()
63	>	if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
64
65	<	self.protocol = self.srm_version
66	<	if self.sched in ['CAF','LSF']:self.protocol = 'direct'
65	>	if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
66	>	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
67	>	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
68	>	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
69	>	raise CrabException(msg)
70
71	+	self.forced_path = '/store/user/'
72	+	if self.sched in ['CAF','LSF']:
73	+	self.srm_version= 'direct'
74	+	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
75	+	if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
76	+
77	+	if not self.usePhedex:
78	+	self.forced_path = self.user_remote_dir
79		return
80
81		def getEndpoint(self):
#	Line 51 | Line 86 | class PhEDExDatasvcInfo:
86
87		#extract the PFN for the given node,LFN,protocol
88		endpoint = self.getStageoutPFN()
89	+	### FEDE added a check for the final /
90	+	if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91	+	######################################
92
93		#extract SE name an SE_PATH (needed for publication)
94		SE, SE_PATH, User = self.splitEndpoint(endpoint)
#	Line 68 | Line 106 | class PhEDExDatasvcInfo:
106		if self.protocol == 'direct':
107		query=endpoint
108		SE_PATH = endpoint
109	+	SE = self.SE[self.sched]
110		else:
111		url = 'http://'+endpoint.split('://')[1]
112		# python > 2.4
#	Line 77 | Line 116 | class PhEDExDatasvcInfo:
116		SE_PATH = endpoint.split(host)[1]
117		USER = (query.split('user')[1]).split('/')[1]
118		else:
119	+	#### to test #####
120	+	# url = 'http://'+endpoint.split('://')[1]
121	+	# scheme, host, path, params, query, fragment = urlparse(url)
122	+	# SE = host.split(':')[0]
123	+	# SE_PATH = endpoint.split(host)[1]
124		SE = self.node
125	<	SE_PATH = self.user_se_path + self.user_lfn
126	<	try:
127	<	USER = (self.lfn.split('user')[1]).split('/')[1]
128	<	except:
129	<	pass
130	<
125	>	SE_PATH = self.user_se_path + self.user_remote_dir
126	>	if self.lfn.find('user'):
127	>	try:
128	>	USER = (self.lfn.split('user')[1]).split('/')[1]
129	>	except:
130	>	pass
131	>	if self.lfn.find('group'):
132	>	try:
133	>	USER = (self.lfn.split('group')[1]).split('/')[1]
134	>	except:
135	>	pass
136		return SE, SE_PATH, USER
137
138
#	Line 98 | Line 147 | class PhEDExDatasvcInfo:
147		## check if storage_name is a T2 (siteDB query)
148		## if yes :match self.user_lfn with LFNBaseName...
149		##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
150	<	lfn = self.user_lfn
150	>	lfn = self.user_remote_dir
151		return lfn
152		if self.publish_data_name == '' and int(self.publish_data) == 1:
153	<	msg = "Eeror. The [USER] section does not have 'publish_data_name'"
153	>	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
154	>	msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
155		raise CrabException(msg)
156		if self.publish_data_name == '' and int(self.usenamespace) == 1:
157		self.publish_data_name = "DefaultDataset"
158	<	if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
158	>	if int(self.publish_data) == 1:
159	>	if self.sched in ['CAF']: l_User=True
160	>	primaryDataset = self.computePrimaryDataset()
161	>	### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
162	>	### for the publication in order to be able to check the lfn length
163	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'
164	>	elif int(self.usenamespace) == 1:
165		if self.sched in ['CAF']: l_User=True
166		primaryDataset = self.computePrimaryDataset()
167	<	lfn = LFNBase(primaryDataset,self.publish_data_name,LocalUser=l_User)  + '/${PSETHASH}/'
167	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'
168		else:
169	<	if self.sched in ['LSF']: l_User=True
170	<	lfn = LFNBase(self.user_remote_dir,LocalUser=l_User)
169	>	if self.sched in ['CAF','LSF']: l_User=True
170	>	lfn = LFNBase(self.forced_path,self.user_remote_dir)
171		return lfn
172
173		def computePrimaryDataset(self):
#	Line 134 | Line 190 | class PhEDExDatasvcInfo:
190		params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
191		params = urllib.urlencode(params)
192		datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
137	–	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
193		try:
194	+	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
195		urlresults = parse(urlresults)
196	+	except IOError:
197	+	msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
198	+	raise CrabException(msg)
199		except:
200		urlresults = None
201
#	Line 192 | Line 251 | class PhEDExDatasvcInfo:
251
252		stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
253		if not stageoutpfn:
254	<	msg ='Unable to get stageout path for Site %s. Maybe it does not correctly export its TFC. \n'%self.node
255	<	msg+='      Please alert the FacOps group through their savannah %s'%self.FacOps_savannah
254	>	msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
255	>	msg+='      Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
256	>	msg+='      reporting: \n'
257	>	msg+='       Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
258	>	msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
259		raise CrabException(msg)
260		else:
261	<	stageoutpfn = 'srm://'+self.node+':8443'+self.user_se_path+self.lfn
261	>	if self.sched in ['CAF','LSF'] :
262	>	stageoutpfn = self.user_se_path+self.lfn
263	>	else:
264	>	stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
265
266		return stageoutpfn
267	+
268	+
269	+
270	+	if __name__ == '__main__':
271	+	"""
272	+	Sort of unit testing to check Phedex API for whatever site and/or lfn.
273	+	Usage:
274	+	python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
275	+
276	+	"""
277	+	import getopt,sys
278	+	from crab_util import *
279	+	import common
280	+	klass_name = 'SchedulerGlite'
281	+	klass = importName(klass_name, klass_name)
282	+	common.scheduler = klass()
283	+
284	+	lfn="/store/user/"
285	+	node='T2_IT_Bari'
286	+	valid = ['node=','lfn=']
287	+	try:
288	+	opts, args = getopt.getopt(sys.argv[1:], "", valid)
289	+	except getopt.GetoptError, ex:
290	+	print str(ex)
291	+	sys.exit(1)
292	+	for o, a in opts:
293	+	if o == "--node":
294	+	node = a
295	+	if o == "--lfn":
296	+	lfn = a
297	+
298	+	mycfg_params = { 'USER.storage_element': node }
299	+	dsvc = PhEDExDatasvcInfo(mycfg_params)
300	+	dsvc.lfn = lfn
301	+	print dsvc.getStageoutPFN()
302	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines