ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.4 by spiga, Sun Sep 21 12:41:57 2008 UTC vs.
Revision 1.19 by fanzago, Fri Jan 30 13:19:25 2009 UTC

#	Line 14 | Line 14 | class PhEDExDatasvcInfo:
14		url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15		self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
16
17	<	self.FacOps_savannah = 'https://savannah.cern.ch/projects/cmscompinfrasup/'
17	>	self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
18	>	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabFaq#How_to_store_output_with_CRAB_2'
19	>	self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
20
21		self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
22		self.node = cfg_params.get('USER.storage_element',None)
23
24	+
25	+
26	+	self.user_lfn = cfg_params.get("USER.lfn",'')
27		self.publish_data = cfg_params.get("USER.publish_data",0)
28		self.usenamespace = cfg_params.get("USER.usenamespace",0)
29	<	self.user_remote_dir = cfg_params.get("USER.remote_dir",'')
29	>	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
30	>	if self.user_remote_dir:
31	>	if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
32	>	if self.user_lfn:
33	>	msg  = 'Warning: lfn has been deprecated, CRAB will ignore it.\n'
34	>	msg += '\t Please use only user_remote_dir removing lfn from your crab.cfg\n'
35	>	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
36	>	common.logger.message(msg)
37	>
38		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
39		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
40
41	<	self.user_lfn = cfg_params.get("USER.lfn",'')
41	>	self.user_port = cfg_params.get("USER.storage_port",'8443')
42		self.user_se_path = cfg_params.get("USER.storage_path",'')
43	<
43	>	if self.user_se_path:
44	>	if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
45	>
46		#check if using "private" Storage
47		self.usePhedex = True
48	+	if not self.node :
49	+	msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
50	+	msg +='\tFor further information please visit : %s'%stage_out_faq
51	+	raise CrabException(msg)
52		if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
53	<	if not self.usePhedex and ( self.user_lfn == '' or self.user_se_path == '' ):
53	>
54	>	if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
55		msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
56	<	msg += '      must specify both lfn and storage_path in the crab.cfg section [USER].\n '
57	<	msg += '      For further information please visit: ADD_TWIKI_LINK'
56	>	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
57	>	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
58		raise CrabException(msg)
59		self.sched = common.scheduler.name().upper()
40	–
60		self.protocol = self.srm_version
61	<	if self.sched in ['CAF','LSF']:self.protocol = 'direct'
61	>	if self.sched in ['CAF','LSF']:
62	>	self.protocol = 'direct'
63	>	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
64
65	+	self.forced_path = '/store/user/'
66	+	if not self.usePhedex:
67	+	self.forced_path = self.user_remote_dir
68		return
69
70		def getEndpoint(self):
#	Line 68 | Line 92 | class PhEDExDatasvcInfo:
92		if self.protocol == 'direct':
93		query=endpoint
94		SE_PATH = endpoint
95	+	SE = self.SE[self.sched]
96		else:
97		url = 'http://'+endpoint.split('://')[1]
98		# python > 2.4
#	Line 77 | Line 102 | class PhEDExDatasvcInfo:
102		SE_PATH = endpoint.split(host)[1]
103		USER = (query.split('user')[1]).split('/')[1]
104		else:
105	<	SE = self.node
106	<	SE_PATH = self.user_se_path + self.user_lfn
105	>	#### to test #####
106	>	url = 'http://'+endpoint.split('://')[1]
107	>	scheme, host, path, params, query, fragment = urlparse(url)
108	>	SE = host.split(':')[0]
109	>	SE_PATH = endpoint.split(host)[1]
110	>	#SE = self.node
111	>	#SE_PATH = self.user_se_path + self.user_remote_dir
112		try:
113		USER = (self.lfn.split('user')[1]).split('/')[1]
114		except:
#	Line 98 | Line 128 | class PhEDExDatasvcInfo:
128		## check if storage_name is a T2 (siteDB query)
129		## if yes :match self.user_lfn with LFNBaseName...
130		##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
131	<	lfn = self.user_lfn
131	>	lfn = self.user_remote_dir
132		return lfn
133		if self.publish_data_name == '' and int(self.publish_data) == 1:
134	<	msg = "Eeror. The [USER] section does not have 'publish_data_name'"
134	>	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
135	>	msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
136		raise CrabException(msg)
137		if self.publish_data_name == '' and int(self.usenamespace) == 1:
138		self.publish_data_name = "DefaultDataset"
139		if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
140		if self.sched in ['CAF']: l_User=True
141		primaryDataset = self.computePrimaryDataset()
142	<	lfn = LFNBase(primaryDataset,self.publish_data_name,LocalUser=l_User)  + '/${PSETHASH}/'
142	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, LocalUser=l_User)  + '/${PSETHASH}/'
143		else:
144	<	if self.sched in ['LSF']: l_User=True
145	<	lfn = LFNBase(self.user_remote_dir,LocalUser=l_User)
144	>	if self.sched in ['CAF','LSF']: l_User=True
145	>	lfn = LFNBase(self.forced_path,self.user_remote_dir,LocalUser=l_User)
146		return lfn
147
148		def computePrimaryDataset(self):
#	Line 192 | Line 223 | class PhEDExDatasvcInfo:
223
224		stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
225		if not stageoutpfn:
226	<	msg ='Unable to get stageout path for Site %s. Maybe it does not correctly export its TFC. \n'%self.node
227	<	msg+='      Please alert the FacOps group through their savannah %s'%self.FacOps_savannah
226	>	msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
227	>	msg+='      Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
228	>	msg+='      reporting: \n'
229	>	msg+='       Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
230	>	msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
231		raise CrabException(msg)
232		else:
233	<	stageoutpfn = 'srm://'+self.node+':8443'+self.user_se_path+self.lfn
233	>	if self.sched in ['CAF','LSF'] :
234	>	stageoutpfn = self.user_se_path+self.lfn
235	>	else:
236	>	stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
237
238		return stageoutpfn
239	+
240	+
241	+
242	+	if __name__ == '__main__':
243	+	"""
244	+	Sort of unit testing to check Phedex API for whatever site and/or lfn.
245	+	Usage:
246	+	python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
247	+
248	+	"""
249	+	import getopt,sys
250	+	from crab_util import *
251	+	import common
252	+	klass_name = 'SchedulerGlite'
253	+	klass = importName(klass_name, klass_name)
254	+	common.scheduler = klass()
255	+
256	+	lfn="/store/user/"
257	+	node='T2_IT_Bari'
258	+	valid = ['node=','lfn=']
259	+	try:
260	+	opts, args = getopt.getopt(sys.argv[1:], "", valid)
261	+	except getopt.GetoptError, ex:
262	+	print str(ex)
263	+	sys.exit(1)
264	+	for o, a in opts:
265	+	if o == "--node":
266	+	node = a
267	+	if o == "--lfn":
268	+	lfn = a
269	+
270	+	mycfg_params = { 'USER.storage_element': node }
271	+	dsvc = PhEDExDatasvcInfo(mycfg_params)
272	+	dsvc.lfn = lfn
273	+	print dsvc.getStageoutPFN()
274	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines