ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.13 by fanzago, Thu Oct 23 17:07:22 2008 UTC vs.
Revision 1.26.2.1 by fanzago, Wed Sep 16 16:28:24 2009 UTC

#	Line 2 | Line 2 | from Actor import *
2		import urllib
3		from xml.dom.minidom import parse
4		from crab_exceptions import *
5	–	from crab_logger import Logger
5		from WorkSpace import *
6		from urlparse import urlparse
7		from LFNBaseName import *
#	Line 15 | Line 14 | class PhEDExDatasvcInfo:
14		self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
15
16		self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17	<
17	>	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
18	>	self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19
20		self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
21		self.node = cfg_params.get('USER.storage_element',None)
22
23	+
24	+
25	+	self.user_lfn = cfg_params.get("USER.lfn",'')
26		self.publish_data = cfg_params.get("USER.publish_data",0)
27		self.usenamespace = cfg_params.get("USER.usenamespace",0)
28		self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
29		if self.user_remote_dir:
30		if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
31	<
31	>	if self.user_lfn:
32	>	msg  = 'Warning: lfn has been deprecated, CRAB will ignore it.\n'
33	>	msg += '\t Please use only user_remote_dir removing lfn from your crab.cfg\n'
34	>	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
35	>	common.logger.info(msg)
36	>
37		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
38		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
39
32	–	self.user_lfn = cfg_params.get("USER.lfn",'')
33	–	if self.user_lfn:
34	–	if ( self.user_lfn[-1] != '/' ) : self.user_lfn = self.user_lfn + '/'
35	–
40		self.user_port = cfg_params.get("USER.storage_port",'8443')
41		self.user_se_path = cfg_params.get("USER.storage_path",'')
42		if self.user_se_path:
43		if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
44
41	–
45		#check if using "private" Storage
46		self.usePhedex = True
44	–	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabFaq#How_to_store_output_with_CRAB_2'
47		if not self.node :
48		msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
49	<	msg +='      For further information please visit : %s'%stage_out_faq
49	>	msg +='\tFor further information please visit : %s'%stage_out_faq
50		raise CrabException(msg)
51		if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
52	<	if not self.usePhedex and ( self.user_lfn == '' or self.user_se_path == '' ):
52	>
53	>	if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
54		msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
55	<	msg += '      must specify both lfn and storage_path in the crab.cfg section [USER].\n '
56	<	msg += '      For further information please visit : %s'%stage_out_faq
55	>	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
56	>	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
57		raise CrabException(msg)
58		self.sched = common.scheduler.name().upper()
59		self.protocol = self.srm_version
57	–	if self.sched in ['CAF','LSF']:self.protocol = 'direct'
60
61	+	self.forced_path = '/store/user/'
62	+	if self.sched in ['CAF','LSF']:
63	+	self.protocol = 'direct'
64	+	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
65	+	if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
66	+
67	+	if not self.usePhedex:
68	+	self.forced_path = self.user_remote_dir
69		return
70
71		def getEndpoint(self):
#	Line 66 | Line 76 | class PhEDExDatasvcInfo:
76
77		#extract the PFN for the given node,LFN,protocol
78		endpoint = self.getStageoutPFN()
79	+	### FEDE added a check for the final /
80	+	if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
81	+	######################################
82
83		#extract SE name an SE_PATH (needed for publication)
84		SE, SE_PATH, User = self.splitEndpoint(endpoint)
#	Line 83 | Line 96 | class PhEDExDatasvcInfo:
96		if self.protocol == 'direct':
97		query=endpoint
98		SE_PATH = endpoint
99	<	### FEDE added SE ###
87	<	SE = self.sched
99	>	SE = self.SE[self.sched]
100		else:
101		url = 'http://'+endpoint.split('://')[1]
102		# python > 2.4
#	Line 94 | Line 106 | class PhEDExDatasvcInfo:
106		SE_PATH = endpoint.split(host)[1]
107		USER = (query.split('user')[1]).split('/')[1]
108		else:
109	+	#### to test #####
110	+	# url = 'http://'+endpoint.split('://')[1]
111	+	# scheme, host, path, params, query, fragment = urlparse(url)
112	+	# SE = host.split(':')[0]
113	+	# SE_PATH = endpoint.split(host)[1]
114		SE = self.node
115	<	SE_PATH = self.user_se_path + self.user_lfn
115	>	SE_PATH = self.user_se_path + self.user_remote_dir
116		try:
117		USER = (self.lfn.split('user')[1]).split('/')[1]
118		except:
#	Line 115 | Line 132 | class PhEDExDatasvcInfo:
132		## check if storage_name is a T2 (siteDB query)
133		## if yes :match self.user_lfn with LFNBaseName...
134		##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
135	<	lfn = self.user_lfn
135	>	lfn = self.user_remote_dir
136		return lfn
137		if self.publish_data_name == '' and int(self.publish_data) == 1:
138	<	msg = "Eeror. The [USER] section does not have 'publish_data_name'"
138	>	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
139	>	msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
140		raise CrabException(msg)
141		if self.publish_data_name == '' and int(self.usenamespace) == 1:
142		self.publish_data_name = "DefaultDataset"
143	<	if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
143	>	if int(self.publish_data) == 1:
144		if self.sched in ['CAF']: l_User=True
145		primaryDataset = self.computePrimaryDataset()
146	<	#lfn = LFNBase(primaryDataset,self.publish_data_name,LocalUser=l_User)  + '/${PSETHASH}/'
147	<	if self.usePhedex:
148	<	lfn = LFNBase('/store/user', primaryDataset, self.publish_data_name, LocalUser=l_User) + '/${PSETHASH}/'
149	<	else:
150	<	lfn = LFNBase(self.user_lfn, primaryDataset, self.publish_data_name, LocalUser=l_User) + '/${PSETHASH}/'
146	>	### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
147	>	### for the publication in order to be able to check the lfn length
148	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'
149	>	elif int(self.usenamespace) == 1:
150	>	if self.sched in ['CAF']: l_User=True
151	>	primaryDataset = self.computePrimaryDataset()
152	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'
153		else:
154		if self.sched in ['CAF','LSF']: l_User=True
155	<	lfn = LFNBase(self.user_remote_dir,LocalUser=l_User)
155	>	lfn = LFNBase(self.forced_path,self.user_remote_dir)
156		return lfn
157
158		def computePrimaryDataset(self):
#	Line 155 | Line 175 | class PhEDExDatasvcInfo:
175		params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
176		params = urllib.urlencode(params)
177		datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
158	–	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
178		try:
179	+	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
180		urlresults = parse(urlresults)
181	+	except IOError:
182	+	msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
183	+	raise CrabException(msg)
184		except:
185		urlresults = None
186

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines