ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.36 by fanzago, Thu Mar 18 17:22:45 2010 UTC vs.
Revision 1.49 by fanzago, Fri Oct 12 13:36:22 2012 UTC

#	Line 14 | Line 14 | class PhEDExDatasvcInfo:
14		self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15
16		self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17	<	self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17	>	self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCrabFaq#Stageout_and_publication'
18		self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19
20		self.usePhedex = True
#	Line 51 | Line 51 | class PhEDExDatasvcInfo:
51		self.datasetpath = cfg_params.get("CMSSW.datasetpath")
52		self.publish_data_name = cfg_params.get('USER.publish_data_name','')
53
54	+	self.pset = cfg_params.get('CMSSW.pset',None)
55	+
56		self.user_port = cfg_params.get("USER.storage_port",'8443')
57		self.user_se_path = cfg_params.get("USER.storage_path",'')
58		if self.user_se_path:
#	Line 64 | Line 66 | class PhEDExDatasvcInfo:
66		if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
67
68		if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
69	<	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
70	<	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
71	<	msg += '\t For further information please visit : \n\t%s'%self.stage_out_faq
69	>	####### FEDE FOR BUG 73010 ############
70	>	msg =  'Error: task ' + common.work_space._top_dir + ' not correctly created. Please remove it. \n'
71	>	msg += '      You are asking to stage out without using CMS Storage Name convention. In this case you \n'
72	>	msg += '      must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n'
73	>	msg += '      For further information please visit : \n\t%s'%self.stage_out_faq
74	>	task = common._db.getTask()
75	>	#add = '\n\n'
76	>	#import shutil
77	>	#try:
78	>	#    add += '      Task not correctly created: removing the working_dir ' +  common.work_space._top_dir + ' \n'
79	>	#    shutil.rmtree(common.work_space._top_dir)
80	>	#except OSError:
81	>	#    add += '      Warning: problems removing the working_dir ' + common.work_space._top_dir + ' \n'
82	>	#    add += '      Please remove it by hand'
83	>	#msg += add
84		raise CrabException(msg)
85
86		self.forced_path = '/store/user/'
87	<	if self.sched in ['CAF','LSF','PBS']:
87	>	if self.sched in ['LSF','PBS']:
88		self.srm_version = 'direct'
89	<	self.SE = {'CAF':'caf.cern.ch', 'LSF':'', 'PBS':''}
76	<	if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
89	>	self.SE = {'LSF':'', 'PBS':''}
90
91	+	if self.sched == 'CAF':
92	+	#### FEDE TEST FOR XROOTD
93	+	######### first solution ################
94	+	#eos = cfg_params.get("USER.caf_eos_area", 0)
95	+	#if eos == 0:
96	+	#    self.forced_path = '/store/caf/user/'
97	+	#else:
98	+	#    self.forced_path = '/store/eos/user'
99	+	#########################################
100	+	######### second solution ###############
101	+	self.forced_path = cfg_params.get("USER.caf_lfn", '/store/caf/user')
102	+	#########################################
103	+	#print "--->>> FORCING THE FIRST PART OF LFN WITH ", self.forced_path
104	+	self.SE = {'CAF':'caf.cern.ch'}
105	+	self.srm_version = 'stageout'
106	+	#print "--->>> query with 'stageout' "
107	+	#########################################
108	+
109		if not self.usePhedex:
110		self.forced_path = self.user_remote_dir
111		return
#	Line 88 | Line 119 | class PhEDExDatasvcInfo:
119		#extract the PFN for the given node,LFN,protocol
120		endpoint = self.getStageoutPFN()
121		if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
122	+	### FEDE bug fix 93573
123	+	if ( self.lfn[-1] != '/' ) : self.lfn = self.lfn + '/'
124	+
125	+	if int(self.publish_data) == 1 or  int(self.usenamespace) == 1:
126	+	self.lfn = self.lfn  + '${PSETHASH}/'
127	+	endpoint = endpoint  + '${PSETHASH}/'
128
129		#extract SE name an SE_PATH (needed for publication)
130		SE, SE_PATH, User = self.splitEndpoint(endpoint)
131
132	+	#### FEDE FOR XROOTD #####
133	+	#print "in getEndpoint di PhEDExDatasvcInfo.py: "
134	+	#print "    SE = ", SE
135	+	#print "    SE_PATH = ", SE_PATH
136	+	#print "    User = ", User
137	+	#print "    endpoint = ", endpoint
138	+	##############################
139	+
140		return endpoint, self.lfn , SE, SE_PATH, User
141
142		def splitEndpoint(self, endpoint):
#	Line 100 | Line 145 | class PhEDExDatasvcInfo:
145		'''
146		SE = ''
147		SE_PATH = ''
148	<	USER = ''
148	>	USER = getUserName()
149		if self.usePhedex:
150	<	if self.protocol == 'direct':
151	<	query=endpoint
107	<	SE_PATH = endpoint
150	>	### FEDE PER TEST WITH XROOTD
151	>	if (self.protocol == 'direct' or self.protocol == 'stageout'):
152		SE = self.SE[self.sched]
153	+	SE_PATH = endpoint
154	+	#############################
155	+	#print "    SE_PATH = ", SE_PATH
156		else:
157		url = 'http://'+endpoint.split('://')[1]
158		scheme, host, path, params, query, fragment = urlparse(url)
159		SE = self.getAuthoritativeSE()
160		SE_PATH = endpoint.split(host)[1]
114	–	### fede ###
115	–	#USER = (query.split('user')[1]).split('/')[1]
161		else:
117	–	#### to test #####
118	–	# url = 'http://'+endpoint.split('://')[1]
119	–	# scheme, host, path, params, query, fragment = urlparse(url)
120	–	# SE = host.split(':')[0]
121	–	# SE_PATH = endpoint.split(host)[1]
162		SE = self.node
163		SE_PATH = self.user_se_path + self.user_remote_dir
164	<	### fede ###
165	<	#if self.lfn.find('user'):
166	<	#    try:
167	<	#        USER = (self.lfn.split('user')[1]).split('/')[1]
168	<	#    except:
129	<	#        pass
130	<	### fede ###
131	<	if self.lfn.find('group') != -1:
132	<	try:
133	<	USER = (self.lfn.split('group')[1]).split('/')[1]
134	<	except:
135	<	pass
136	<	else:
137	<	USER = getUserName()
164	>	if self.lfn.find('group') != -1:
165	>	try:
166	>	USER = (self.lfn.split('group')[1]).split('/')[1]
167	>	except:
168	>	pass
169		return SE, SE_PATH, USER
139	–
170
171		def getLFN(self):
172		"""
#	Line 162 | Line 192 | class PhEDExDatasvcInfo:
192		primaryDataset = self.computePrimaryDataset()
193		### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
194		### for the publication in order to be able to check the lfn length
195	<	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'
195	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
196		elif int(self.usenamespace) == 1:
197		if self.sched in ['CAF']: l_User=True
198		primaryDataset = self.computePrimaryDataset()
199	<	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'
199	>	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)
200		else:
201		if self.sched in ['CAF','LSF']: l_User=True
202		lfn = LFNBase(self.forced_path,self.user_remote_dir)
203	+
204		if ( lfn[-1] != '/' ) : lfn = lfn + '/'
205	+
206		return lfn
207
208		def computePrimaryDataset(self):
#	Line 242 | Line 274 | class PhEDExDatasvcInfo:
274		params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
275		datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
276		fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
277	+	#print "--->>> fullurl = ", fullurl
278		domlfn2pfn = self.domPhedex(params,datasvc_lfn2pfn)
279		if not domlfn2pfn :
280		msg="Unable to get info from %s"%fullurl
#	Line 262 | Line 295 | class PhEDExDatasvcInfo:
295		raise CrabException(msg)
296		else:
297		if self.sched in ['CAF','LSF','PBS'] :
298	<	stageoutpfn = self.user_se_path+self.lfn
298	>	if (self.user_se_path[-1]=='/') and (self.lfn[0]=='/'):
299	>	stageoutpfn = self.user_se_path+(self.lfn).lstrip('/')
300	>	else:
301	>	stageoutpfn = self.user_se_path+self.lfn
302		else:
303		stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
304

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines