ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py
Revision: 1.26.2.5
Committed: Thu Oct 8 21:22:21 2009 UTC (15 years, 6 months ago) by spiga
Content type: text/x-python
Branch: CRAB_2_6_X_br
CVS Tags: CRAB_2_6_6_pre6, CRAB_2_6_6_pre5, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_6_5, CRAB_2_6_5_pre1, CRAB_2_6_4, CRAB_2_6_4_pre1, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1
Changes since 1.26.2.4: +1 -1 lines
Log Message:
align with fixes in the HEAD

File Contents

# User Rev Content
1 spiga 1.3 from Actor import *
2 afanfani 1.1 import urllib
3     from xml.dom.minidom import parse
4     from crab_exceptions import *
5 spiga 1.3 from WorkSpace import *
6     from urlparse import urlparse
7     from LFNBaseName import *
8 afanfani 1.1
9     class PhEDExDatasvcInfo:
10 fanzago 1.26.2.2 def __init__( self , cfg_params=None, config=None ):
11 spiga 1.3
12     ## PhEDEx Data Service URL
13 fanzago 1.26.2.2 self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
14 spiga 1.3
15 afanfani 1.6 self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
16 spiga 1.23 stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17 spiga 1.17 self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
18 spiga 1.3
19 fanzago 1.26.2.2 self.usePhedex = True
20 spiga 1.26.2.3 self.sched = common.scheduler.name().upper()
21 fanzago 1.26.2.2
22     if config!=None:
23     self.checkConfig(config)
24     else:
25     self.checkCfgConfig(cfg_params)
26    
27     self.protocol = self.srm_version
28    
29    
30     def checkConfig(self,config):
31     """
32     """
33     self.srm_version = config.get("srm_version",'srmv2')
34     self.node = config.get('storage_element',None)
35     self.lfn='/store/'
36    
37     def checkCfgConfig(self,cfg_params):
38     """
39     """
40     self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
41 spiga 1.3 self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
42     self.node = cfg_params.get('USER.storage_element',None)
43 spiga 1.17
44 spiga 1.3 self.publish_data = cfg_params.get("USER.publish_data",0)
45     self.usenamespace = cfg_params.get("USER.usenamespace",0)
46 fanzago 1.10 self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
47 fanzago 1.7 if self.user_remote_dir:
48     if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
49 spiga 1.17
50 spiga 1.3 self.datasetpath = cfg_params.get("CMSSW.datasetpath")
51     self.publish_data_name = cfg_params.get('USER.publish_data_name','')
52    
53 spiga 1.8 self.user_port = cfg_params.get("USER.storage_port",'8443')
54 spiga 1.3 self.user_se_path = cfg_params.get("USER.storage_path",'')
55 fanzago 1.7 if self.user_se_path:
56     if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
57    
58 spiga 1.3 #check if using "private" Storage
59 spiga 1.11 if not self.node :
60     msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
61 spiga 1.17 msg +='\tFor further information please visit : %s'%stage_out_faq
62 spiga 1.11 raise CrabException(msg)
63 spiga 1.4 if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
64 spiga 1.15
65     if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
66 spiga 1.3 msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
67 spiga 1.15 msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
68 spiga 1.17 msg += '\t For further information please visit : \n\t%s'%stage_out_faq
69 spiga 1.3 raise CrabException(msg)
70 spiga 1.22
71     self.forced_path = '/store/user/'
72 fanzago 1.18 if self.sched in ['CAF','LSF']:
73 spiga 1.26.2.5 self.srm_version= 'direct'
74 fanzago 1.18 self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
75 spiga 1.22 if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
76    
77 spiga 1.14 if not self.usePhedex:
78 spiga 1.15 self.forced_path = self.user_remote_dir
79 spiga 1.3 return
80    
81     def getEndpoint(self):
82     '''
83     Return full SE endpoint and related infos
84     '''
85     self.lfn = self.getLFN()
86    
87     #extract the PFN for the given node,LFN,protocol
88     endpoint = self.getStageoutPFN()
89 fanzago 1.26.2.1 ### FEDE added a check for the final /
90     if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91     ######################################
92 spiga 1.3
93     #extract SE name an SE_PATH (needed for publication)
94     SE, SE_PATH, User = self.splitEndpoint(endpoint)
95    
96     return endpoint, self.lfn , SE, SE_PATH, User
97    
98     def splitEndpoint(self, endpoint):
99     '''
100     Return relevant infos from endpoint
101     '''
102     SE = ''
103     SE_PATH = ''
104     USER = ''
105     if self.usePhedex:
106     if self.protocol == 'direct':
107     query=endpoint
108     SE_PATH = endpoint
109 fanzago 1.18 SE = self.SE[self.sched]
110 spiga 1.3 else:
111     url = 'http://'+endpoint.split('://')[1]
112     # python > 2.4
113     # SE = urlparse(url).hostname
114     scheme, host, path, params, query, fragment = urlparse(url)
115     SE = host.split(':')[0]
116     SE_PATH = endpoint.split(host)[1]
117     USER = (query.split('user')[1]).split('/')[1]
118     else:
119 fanzago 1.19 #### to test #####
120 spiga 1.21 # url = 'http://'+endpoint.split('://')[1]
121     # scheme, host, path, params, query, fragment = urlparse(url)
122     # SE = host.split(':')[0]
123     # SE_PATH = endpoint.split(host)[1]
124 spiga 1.20 SE = self.node
125     SE_PATH = self.user_se_path + self.user_remote_dir
126 spiga 1.26.2.4 if self.lfn.find('user'):
127     try:
128     USER = (self.lfn.split('user')[1]).split('/')[1]
129     except:
130     pass
131     if self.lfn.find('group'):
132     try:
133     USER = (self.lfn.split('group')[1]).split('/')[1]
134     except:
135     pass
136 spiga 1.3 return SE, SE_PATH, USER
137 afanfani 1.2
138 spiga 1.3
139     def getLFN(self):
140     """
141     define the LFN composing the needed pieces
142     """
143     lfn = ''
144     l_User = False
145     if not self.usePhedex and (int(self.publish_data) == 0 and int(self.usenamespace) == 0) :
146     ### add here check if user is trying to force a wrong LFN using a T2 TODO
147     ## check if storage_name is a T2 (siteDB query)
148     ## if yes :match self.user_lfn with LFNBaseName...
149     ## if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
150 spiga 1.15 lfn = self.user_remote_dir
151 spiga 1.3 return lfn
152     if self.publish_data_name == '' and int(self.publish_data) == 1:
153 spiga 1.17 msg = "Error. The [USER] section does not have 'publish_data_name'\n"
154     msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
155 spiga 1.3 raise CrabException(msg)
156     if self.publish_data_name == '' and int(self.usenamespace) == 1:
157     self.publish_data_name = "DefaultDataset"
158 fanzago 1.26 if int(self.publish_data) == 1:
159     if self.sched in ['CAF']: l_User=True
160     primaryDataset = self.computePrimaryDataset()
161     ### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
162     ### for the publication in order to be able to check the lfn length
163     lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True) + '/${PSETHASH}/'
164     elif int(self.usenamespace) == 1:
165 spiga 1.3 if self.sched in ['CAF']: l_User=True
166     primaryDataset = self.computePrimaryDataset()
167 spiga 1.21 lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name) + '/${PSETHASH}/'
168 spiga 1.3 else:
169 spiga 1.5 if self.sched in ['CAF','LSF']: l_User=True
170 spiga 1.21 lfn = LFNBase(self.forced_path,self.user_remote_dir)
171 spiga 1.3 return lfn
172    
173     def computePrimaryDataset(self):
174     """
175     compute the last part for the LFN in case of publication
176     """
177     if (self.datasetpath.upper() != 'NONE'):
178     primarydataset = self.datasetpath.split("/")[1]
179     else:
180     primarydataset = self.publish_data_name
181     return primarydataset
182    
183     def lfn2pfn(self):
184     """
185     PhEDEx Data Service lfn2pfn call
186    
187     input: LFN,node name,protocol
188     returns: DOM object with the content of the PhEDEx Data Service call
189     """
190     params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
191     params = urllib.urlencode(params)
192     datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
193     try:
194 slacapra 1.24 urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
195 spiga 1.3 urlresults = parse(urlresults)
196 slacapra 1.24 except IOError:
197     msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
198     raise CrabException(msg)
199 spiga 1.3 except:
200     urlresults = None
201    
202     return urlresults
203 afanfani 1.1
204 spiga 1.3 def parse_error(self,urlresults):
205     """
206     look for errors in the DOM object returned by PhEDEx Data Service call
207     """
208     errormsg = None
209     errors=urlresults.getElementsByTagName('error')
210     for error in errors:
211     errormsg=error.childNodes[0].data
212     if len(error.childNodes)>1:
213     errormsg+=error.childNodes[1].data
214     return errormsg
215    
216     def parse_lfn2pfn(self,urlresults):
217     """
218     Parse the content of the result of lfn2pfn PhEDEx Data Service call
219    
220     input: DOM object with the content of the lfn2pfn call
221     returns: PFN
222     """
223     result = urlresults.getElementsByTagName('phedex')
224    
225     if not result:
226     return []
227     result = result[0]
228     pfn = None
229     mapping = result.getElementsByTagName('mapping')
230     for m in mapping:
231     pfn=m.getAttribute("pfn")
232     if pfn:
233     return pfn
234    
235     def getStageoutPFN( self ):
236     """
237     input: LFN,node name,protocol
238     returns: PFN
239     """
240     if self.usePhedex:
241     fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
242     domlfn2pfn = self.lfn2pfn()
243     if not domlfn2pfn :
244     msg="Unable to get info from %s"%fullurl
245     raise CrabException(msg)
246    
247     errormsg = self.parse_error(domlfn2pfn)
248     if errormsg:
249     msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
250     raise CrabException(msg)
251    
252     stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
253     if not stageoutpfn:
254 afanfani 1.6 msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
255     msg+=' Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
256     msg+=' reporting: \n'
257     msg+=' Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
258     msg+=' OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
259 spiga 1.3 raise CrabException(msg)
260     else:
261 spiga 1.11 if self.sched in ['CAF','LSF'] :
262 spiga 1.12 stageoutpfn = self.user_se_path+self.lfn
263 spiga 1.11 else:
264     stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
265 spiga 1.3
266     return stageoutpfn
267 afanfani 1.6
268    
269    
270     if __name__ == '__main__':
271     """
272     Sort of unit testing to check Phedex API for whatever site and/or lfn.
273     Usage:
274     python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
275    
276     """
277     import getopt,sys
278     from crab_util import *
279     import common
280     klass_name = 'SchedulerGlite'
281     klass = importName(klass_name, klass_name)
282     common.scheduler = klass()
283    
284     lfn="/store/user/"
285     node='T2_IT_Bari'
286     valid = ['node=','lfn=']
287     try:
288     opts, args = getopt.getopt(sys.argv[1:], "", valid)
289     except getopt.GetoptError, ex:
290     print str(ex)
291     sys.exit(1)
292     for o, a in opts:
293     if o == "--node":
294     node = a
295     if o == "--lfn":
296     lfn = a
297    
298     mycfg_params = { 'USER.storage_element': node }
299     dsvc = PhEDExDatasvcInfo(mycfg_params)
300     dsvc.lfn = lfn
301     print dsvc.getStageoutPFN()
302