ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py
(Generate patch)

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.28 by fanzago, Fri Sep 18 16:31:13 2009 UTC vs.
Revision 1.41 by fanzago, Fri Aug 5 15:39:08 2011 UTC

# Line 5 | Line 5 | from crab_exceptions import *
5   from WorkSpace import *
6   from urlparse import urlparse
7   from LFNBaseName import *
8 + from crab_util import getUserName
9  
10   class PhEDExDatasvcInfo:
11      def __init__( self , cfg_params=None, config=None ):
# Line 13 | Line 14 | class PhEDExDatasvcInfo:
14          self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15  
16          self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
17 <        stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17 >        self.stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
18          self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
19  
20          self.usePhedex = True
21 +        self.sched = common.scheduler.name().upper()
22  
23          if config!=None:
24              self.checkConfig(config)  
25          else:
26              self.checkCfgConfig(cfg_params)  
27  
26 –        self.sched = common.scheduler.name().upper()
28          self.protocol = self.srm_version
29  
30  
# Line 58 | Line 59 | class PhEDExDatasvcInfo:
59          #check if using "private" Storage
60          if not self.node :
61              msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
62 <            msg +='\tFor further information please visit : %s'%stage_out_faq
62 >            msg +='\tFor further information please visit : %s'%self.stage_out_faq
63              raise CrabException(msg)
64          if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
65  
66          if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
67 <            msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
68 <            msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
69 <            msg += '\t For further information please visit : \n\t%s'%stage_out_faq
67 >            ####### FEDE FOR BUG 73010 ############
68 >            msg = 'Error: you are asking to stage out without using CMS Storage Name convention. In this case you \n'
69 >            msg += '      must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n'
70 >            msg += '      For further information please visit : \n\t%s'%self.stage_out_faq
71 >            task = common._db.getTask()
72 >            add = '\n\n'
73 >            import shutil
74 >            try:
75 >                add += '      Task not correctly created: removing the working_dir ' +  common.work_space._top_dir + ' \n'
76 >                shutil.rmtree(common.work_space._top_dir)
77 >            except OSError:
78 >                add += '      Warning: problems removing the working_dir ' + common.work_space._top_dir + ' \n'
79 >                add += '      Please remove it by hand'
80 >            msg += add
81              raise CrabException(msg)
82 +            #########################################
83  
84          self.forced_path = '/store/user/'
85 <        if self.sched in ['CAF','LSF']:
86 <            self.protocol = 'direct'
87 <            self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
85 >        if self.sched in ['CAF','LSF','PBS']:
86 >            self.srm_version = 'direct'
87 >            self.SE = {'CAF':'caf.cern.ch', 'LSF':'', 'PBS':''}
88              if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
89              
90          if not self.usePhedex:
# Line 86 | Line 99 | class PhEDExDatasvcInfo:
99  
100          #extract the PFN for the given node,LFN,protocol
101          endpoint = self.getStageoutPFN()
89 –        ### FEDE added a check for the final /
102          if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91 –        ######################################
103    
104          #extract SE name an SE_PATH (needed for publication)
105          SE, SE_PATH, User = self.splitEndpoint(endpoint)
# Line 101 | Line 112 | class PhEDExDatasvcInfo:
112          '''
113          SE = ''
114          SE_PATH = ''
115 <        USER = ''
115 >        USER = getUserName()
116          if self.usePhedex:
117              if self.protocol == 'direct':
118                  query=endpoint
# Line 109 | Line 120 | class PhEDExDatasvcInfo:
120                  SE = self.SE[self.sched]
121              else:
122                  url = 'http://'+endpoint.split('://')[1]
112 –                # python > 2.4
113 –                # SE = urlparse(url).hostname
123                  scheme, host, path, params, query, fragment = urlparse(url)
124 <                SE = host.split(':')[0]
124 >                SE = self.getAuthoritativeSE()
125                  SE_PATH = endpoint.split(host)[1]
117 –            USER = (query.split('user')[1]).split('/')[1]
126          else:
119 –            #### to test #####
120 –           # url = 'http://'+endpoint.split('://')[1]
121 –           # scheme, host, path, params, query, fragment = urlparse(url)
122 –           # SE = host.split(':')[0]
123 –           # SE_PATH = endpoint.split(host)[1]
127              SE = self.node
128              SE_PATH = self.user_se_path + self.user_remote_dir
129 <            try:
130 <                USER = (self.lfn.split('user')[1]).split('/')[1]
131 <            except:
132 <                pass
133 <
129 >            if self.lfn.find('group') != -1:
130 >                try:
131 >                    USER = (self.lfn.split('group')[1]).split('/')[1]
132 >                except:
133 >                    pass
134          return SE, SE_PATH, USER
132 –  
135  
136      def getLFN(self):
137          """
# Line 163 | Line 165 | class PhEDExDatasvcInfo:
165          else:
166              if self.sched in ['CAF','LSF']: l_User=True
167              lfn = LFNBase(self.forced_path,self.user_remote_dir)
168 +        if ( lfn[-1] != '/' ) : lfn = lfn + '/'
169          return lfn
170  
171      def computePrimaryDataset(self):
# Line 175 | Line 178 | class PhEDExDatasvcInfo:
178              primarydataset = self.publish_data_name
179          return primarydataset
180      
181 <    def lfn2pfn(self):
181 >    def domPhedex(self,params,datasvc_baseUrl):
182          """
183          PhEDEx Data Service lfn2pfn call
184  
185 <        input:   LFN,node name,protocol
185 >        input:   params,datasvc_baseUrl
186          returns: DOM object with the content of the PhEDEx Data Service call
187          """  
185 –        params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
188          params = urllib.urlencode(params)
187 –        datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
189          try:
190 <            urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
190 >            urlresults = urllib.urlopen(datasvc_baseUrl, params)
191              urlresults = parse(urlresults)
192          except IOError:
193 <            msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
193 >            msg="Unable to access PhEDEx Data Service at %s"%datasvc_baseUrl
194              raise CrabException(msg)
195          except:
196              urlresults = None
# Line 233 | Line 234 | class PhEDExDatasvcInfo:
234          returns: PFN
235          """
236          if self.usePhedex:
237 +            params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
238 +            datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
239              fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
240 <            domlfn2pfn = self.lfn2pfn()
240 >            domlfn2pfn = self.domPhedex(params,datasvc_lfn2pfn)
241              if not domlfn2pfn :
242                  msg="Unable to get info from %s"%fullurl
243                  raise CrabException(msg)
# Line 253 | Line 256 | class PhEDExDatasvcInfo:
256                  msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
257                  raise CrabException(msg)
258          else:
259 <            if self.sched in ['CAF','LSF'] :
259 >            if self.sched in ['CAF','LSF','PBS'] :
260                  stageoutpfn = self.user_se_path+self.lfn
261              else:
262                  stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
263  
264 +        if ( stageoutpfn[-1] != '/' ) : stageoutpfn = stageoutpfn + '/'
265          return stageoutpfn
266  
267 +    def getAuthoritativeSE(self):
268 +        """
269 +        input:   node name
270 +        returns: AuthoritativeSE
271 +        """
272 +        params = {'node' : self.node }
273 +        datasvc_nodes="%s/nodes"%self.datasvc_url
274 +        fullurl="%s/nodes/?node=%s"%(self.datasvc_url,self.node)
275 +        domnodes = self.domPhedex(params,datasvc_nodes)
276 +
277 +        if not domnodes :
278 +            msg="Unable to get info from %s"%fullurl
279 +            raise CrabException(msg)
280 +
281 +        errormsg = self.parse_error(domnodes)
282 +        if errormsg:
283 +            msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
284 +            raise CrabException(msg)
285 +        result = domnodes.getElementsByTagName('phedex')
286 +        if not result:
287 +              return []
288 +        result = result[0]
289 +        se = None
290 +        node = result.getElementsByTagName('node')
291 +        for m in node:
292 +            se=m.getAttribute("se")
293 +            if se:
294 +                return se
295  
296  
297   if __name__ == '__main__':

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines