CRAB/python/PhEDExDatasvcInfo.py

from Actor import *
import urllib
from xml.dom.minidom import parse
from crab_exceptions import *
from crab_logger import Logger
from WorkSpace import *
from urlparse import urlparse 
from LFNBaseName import *

class PhEDExDatasvcInfo:
    def __init__( self , cfg_params ):
 
        ## PhEDEx Data Service URL
        url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
        self.datasvc_url = cfg_params.get("USER.datasvc_url",url)

        self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
        stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabFaq#How_to_store_output_with_CRAB_2'
        self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'

        self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
        self.node = cfg_params.get('USER.storage_element',None)
        

        self.user_lfn = cfg_params.get("USER.lfn",'')
        self.publish_data = cfg_params.get("USER.publish_data",0)
        self.usenamespace = cfg_params.get("USER.usenamespace",0)
        self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
        if self.user_remote_dir:
            if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
        if self.user_lfn:    
            msg  = 'Warning: lfn has been deprecated, CRAB will ignore it.\n'
            msg += '\t Please use only user_remote_dir removing lfn from your crab.cfg\n'
            msg += '\t For further information please visit : \n\t%s'%stage_out_faq 
            common.logger.message(msg)
          
        self.datasetpath = cfg_params.get("CMSSW.datasetpath")
        self.publish_data_name = cfg_params.get('USER.publish_data_name','')

        self.user_port = cfg_params.get("USER.storage_port",'8443')
        self.user_se_path = cfg_params.get("USER.storage_path",'')
        if self.user_se_path:
            if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
                                                    
        #check if using "private" Storage
        self.usePhedex = True 
        if not self.node :
            msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
            msg +='\tFor further information please visit : %s'%stage_out_faq 
            raise CrabException(msg)
        if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False 

        if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
            msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n' 
            msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
            msg += '\t For further information please visit : \n\t%s'%stage_out_faq
            raise CrabException(msg)
        self.sched = common.scheduler.name().upper() 
        self.protocol = self.srm_version
        if self.sched in ['CAF','LSF']:
            self.protocol = 'direct'
            self.SE = {'CAF':'caf.cern.ch', 'LSF':''}

        self.forced_path = '/store/user/'
        if not self.usePhedex: 
            self.forced_path = self.user_remote_dir
        return
 
    def getEndpoint(self):   
        '''
        Return full SE endpoint and related infos
        '''
        self.lfn = self.getLFN()
 
        #extract the PFN for the given node,LFN,protocol
        endpoint = self.getStageoutPFN()
   
        #extract SE name an SE_PATH (needed for publication)
        SE, SE_PATH, User = self.splitEndpoint(endpoint)

        return endpoint, self.lfn , SE, SE_PATH, User         
       
    def splitEndpoint(self, endpoint):
        '''
        Return relevant infos from endpoint  
        '''
        SE = ''
        SE_PATH = ''
        USER = ''
        if self.usePhedex: 
            if self.protocol == 'direct':
                query=endpoint
                SE_PATH = endpoint
                SE = self.SE[self.sched]
            else: 
                url = 'http://'+endpoint.split('://')[1]
                # python > 2.4
                # SE = urlparse(url).hostname 
                scheme, host, path, params, query, fragment = urlparse(url)
                SE = host.split(':')[0]
                SE_PATH = endpoint.split(host)[1]
            USER = (query.split('user')[1]).split('/')[1]
        else:
            #### to test #####
            #url = 'http://'+endpoint.split('://')[1]
            #scheme, host, path, params, query, fragment = urlparse(url)
            #SE = host.split(':')[0]
            #SE_PATH = endpoint.split(host)[1]
            SE = self.node
            SE_PATH = self.user_se_path + self.user_remote_dir 
            try:
                USER = (self.lfn.split('user')[1]).split('/')[1]
            except:
                pass

        return SE, SE_PATH, USER 
   

    def getLFN(self):
        """
        define the LFN composing the needed pieces
        """
        lfn = ''
        l_User = False
        if not self.usePhedex and (int(self.publish_data) == 0 and int(self.usenamespace) == 0) :
            ### add here check if user is trying to force a wrong LFN using a T2  TODO
            ## check if storage_name is a T2 (siteDB query)
            ## if yes :match self.user_lfn with LFNBaseName...
            ##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)   
            lfn = self.user_remote_dir
            return lfn
        if self.publish_data_name == '' and int(self.publish_data) == 1:
            msg = "Error. The [USER] section does not have 'publish_data_name'\n"
            msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
            raise CrabException(msg)
        if self.publish_data_name == '' and int(self.usenamespace) == 1:
           self.publish_data_name = "DefaultDataset"
        if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
            if self.sched in ['CAF']: l_User=True 
            primaryDataset = self.computePrimaryDataset()
            lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, LocalUser=l_User)  + '/${PSETHASH}/'    
        else:
            if self.sched in ['CAF','LSF']: l_User=True 
            lfn = LFNBase(self.forced_path,self.user_remote_dir,LocalUser=l_User)
        return lfn
 
    def computePrimaryDataset(self):
        """
        compute the last part for the LFN in case of publication     
        """
        if (self.datasetpath.upper() != 'NONE'):
            primarydataset = self.datasetpath.split("/")[1]
        else:
            primarydataset = self.publish_data_name
        return primarydataset
    
    def lfn2pfn(self):
        """
        PhEDEx Data Service lfn2pfn call
 
        input:   LFN,node name,protocol
        returns: DOM object with the content of the PhEDEx Data Service call
        """  
        params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
        params = urllib.urlencode(params)
        datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
        urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
        try:
            urlresults = parse(urlresults)
        except:
            urlresults = None

        return urlresults
 
    def parse_error(self,urlresults):
        """
        look for errors in the DOM object returned by PhEDEx Data Service call
        """
        errormsg = None 
        errors=urlresults.getElementsByTagName('error')
        for error in errors:
            errormsg=error.childNodes[0].data
            if len(error.childNodes)>1:
               errormsg+=error.childNodes[1].data
        return errormsg
 
    def parse_lfn2pfn(self,urlresults):
        """
        Parse the content of the result of lfn2pfn PhEDEx Data Service  call
 
        input:    DOM object with the content of the lfn2pfn call
        returns:  PFN  
        """
        result = urlresults.getElementsByTagName('phedex')
               
        if not result:
              return []
        result = result[0]
        pfn = None
        mapping = result.getElementsByTagName('mapping')
        for m in mapping:
            pfn=m.getAttribute("pfn")
            if pfn:
              return pfn
 
    def getStageoutPFN( self ):
        """
        input:   LFN,node name,protocol
        returns: PFN 
        """
        if self.usePhedex:
            fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol) 
            domlfn2pfn = self.lfn2pfn()
            if not domlfn2pfn :
                msg="Unable to get info from %s"%fullurl
                raise CrabException(msg)
  
            errormsg = self.parse_error(domlfn2pfn)
            if errormsg: 
                msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
                raise CrabException(msg)
  
            stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
            if not stageoutpfn:
                msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
                msg+='      Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
                msg+='      reporting: \n'
                msg+='       Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
                msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
                raise CrabException(msg)
        else:
            if self.sched in ['CAF','LSF'] :
                stageoutpfn = self.user_se_path+self.lfn 
            else: 
                stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn 

        return stageoutpfn 


if __name__ == '__main__':
  """
  Sort of unit testing to check Phedex API for whatever site and/or lfn.
  Usage:
     python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma

  """
  import getopt,sys
  from crab_util import *
  import common
  klass_name = 'SchedulerGlite'
  klass = importName(klass_name, klass_name)
  common.scheduler = klass()

  lfn="/store/user/"
  node='T2_IT_Bari'
  valid = ['node=','lfn=']
  try:
       opts, args = getopt.getopt(sys.argv[1:], "", valid)
  except getopt.GetoptError, ex:
       print str(ex)
       sys.exit(1)
  for o, a in opts:
        if o == "--node":
            node = a
        if o == "--lfn":
            lfn = a
  
  mycfg_params = { 'USER.storage_element': node }
  dsvc = PhEDExDatasvcInfo(mycfg_params)
  dsvc.lfn = lfn
  print dsvc.getStageoutPFN()

Revision:	1.20
Committed:	Fri Mar 6 17:01:52 2009 UTC (16 years, 1 month ago) by spiga
Content type:	text/x-python
Branch:	MAIN
Changes since 1.19:	+6 -6 lines
Log Message:	roll back...
#	User	Rev	Content
1	spiga	1.3	from Actor import *
2	afanfani	1.1	import urllib
3			from xml.dom.minidom import parse
4			from crab_exceptions import *
5	spiga	1.3	from crab_logger import Logger
6			from WorkSpace import *
7			from urlparse import urlparse
8			from LFNBaseName import *
9	afanfani	1.1
10			class PhEDExDatasvcInfo:
11	spiga	1.3	def __init__( self , cfg_params ):
12
13			## PhEDEx Data Service URL
14			url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15			self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
16
17	afanfani	1.6	self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
18	spiga	1.17	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabFaq#How_to_store_output_with_CRAB_2'
19			self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
20	spiga	1.3
21			self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
22			self.node = cfg_params.get('USER.storage_element',None)
23
24	spiga	1.17
25
26			self.user_lfn = cfg_params.get("USER.lfn",'')
27	spiga	1.3	self.publish_data = cfg_params.get("USER.publish_data",0)
28			self.usenamespace = cfg_params.get("USER.usenamespace",0)
29	fanzago	1.10	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
30	fanzago	1.7	if self.user_remote_dir:
31			if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
32	spiga	1.17	if self.user_lfn:
33			msg = 'Warning: lfn has been deprecated, CRAB will ignore it.\n'
34			msg += '\t Please use only user_remote_dir removing lfn from your crab.cfg\n'
35			msg += '\t For further information please visit : \n\t%s'%stage_out_faq
36			common.logger.message(msg)
37
38	spiga	1.3	self.datasetpath = cfg_params.get("CMSSW.datasetpath")
39			self.publish_data_name = cfg_params.get('USER.publish_data_name','')
40
41	spiga	1.8	self.user_port = cfg_params.get("USER.storage_port",'8443')
42	spiga	1.3	self.user_se_path = cfg_params.get("USER.storage_path",'')
43	fanzago	1.7	if self.user_se_path:
44			if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
45
46	spiga	1.3	#check if using "private" Storage
47			self.usePhedex = True
48	spiga	1.11	if not self.node :
49			msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
50	spiga	1.17	msg +='\tFor further information please visit : %s'%stage_out_faq
51	spiga	1.11	raise CrabException(msg)
52	spiga	1.4	if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
53	spiga	1.15
54			if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
55	spiga	1.3	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
56	spiga	1.15	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
57	spiga	1.17	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
58	spiga	1.3	raise CrabException(msg)
59			self.sched = common.scheduler.name().upper()
60			self.protocol = self.srm_version
61	fanzago	1.18	if self.sched in ['CAF','LSF']:
62			self.protocol = 'direct'
63			self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
64	spiga	1.3
65	spiga	1.14	self.forced_path = '/store/user/'
66			if not self.usePhedex:
67	spiga	1.15	self.forced_path = self.user_remote_dir
68	spiga	1.3	return
69
70			def getEndpoint(self):
71			'''
72			Return full SE endpoint and related infos
73			'''
74			self.lfn = self.getLFN()
75
76			#extract the PFN for the given node,LFN,protocol
77			endpoint = self.getStageoutPFN()
78
79			#extract SE name an SE_PATH (needed for publication)
80			SE, SE_PATH, User = self.splitEndpoint(endpoint)
81
82			return endpoint, self.lfn , SE, SE_PATH, User
83
84			def splitEndpoint(self, endpoint):
85			'''
86			Return relevant infos from endpoint
87			'''
88			SE = ''
89			SE_PATH = ''
90			USER = ''
91			if self.usePhedex:
92			if self.protocol == 'direct':
93			query=endpoint
94			SE_PATH = endpoint
95	fanzago	1.18	SE = self.SE[self.sched]
96	spiga	1.3	else:
97			url = 'http://'+endpoint.split('://')[1]
98			# python > 2.4
99			# SE = urlparse(url).hostname
100			scheme, host, path, params, query, fragment = urlparse(url)
101			SE = host.split(':')[0]
102			SE_PATH = endpoint.split(host)[1]
103			USER = (query.split('user')[1]).split('/')[1]
104			else:
105	fanzago	1.19	#### to test #####
106	spiga	1.20	#url = 'http://'+endpoint.split('://')[1]
107			#scheme, host, path, params, query, fragment = urlparse(url)
108			#SE = host.split(':')[0]
109			#SE_PATH = endpoint.split(host)[1]
110			SE = self.node
111			SE_PATH = self.user_se_path + self.user_remote_dir
112	spiga	1.3	try:
113			USER = (self.lfn.split('user')[1]).split('/')[1]
114			except:
115			pass
116
117			return SE, SE_PATH, USER
118	afanfani	1.2
119	spiga	1.3
120			def getLFN(self):
121			"""
122			define the LFN composing the needed pieces
123			"""
124			lfn = ''
125			l_User = False
126			if not self.usePhedex and (int(self.publish_data) == 0 and int(self.usenamespace) == 0) :
127			### add here check if user is trying to force a wrong LFN using a T2 TODO
128			## check if storage_name is a T2 (siteDB query)
129			## if yes :match self.user_lfn with LFNBaseName...
130			## if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
131	spiga	1.15	lfn = self.user_remote_dir
132	spiga	1.3	return lfn
133			if self.publish_data_name == '' and int(self.publish_data) == 1:
134	spiga	1.17	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
135			msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
136	spiga	1.3	raise CrabException(msg)
137			if self.publish_data_name == '' and int(self.usenamespace) == 1:
138			self.publish_data_name = "DefaultDataset"
139			if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
140			if self.sched in ['CAF']: l_User=True
141			primaryDataset = self.computePrimaryDataset()
142	spiga	1.14	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, LocalUser=l_User) + '/${PSETHASH}/'
143	spiga	1.3	else:
144	spiga	1.5	if self.sched in ['CAF','LSF']: l_User=True
145	spiga	1.14	lfn = LFNBase(self.forced_path,self.user_remote_dir,LocalUser=l_User)
146	spiga	1.3	return lfn
147
148			def computePrimaryDataset(self):
149			"""
150			compute the last part for the LFN in case of publication
151			"""
152			if (self.datasetpath.upper() != 'NONE'):
153			primarydataset = self.datasetpath.split("/")[1]
154			else:
155			primarydataset = self.publish_data_name
156			return primarydataset
157
158			def lfn2pfn(self):
159			"""
160			PhEDEx Data Service lfn2pfn call
161
162			input: LFN,node name,protocol
163			returns: DOM object with the content of the PhEDEx Data Service call
164			"""
165			params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
166			params = urllib.urlencode(params)
167			datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
168			urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
169			try:
170			urlresults = parse(urlresults)
171			except:
172			urlresults = None
173
174			return urlresults
175	afanfani	1.1
176	spiga	1.3	def parse_error(self,urlresults):
177			"""
178			look for errors in the DOM object returned by PhEDEx Data Service call
179			"""
180			errormsg = None
181			errors=urlresults.getElementsByTagName('error')
182			for error in errors:
183			errormsg=error.childNodes[0].data
184			if len(error.childNodes)>1:
185			errormsg+=error.childNodes[1].data
186			return errormsg
187
188			def parse_lfn2pfn(self,urlresults):
189			"""
190			Parse the content of the result of lfn2pfn PhEDEx Data Service call
191
192			input: DOM object with the content of the lfn2pfn call
193			returns: PFN
194			"""
195			result = urlresults.getElementsByTagName('phedex')
196
197			if not result:
198			return []
199			result = result[0]
200			pfn = None
201			mapping = result.getElementsByTagName('mapping')
202			for m in mapping:
203			pfn=m.getAttribute("pfn")
204			if pfn:
205			return pfn
206
207			def getStageoutPFN( self ):
208			"""
209			input: LFN,node name,protocol
210			returns: PFN
211			"""
212			if self.usePhedex:
213			fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
214			domlfn2pfn = self.lfn2pfn()
215			if not domlfn2pfn :
216			msg="Unable to get info from %s"%fullurl
217			raise CrabException(msg)
218
219			errormsg = self.parse_error(domlfn2pfn)
220			if errormsg:
221			msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
222			raise CrabException(msg)
223
224			stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
225			if not stageoutpfn:
226	afanfani	1.6	msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
227			msg+=' Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
228			msg+=' reporting: \n'
229			msg+=' Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
230			msg+=' OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
231	spiga	1.3	raise CrabException(msg)
232			else:
233	spiga	1.11	if self.sched in ['CAF','LSF'] :
234	spiga	1.12	stageoutpfn = self.user_se_path+self.lfn
235	spiga	1.11	else:
236			stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
237	spiga	1.3
238			return stageoutpfn
239	afanfani	1.6
240
241
242			if __name__ == '__main__':
243			"""
244			Sort of unit testing to check Phedex API for whatever site and/or lfn.
245			Usage:
246			python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
247
248			"""
249			import getopt,sys
250			from crab_util import *
251			import common
252			klass_name = 'SchedulerGlite'
253			klass = importName(klass_name, klass_name)
254			common.scheduler = klass()
255
256			lfn="/store/user/"
257			node='T2_IT_Bari'
258			valid = ['node=','lfn=']
259			try:
260			opts, args = getopt.getopt(sys.argv[1:], "", valid)
261			except getopt.GetoptError, ex:
262			print str(ex)
263			sys.exit(1)
264			for o, a in opts:
265			if o == "--node":
266			node = a
267			if o == "--lfn":
268			lfn = a
269
270			mycfg_params = { 'USER.storage_element': node }
271			dsvc = PhEDExDatasvcInfo(mycfg_params)
272			dsvc.lfn = lfn
273			print dsvc.getStageoutPFN()
274