CRAB/python/PhEDExDatasvcInfo.py

from Actor import *
import urllib
from xml.dom.minidom import parse
from crab_exceptions import *
from WorkSpace import *
from urlparse import urlparse 
from LFNBaseName import *

class PhEDExDatasvcInfo:
    def __init__( self , cfg_params=None, config=None ):
 
        ## PhEDEx Data Service URL
        self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"

        self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
        stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication' 
        self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'

        self.usePhedex = True 
        self.sched = common.scheduler.name().upper() 

        if config!=None:
            self.checkConfig(config)  
        else:
            self.checkCfgConfig(cfg_params)  

        self.protocol = self.srm_version
 

    def checkConfig(self,config):
        """
        """
        self.srm_version = config.get("srm_version",'srmv2')
        self.node = config.get('storage_element',None)
        self.lfn='/store/'

    def checkCfgConfig(self,cfg_params):
        """
        """
        self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
        self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
        self.node = cfg_params.get('USER.storage_element',None)

        self.publish_data = cfg_params.get("USER.publish_data",0)
        self.usenamespace = cfg_params.get("USER.usenamespace",0)
        self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
        if self.user_remote_dir:
            if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
          
        self.datasetpath = cfg_params.get("CMSSW.datasetpath")
        self.publish_data_name = cfg_params.get('USER.publish_data_name','')

        self.user_port = cfg_params.get("USER.storage_port",'8443')
        self.user_se_path = cfg_params.get("USER.storage_path",'')
        if self.user_se_path:
            if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
                                                    
        #check if using "private" Storage
        if not self.node :
            msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
            msg +='\tFor further information please visit : %s'%stage_out_faq 
            raise CrabException(msg)
        if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False 

        if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
            msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n' 
            msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
            msg += '\t For further information please visit : \n\t%s'%stage_out_faq
            raise CrabException(msg)

        self.forced_path = '/store/user/'
        if self.sched in ['CAF','LSF']:
            self.protocol = 'direct'
            self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
            if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
            
        if not self.usePhedex: 
            self.forced_path = self.user_remote_dir
        return
 
    def getEndpoint(self):   
        '''
        Return full SE endpoint and related infos
        '''
        self.lfn = self.getLFN()
 
        #extract the PFN for the given node,LFN,protocol
        endpoint = self.getStageoutPFN()
        ### FEDE added a check for the final / 
        if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
        ######################################
   
        #extract SE name an SE_PATH (needed for publication)
        SE, SE_PATH, User = self.splitEndpoint(endpoint)

        return endpoint, self.lfn , SE, SE_PATH, User         
       
    def splitEndpoint(self, endpoint):
        '''
        Return relevant infos from endpoint  
        '''
        SE = ''
        SE_PATH = ''
        USER = ''
        if self.usePhedex: 
            if self.protocol == 'direct':
                query=endpoint
                SE_PATH = endpoint
                SE = self.SE[self.sched]
            else: 
                url = 'http://'+endpoint.split('://')[1]
                # python > 2.4
                # SE = urlparse(url).hostname 
                scheme, host, path, params, query, fragment = urlparse(url)
                SE = host.split(':')[0]
                SE_PATH = endpoint.split(host)[1]
            USER = (query.split('user')[1]).split('/')[1]
        else:
            #### to test #####
           # url = 'http://'+endpoint.split('://')[1]
           # scheme, host, path, params, query, fragment = urlparse(url)
           # SE = host.split(':')[0]
           # SE_PATH = endpoint.split(host)[1]
            SE = self.node
            SE_PATH = self.user_se_path + self.user_remote_dir 
            try:
                USER = (self.lfn.split('user')[1]).split('/')[1]
            except:
                pass

        return SE, SE_PATH, USER 
   

    def getLFN(self):
        """
        define the LFN composing the needed pieces
        """
        lfn = ''
        l_User = False
        if not self.usePhedex and (int(self.publish_data) == 0 and int(self.usenamespace) == 0) :
            ### add here check if user is trying to force a wrong LFN using a T2  TODO
            ## check if storage_name is a T2 (siteDB query)
            ## if yes :match self.user_lfn with LFNBaseName...
            ##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)   
            lfn = self.user_remote_dir
            return lfn
        if self.publish_data_name == '' and int(self.publish_data) == 1:
            msg = "Error. The [USER] section does not have 'publish_data_name'\n"
            msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
            raise CrabException(msg)
        if self.publish_data_name == '' and int(self.usenamespace) == 1:
           self.publish_data_name = "DefaultDataset"
        if int(self.publish_data) == 1:
            if self.sched in ['CAF']: l_User=True 
            primaryDataset = self.computePrimaryDataset()
            ### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
            ### for the publication in order to be able to check the lfn length  
            lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'    
        elif int(self.usenamespace) == 1:
            if self.sched in ['CAF']: l_User=True 
            primaryDataset = self.computePrimaryDataset()
            lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'    
        else:
            if self.sched in ['CAF','LSF']: l_User=True 
            lfn = LFNBase(self.forced_path,self.user_remote_dir)
        return lfn
 
    def computePrimaryDataset(self):
        """
        compute the last part for the LFN in case of publication     
        """
        if (self.datasetpath.upper() != 'NONE'):
            primarydataset = self.datasetpath.split("/")[1]
        else:
            primarydataset = self.publish_data_name
        return primarydataset
    
    def lfn2pfn(self):
        """
        PhEDEx Data Service lfn2pfn call
 
        input:   LFN,node name,protocol
        returns: DOM object with the content of the PhEDEx Data Service call
        """  
        params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
        params = urllib.urlencode(params)
        datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
        try:
            urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
            urlresults = parse(urlresults)
        except IOError:
            msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
            raise CrabException(msg)
        except:
            urlresults = None

        return urlresults
 
    def parse_error(self,urlresults):
        """
        look for errors in the DOM object returned by PhEDEx Data Service call
        """
        errormsg = None 
        errors=urlresults.getElementsByTagName('error')
        for error in errors:
            errormsg=error.childNodes[0].data
            if len(error.childNodes)>1:
               errormsg+=error.childNodes[1].data
        return errormsg
 
    def parse_lfn2pfn(self,urlresults):
        """
        Parse the content of the result of lfn2pfn PhEDEx Data Service  call
 
        input:    DOM object with the content of the lfn2pfn call
        returns:  PFN  
        """
        result = urlresults.getElementsByTagName('phedex')
               
        if not result:
              return []
        result = result[0]
        pfn = None
        mapping = result.getElementsByTagName('mapping')
        for m in mapping:
            pfn=m.getAttribute("pfn")
            if pfn:
              return pfn
 
    def getStageoutPFN( self ):
        """
        input:   LFN,node name,protocol
        returns: PFN 
        """
        if self.usePhedex:
            fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol) 
            domlfn2pfn = self.lfn2pfn()
            if not domlfn2pfn :
                msg="Unable to get info from %s"%fullurl
                raise CrabException(msg)
  
            errormsg = self.parse_error(domlfn2pfn)
            if errormsg: 
                msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
                raise CrabException(msg)
  
            stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
            if not stageoutpfn:
                msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
                msg+='      Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
                msg+='      reporting: \n'
                msg+='       Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
                msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
                raise CrabException(msg)
        else:
            if self.sched in ['CAF','LSF'] :
                stageoutpfn = self.user_se_path+self.lfn 
            else: 
                stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn 

        return stageoutpfn 


if __name__ == '__main__':
  """
  Sort of unit testing to check Phedex API for whatever site and/or lfn.
  Usage:
     python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma

  """
  import getopt,sys
  from crab_util import *
  import common
  klass_name = 'SchedulerGlite'
  klass = importName(klass_name, klass_name)
  common.scheduler = klass()

  lfn="/store/user/"
  node='T2_IT_Bari'
  valid = ['node=','lfn=']
  try:
       opts, args = getopt.getopt(sys.argv[1:], "", valid)
  except getopt.GetoptError, ex:
       print str(ex)
       sys.exit(1)
  for o, a in opts:
        if o == "--node":
            node = a
        if o == "--lfn":
            lfn = a
  
  mycfg_params = { 'USER.storage_element': node }
  dsvc = PhEDExDatasvcInfo(mycfg_params)
  dsvc.lfn = lfn
  print dsvc.getStageoutPFN()

Revision:	1.26.2.3
Committed:	Sat Sep 19 11:15:57 2009 UTC (15 years, 7 months ago) by spiga
Content type:	text/x-python
Branch:	CRAB_2_6_X_br
CVS Tags:	CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2
Changes since 1.26.2.2:	+1 -1 lines
Log Message:	sync with head
#	Content
1	from Actor import *
2	import urllib
3	from xml.dom.minidom import parse
4	from crab_exceptions import *
5	from WorkSpace import *
6	from urlparse import urlparse
7	from LFNBaseName import *
8
9	class PhEDExDatasvcInfo:
10	def __init__( self , cfg_params=None, config=None ):
11
12	## PhEDEx Data Service URL
13	self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
14
15	self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
16	stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17	self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
18
19	self.usePhedex = True
20	self.sched = common.scheduler.name().upper()
21
22	if config!=None:
23	self.checkConfig(config)
24	else:
25	self.checkCfgConfig(cfg_params)
26
27	self.protocol = self.srm_version
28
29
30	def checkConfig(self,config):
31	"""
32	"""
33	self.srm_version = config.get("srm_version",'srmv2')
34	self.node = config.get('storage_element',None)
35	self.lfn='/store/'
36
37	def checkCfgConfig(self,cfg_params):
38	"""
39	"""
40	self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
41	self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
42	self.node = cfg_params.get('USER.storage_element',None)
43
44	self.publish_data = cfg_params.get("USER.publish_data",0)
45	self.usenamespace = cfg_params.get("USER.usenamespace",0)
46	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
47	if self.user_remote_dir:
48	if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
49
50	self.datasetpath = cfg_params.get("CMSSW.datasetpath")
51	self.publish_data_name = cfg_params.get('USER.publish_data_name','')
52
53	self.user_port = cfg_params.get("USER.storage_port",'8443')
54	self.user_se_path = cfg_params.get("USER.storage_path",'')
55	if self.user_se_path:
56	if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
57
58	#check if using "private" Storage
59	if not self.node :
60	msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
61	msg +='\tFor further information please visit : %s'%stage_out_faq
62	raise CrabException(msg)
63	if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
64
65	if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
66	msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
67	msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
68	msg += '\t For further information please visit : \n\t%s'%stage_out_faq
69	raise CrabException(msg)
70
71	self.forced_path = '/store/user/'
72	if self.sched in ['CAF','LSF']:
73	self.protocol = 'direct'
74	self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
75	if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
76
77	if not self.usePhedex:
78	self.forced_path = self.user_remote_dir
79	return
80
81	def getEndpoint(self):
82	'''
83	Return full SE endpoint and related infos
84	'''
85	self.lfn = self.getLFN()
86
87	#extract the PFN for the given node,LFN,protocol
88	endpoint = self.getStageoutPFN()
89	### FEDE added a check for the final /
90	if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91	######################################
92
93	#extract SE name an SE_PATH (needed for publication)
94	SE, SE_PATH, User = self.splitEndpoint(endpoint)
95
96	return endpoint, self.lfn , SE, SE_PATH, User
97
98	def splitEndpoint(self, endpoint):
99	'''
100	Return relevant infos from endpoint
101	'''
102	SE = ''
103	SE_PATH = ''
104	USER = ''
105	if self.usePhedex:
106	if self.protocol == 'direct':
107	query=endpoint
108	SE_PATH = endpoint
109	SE = self.SE[self.sched]
110	else:
111	url = 'http://'+endpoint.split('://')[1]
112	# python > 2.4
113	# SE = urlparse(url).hostname
114	scheme, host, path, params, query, fragment = urlparse(url)
115	SE = host.split(':')[0]
116	SE_PATH = endpoint.split(host)[1]
117	USER = (query.split('user')[1]).split('/')[1]
118	else:
119	#### to test #####
120	# url = 'http://'+endpoint.split('://')[1]
121	# scheme, host, path, params, query, fragment = urlparse(url)
122	# SE = host.split(':')[0]
123	# SE_PATH = endpoint.split(host)[1]
124	SE = self.node
125	SE_PATH = self.user_se_path + self.user_remote_dir
126	try:
127	USER = (self.lfn.split('user')[1]).split('/')[1]
128	except:
129	pass
130
131	return SE, SE_PATH, USER
132
133
134	def getLFN(self):
135	"""
136	define the LFN composing the needed pieces
137	"""
138	lfn = ''
139	l_User = False
140	if not self.usePhedex and (int(self.publish_data) == 0 and int(self.usenamespace) == 0) :
141	### add here check if user is trying to force a wrong LFN using a T2 TODO
142	## check if storage_name is a T2 (siteDB query)
143	## if yes :match self.user_lfn with LFNBaseName...
144	## if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)
145	lfn = self.user_remote_dir
146	return lfn
147	if self.publish_data_name == '' and int(self.publish_data) == 1:
148	msg = "Error. The [USER] section does not have 'publish_data_name'\n"
149	msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
150	raise CrabException(msg)
151	if self.publish_data_name == '' and int(self.usenamespace) == 1:
152	self.publish_data_name = "DefaultDataset"
153	if int(self.publish_data) == 1:
154	if self.sched in ['CAF']: l_User=True
155	primaryDataset = self.computePrimaryDataset()
156	### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
157	### for the publication in order to be able to check the lfn length
158	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True) + '/${PSETHASH}/'
159	elif int(self.usenamespace) == 1:
160	if self.sched in ['CAF']: l_User=True
161	primaryDataset = self.computePrimaryDataset()
162	lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name) + '/${PSETHASH}/'
163	else:
164	if self.sched in ['CAF','LSF']: l_User=True
165	lfn = LFNBase(self.forced_path,self.user_remote_dir)
166	return lfn
167
168	def computePrimaryDataset(self):
169	"""
170	compute the last part for the LFN in case of publication
171	"""
172	if (self.datasetpath.upper() != 'NONE'):
173	primarydataset = self.datasetpath.split("/")[1]
174	else:
175	primarydataset = self.publish_data_name
176	return primarydataset
177
178	def lfn2pfn(self):
179	"""
180	PhEDEx Data Service lfn2pfn call
181
182	input: LFN,node name,protocol
183	returns: DOM object with the content of the PhEDEx Data Service call
184	"""
185	params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
186	params = urllib.urlencode(params)
187	datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
188	try:
189	urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
190	urlresults = parse(urlresults)
191	except IOError:
192	msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
193	raise CrabException(msg)
194	except:
195	urlresults = None
196
197	return urlresults
198
199	def parse_error(self,urlresults):
200	"""
201	look for errors in the DOM object returned by PhEDEx Data Service call
202	"""
203	errormsg = None
204	errors=urlresults.getElementsByTagName('error')
205	for error in errors:
206	errormsg=error.childNodes[0].data
207	if len(error.childNodes)>1:
208	errormsg+=error.childNodes[1].data
209	return errormsg
210
211	def parse_lfn2pfn(self,urlresults):
212	"""
213	Parse the content of the result of lfn2pfn PhEDEx Data Service call
214
215	input: DOM object with the content of the lfn2pfn call
216	returns: PFN
217	"""
218	result = urlresults.getElementsByTagName('phedex')
219
220	if not result:
221	return []
222	result = result[0]
223	pfn = None
224	mapping = result.getElementsByTagName('mapping')
225	for m in mapping:
226	pfn=m.getAttribute("pfn")
227	if pfn:
228	return pfn
229
230	def getStageoutPFN( self ):
231	"""
232	input: LFN,node name,protocol
233	returns: PFN
234	"""
235	if self.usePhedex:
236	fullurl="%s/lfn2pfn?node=%s&lfn=%s&protocol=%s"%(self.datasvc_url,self.node,self.lfn,self.protocol)
237	domlfn2pfn = self.lfn2pfn()
238	if not domlfn2pfn :
239	msg="Unable to get info from %s"%fullurl
240	raise CrabException(msg)
241
242	errormsg = self.parse_error(domlfn2pfn)
243	if errormsg:
244	msg="Error extracting info from %s due to: %s"%(fullurl,errormsg)
245	raise CrabException(msg)
246
247	stageoutpfn = self.parse_lfn2pfn(domlfn2pfn)
248	if not stageoutpfn:
249	msg ='Unable to get stageout path from TFC at Site %s \n'%self.node
250	msg+=' Please alert the CompInfraSup group through their savannah %s \n'%self.FacOps_savannah
251	msg+=' reporting: \n'
252	msg+=' Summary: Unable to get user stageout from TFC at Site %s \n'%self.node
253	msg+=' OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
254	raise CrabException(msg)
255	else:
256	if self.sched in ['CAF','LSF'] :
257	stageoutpfn = self.user_se_path+self.lfn
258	else:
259	stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
260
261	return stageoutpfn
262
263
264
265	if __name__ == '__main__':
266	"""
267	Sort of unit testing to check Phedex API for whatever site and/or lfn.
268	Usage:
269	python PhEDExDatasvcInfo.py --node T2_IT_Bari --lfn /store/maremma
270
271	"""
272	import getopt,sys
273	from crab_util import *
274	import common
275	klass_name = 'SchedulerGlite'
276	klass = importName(klass_name, klass_name)
277	common.scheduler = klass()
278
279	lfn="/store/user/"
280	node='T2_IT_Bari'
281	valid = ['node=','lfn=']
282	try:
283	opts, args = getopt.getopt(sys.argv[1:], "", valid)
284	except getopt.GetoptError, ex:
285	print str(ex)
286	sys.exit(1)
287	for o, a in opts:
288	if o == "--node":
289	node = a
290	if o == "--lfn":
291	lfn = a
292
293	mycfg_params = { 'USER.storage_element': node }
294	dsvc = PhEDExDatasvcInfo(mycfg_params)
295	dsvc.lfn = lfn
296	print dsvc.getStageoutPFN()
297