CRAB/python/DataLocation.py

#!/usr/bin/env python
import sys, os, string, re
import urllib, urllister
import urllib2
import common
from DLSInfo import *

# ####################################
class DataLocationError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass
                                                                                      
    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to extact data location
class DataLocation:
    def __init__(self, Listfileblocks, cfg_params):

#       Attributes
        self.Listfileblocks = Listfileblocks  # DLS input: list of fileblocks lists

        self.cfg_params = cfg_params

        self.SelectedSites = {}        # DLS output: list of sites hosting fileblocks
                                       #  retrieved using method getSites

        SEBlackList = []
        try:
            tmpBad = string.split(self.cfg_params['EDG.se_black_list'],',')
            for tmp in tmpBad:
                tmp=string.strip(tmp)
                SEBlackList.append(tmp)
        except KeyError:
            pass
        common.logger.debug(5,'SEBlackList: '+str(SEBlackList))
        self.reSEBlackList=[]
        for bad in SEBlackList:
            self.reSEBlackList.append(re.compile( string.lower(bad) ))

        SEWhiteList = []
        try:
            tmpGood = string.split(self.cfg_params['EDG.se_white_list'],',')
            for tmp in tmpGood:
                tmp=string.strip(tmp)
                SEWhiteList.append(tmp)
        except KeyError:
            pass
        common.logger.debug(5,'SEWhiteList: '+str(SEWhiteList))
        self.reSEWhiteList=[]
        for good in SEWhiteList:
            self.reSEWhiteList.append(re.compile( string.lower(good) ))

        self.osgSitesDictionary = {"Wisconsin":"cmssrm.hep.wisc.edu", \
                                   "Purdue":"dcache.rcac.purdue.edu", \
                                   "Florida":"ufdcache.phys.ufl.edu", \
                                   "Nebraska":"thpc-1.unl.edu", \
                                   "Caltech":"cithep59.ultralight.org", \
                                   "UCSD":"t2data2.t2.ucsd.edu", \
                                   "fnal":"cmssrm.fnal.gov", \
                                   "MIT":"se01.cmsaf.mit.edu"}

# #######################################################################
    def fetchDLSInfo(self):
        """
        Contact DLS
        """
        countblock=0
        Sites = []
        allblockSites = []

        try:
            dlstype=self.cfg_params['CMSSW.dls_type']
        except KeyError:
            dlstype='dli'
        #DLS_type="DLS_TYPE_MYSQL"
        DLS_type="DLS_TYPE_%s"%dlstype.upper()

        ## find the replicas for each block
        blockSites = {}
        failCount = 0
        for fileblocks in self.Listfileblocks:
            countblock=countblock+1
            #dbspath=string.split(afileblock,'#')[0]
            #(null,ds,tier,ow)=string.split(dbspath,'/')
            #datablock=ow+"/"+ds
            #
            dls=DLSInfo(DLS_type,self.cfg_params)
            try:
                replicas=dls.getReplicas(fileblocks)
                common.logger.debug(5,"sites are %s"%replicas)
                replicas = self.checkOSGt2(replicas)
                replicas = self.checkBlackList(replicas, fileblocks)
                if len(replicas)!=0:
                    replicas = self.checkWhiteList(replicas, fileblocks)
                if len(replicas)!=0:
                    blockSites[fileblocks] = replicas
            except DLSNoReplicas, ex:
                common.logger.debug(5,str(ex.getErrorMessage()))
                common.logger.debug(5,"Proceeding without this block.\n")
                failCount = failCount + 1
            except:
                raise DataLocationError('')

        if countblock == failCount:
            msg = "All data blocks encountered a DLS error.  Quitting."
            raise DataLocationError(msg)

        if len(blockSites)==0:
            msg = 'No sites remaining that host any part of the requested data! Exiting... '
            raise CrabException(msg)

        self.SelectedSites = blockSites

# #######################################################################
    def getSites(self):
        """
        get the sites hosting all the needed data 
        """
        return self.SelectedSites

# #######################################################################
    def checkBlackList(self, Sites, fileblocks):
        """
        select sites that are not excluded by the user (via SE black list)
        """
        goodSites = []
        for aSite in Sites:
            common.logger.debug(10,'Site '+aSite)
            good=1
            for re in self.reSEBlackList:
                if re.search(string.lower(aSite)):
                    common.logger.debug(5,'SE in black list, skipping site '+aSite)
                    good=0
                pass
            if good: goodSites.append(aSite)
        if len(goodSites) == 0:
            msg = "No sites hosting the block %s after BlackList" % fileblocks
            common.logger.debug(5,msg)
            common.logger.debug(5,"Proceeding without this block.\n")
        else:
            common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via BlackList are "+str(goodSites)+"\n")
        return goodSites

# #######################################################################
    def checkWhiteList(self, Sites, fileblocks):
        """
        select sites that are defined by the user (via SE white list)
        """
        if len(self.reSEWhiteList)==0: return Sites
        goodSites = []
        for aSite in Sites:
            good=0
            for re in self.reSEWhiteList:
                if re.search(string.lower(aSite)):
                    common.logger.debug(5,'SE in white list, adding site '+aSite)
                    good=1
                pass
            if good: goodSites.append(aSite)
        if len(goodSites) == 0:
            msg = "No sites hosting the block %s after WhiteList" % fileblocks
            common.logger.debug(5,msg)
            common.logger.debug(5,"Proceeding without this block.\n")
        else:
            common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via WhiteList are "+str(goodSites)+"\n")
        return goodSites 

# #######################################################################
    def checkOSGt2(self, sites):
        fixedSites = []
        osgKeys = self.osgSitesDictionary.keys()
        for site in sites:
            fix = 0
            for osgSite in osgKeys:
                if string.lower(site) == string.lower(osgSite):
                    fixedSites.append(self.osgSitesDictionary[osgSite])
                    fix = 1
            if (fix == 0):
                fixedSites.append(site)

        return fixedSites

#######################################################################
    def uniquelist(self, old):
        """
        remove duplicates from a list
        """
        nd={}
        for e in old:
            nd[e]=0
        return nd.keys()
Revision:	1.19
Committed:	Wed Oct 11 23:52:00 2006 UTC (18 years, 6 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_5_0_pre3, CRAB_1_5_0_pre2, CRAB_1_4_2, CRAB_1_5_0_pre1, CRAB_1_4_1, CRAB_1_4_1_pre2, CRAB_1_4_1_pre1, CRAB_1_4_0, CRAB_1_4_0_pre4, CRAB_1_4_0_pre3
Branch point for:	branch_1_4_1
Changes since 1.18:	+21 -21 lines
Log Message:	changed ce_ to se_ and added additional flags to crab.cfg
#	User	Rev	Content
1	gutsche	1.7	#!/usr/bin/env python
2	afanfani	1.1	import sys, os, string, re
3			import urllib, urllister
4			import urllib2
5			import common
6			from DLSInfo import *
7
8			# ####################################
9	afanfani	1.3	class DataLocationError(exceptions.Exception):
10	slacapra	1.5	def __init__(self, errorMessage):
11			args=errorMessage
12			exceptions.Exception.__init__(self, args)
13			pass
14	afanfani	1.3
15	slacapra	1.5	def getErrorMessage(self):
16			""" Return exception error """
17			return "%s" % (self.args)
18	afanfani	1.1
19			# ####################################
20			# class to extact data location
21			class DataLocation:
22			def __init__(self, Listfileblocks, cfg_params):
23
24			# Attributes
25			self.Listfileblocks = Listfileblocks # DLS input: list of fileblocks lists
26	slacapra	1.5
27	afanfani	1.1	self.cfg_params = cfg_params
28
29	slacapra	1.15	self.SelectedSites = {} # DLS output: list of sites hosting fileblocks
30			# retrieved using method getSites
31	afanfani	1.1
32	gutsche	1.19	SEBlackList = []
33	afanfani	1.1	try:
34	gutsche	1.19	tmpBad = string.split(self.cfg_params['EDG.se_black_list'],',')
35	afanfani	1.1	for tmp in tmpBad:
36			tmp=string.strip(tmp)
37	gutsche	1.19	SEBlackList.append(tmp)
38	afanfani	1.1	except KeyError:
39			pass
40	gutsche	1.19	common.logger.debug(5,'SEBlackList: '+str(SEBlackList))
41			self.reSEBlackList=[]
42			for bad in SEBlackList:
43			self.reSEBlackList.append(re.compile( string.lower(bad) ))
44	afanfani	1.1
45	gutsche	1.19	SEWhiteList = []
46	afanfani	1.4	try:
47	gutsche	1.19	tmpGood = string.split(self.cfg_params['EDG.se_white_list'],',')
48	afanfani	1.4	for tmp in tmpGood:
49			tmp=string.strip(tmp)
50	gutsche	1.19	SEWhiteList.append(tmp)
51	afanfani	1.4	except KeyError:
52			pass
53	gutsche	1.19	common.logger.debug(5,'SEWhiteList: '+str(SEWhiteList))
54			self.reSEWhiteList=[]
55			for good in SEWhiteList:
56			self.reSEWhiteList.append(re.compile( string.lower(good) ))
57	afanfani	1.4
58	slacapra	1.15	self.osgSitesDictionary = {"Wisconsin":"cmssrm.hep.wisc.edu", \
59			"Purdue":"dcache.rcac.purdue.edu", \
60			"Florida":"ufdcache.phys.ufl.edu", \
61			"Nebraska":"thpc-1.unl.edu", \
62			"Caltech":"cithep59.ultralight.org", \
63			"UCSD":"t2data2.t2.ucsd.edu", \
64			"fnal":"cmssrm.fnal.gov", \
65			"MIT":"se01.cmsaf.mit.edu"}
66	afanfani	1.4
67	afanfani	1.1	# #######################################################################
68			def fetchDLSInfo(self):
69			"""
70			Contact DLS
71			"""
72			countblock=0
73			Sites = []
74			allblockSites = []
75
76	afanfani	1.8	try:
77	slacapra	1.15	dlstype=self.cfg_params['CMSSW.dls_type']
78	afanfani	1.8	except KeyError:
79	gutsche	1.16	dlstype='dli'
80	slacapra	1.15	#DLS_type="DLS_TYPE_MYSQL"
81	afanfani	1.8	DLS_type="DLS_TYPE_%s"%dlstype.upper()
82	afanfani	1.6
83	mkirn	1.11	## find the replicas for each block
84	slacapra	1.15	blockSites = {}
85			failCount = 0
86	afanfani	1.1	for fileblocks in self.Listfileblocks:
87	slacapra	1.15	countblock=countblock+1
88			#dbspath=string.split(afileblock,'#')[0]
89			#(null,ds,tier,ow)=string.split(dbspath,'/')
90			#datablock=ow+"/"+ds
91			#
92			dls=DLSInfo(DLS_type,self.cfg_params)
93			try:
94			replicas=dls.getReplicas(fileblocks)
95			common.logger.debug(5,"sites are %s"%replicas)
96			replicas = self.checkOSGt2(replicas)
97			replicas = self.checkBlackList(replicas, fileblocks)
98			if len(replicas)!=0:
99			replicas = self.checkWhiteList(replicas, fileblocks)
100			if len(replicas)!=0:
101			blockSites[fileblocks] = replicas
102			except DLSNoReplicas, ex:
103	gutsche	1.17	common.logger.debug(5,str(ex.getErrorMessage()))
104			common.logger.debug(5,"Proceeding without this block.\n")
105	slacapra	1.15	failCount = failCount + 1
106			except:
107			raise DataLocationError('')
108
109			if countblock == failCount:
110			msg = "All data blocks encountered a DLS error. Quitting."
111			raise DataLocationError(msg)
112
113			if len(blockSites)==0:
114			msg = 'No sites remaining that host any part of the requested data! Exiting... '
115			raise CrabException(msg)
116
117			self.SelectedSites = blockSites
118	afanfani	1.1
119			# #######################################################################
120			def getSites(self):
121			"""
122	slacapra	1.5	get the sites hosting all the needed data
123	afanfani	1.1	"""
124			return self.SelectedSites
125
126			# #######################################################################
127	slacapra	1.15	def checkBlackList(self, Sites, fileblocks):
128	afanfani	1.1	"""
129	gutsche	1.19	select sites that are not excluded by the user (via SE black list)
130	afanfani	1.1	"""
131			goodSites = []
132			for aSite in Sites:
133	slacapra	1.15	common.logger.debug(10,'Site '+aSite)
134	afanfani	1.1	good=1
135	gutsche	1.19	for re in self.reSEBlackList:
136	slacapra	1.15	if re.search(string.lower(aSite)):
137	gutsche	1.19	common.logger.debug(5,'SE in black list, skipping site '+aSite)
138	afanfani	1.1	good=0
139			pass
140			if good: goodSites.append(aSite)
141			if len(goodSites) == 0:
142	slacapra	1.15	msg = "No sites hosting the block %s after BlackList" % fileblocks
143	gutsche	1.17	common.logger.debug(5,msg)
144			common.logger.debug(5,"Proceeding without this block.\n")
145	slacapra	1.15	else:
146			common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via BlackList are "+str(goodSites)+"\n")
147	afanfani	1.1	return goodSites
148
149	afanfani	1.4	# #######################################################################
150	slacapra	1.15	def checkWhiteList(self, Sites, fileblocks):
151	afanfani	1.4	"""
152	gutsche	1.19	select sites that are defined by the user (via SE white list)
153	afanfani	1.4	"""
154	gutsche	1.19	if len(self.reSEWhiteList)==0: return Sites
155	afanfani	1.4	goodSites = []
156			for aSite in Sites:
157			good=0
158	gutsche	1.19	for re in self.reSEWhiteList:
159	slacapra	1.15	if re.search(string.lower(aSite)):
160	gutsche	1.19	common.logger.debug(5,'SE in white list, adding site '+aSite)
161	afanfani	1.4	good=1
162			pass
163			if good: goodSites.append(aSite)
164			if len(goodSites) == 0:
165	slacapra	1.15	msg = "No sites hosting the block %s after WhiteList" % fileblocks
166	gutsche	1.17	common.logger.debug(5,msg)
167			common.logger.debug(5,"Proceeding without this block.\n")
168	slacapra	1.15	else:
169			common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via WhiteList are "+str(goodSites)+"\n")
170	afanfani	1.4	return goodSites
171
172	slacapra	1.15	# #######################################################################
173			def checkOSGt2(self, sites):
174			fixedSites = []
175			osgKeys = self.osgSitesDictionary.keys()
176			for site in sites:
177			fix = 0
178			for osgSite in osgKeys:
179			if string.lower(site) == string.lower(osgSite):
180			fixedSites.append(self.osgSitesDictionary[osgSite])
181			fix = 1
182			if (fix == 0):
183			fixedSites.append(site)
184
185			return fixedSites
186
187	afanfani	1.1	#######################################################################
188			def uniquelist(self, old):
189			"""
190	slacapra	1.5	remove duplicates from a list
191	afanfani	1.1	"""
192			nd={}
193			for e in old:
194			nd[e]=0
195			return nd.keys()