CRAB/python/DataLocation.py

#!/usr/bin/env python
import sys, os, string, re
import urllib, urllister
import urllib2
import common
from DLSInfo import *

# ####################################
class DataLocationError(exceptions.Exception):
    def __init__(self, errorMessage):
        args=errorMessage
        exceptions.Exception.__init__(self, args)
        pass
                                                                                      
    def getErrorMessage(self):
        """ Return exception error """
        return "%s" % (self.args)

# ####################################
# class to extact data location
class DataLocation:
    def __init__(self, Listfileblocks, cfg_params):

#       Attributes
        self.Listfileblocks = Listfileblocks  # DLS input: list of fileblocks lists

        self.cfg_params = cfg_params

        self.SelectedSites = []        # DLS output: list of sites hosting all the needed fileblocks
                                       #  retireved using method getSites

        CEBlackList = []
        try:
            tmpBad = string.split(self.cfg_params['EDG.ce_black_list'],',')
            for tmp in tmpBad:
                tmp=string.strip(tmp)
                CEBlackList.append(tmp)
        except KeyError:
            pass
        common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
        self.reCEBlackList=[]
        for bad in CEBlackList:
            self.reCEBlackList.append(re.compile( bad ))

        CEWhiteList = []
        try:
            tmpGood = string.split(self.cfg_params['EDG.ce_white_list'],',')
            for tmp in tmpGood:
                tmp=string.strip(tmp)
                CEWhiteList.append(tmp)
        except KeyError:
            pass
        common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
        self.reCEWhiteList=[]
        for good in CEWhiteList:
            self.reCEWhiteList.append(re.compile( good ))


# #######################################################################
    def fetchDLSInfo(self):
        """
        Contact DLS
        """
        countblock=0
        Sites = []
        allblockSites = []


        try:
            dlstype=self.cfg_params['ORCA.dls_type']
        except KeyError:
            ## default to DLS-DLI:
            dlstype='dli' 

        DLS_type="DLS_TYPE_%s"%dlstype.upper()

        ## find the replicas for each block
        tryCount = 0
        failCount = 0
        for fileblocks in self.Listfileblocks:
            for afileblock in fileblocks:
                countblock=countblock+1
                dbspath=string.split(afileblock,'#')[0]
                (null,ds,tier,ow)=string.split(dbspath,'/')
                datablock=ow+"/"+ds
                #
                dls=DLSInfo(DLS_type,self.cfg_params['CRAB.jobtype'])
                try:
                    replicas=dls.getReplicas(datablock)
                    tryCount = tryCount + 1
                except DLSNoReplicas, ex:
                    #raise DataLocationError(ex.getErrorMessage())
                    msg = "%s      -Proceeding without this file block." % str(ex.getErrorMessage())
                    common.logger.message(msg)
                    tryCount = tryCount + 1
                    failCount = failCount + 1
                except:
                    raise DataLocationError('')

                for replica in replicas:
                    Sites.append(replica)

        if tryCount == failCount:
            msg = "All data blocks encountered a DLS error.  Quitting."
            raise DataLocationError(msg)

        ## select only sites that contains _all_ the fileblocks
        allblockSites=self.SelectSites(countblock,Sites)
        #for as in allblockSites:
        #   print " site is "+as
        ## select sites that are not in a BlackList , if any
        self.SelectedNotBlackSites=self.checkBlackList(allblockSites)
        ## select sites there are in the white list , if any
        self.SelectedSites=self.checkWhiteList(self.SelectedNotBlackSites)

# #######################################################################
    def getSites(self):
        """
        get the sites hosting all the needed data 
        """
        return self.SelectedSites

# #######################################################################
    def SelectSites(self, countblock, Sites ):
        """
        select only sites that contains _all_ the fileblocks
        """ 
        goodSites=[]
        for aSite in Sites :
            if ( Sites.count(aSite)==countblock ) :
                goodSites.append(aSite)

        return self.uniquelist(goodSites)

# #######################################################################
    def checkBlackList(self, Sites):
        """
        select sites that are not excluded by the user (via CE black list)
        """
        goodSites = []
        for aSite in Sites:
            good=1
            for re in self.reCEBlackList:
                if re.search(aSite):
                    common.logger.message('CE in black list, skipping site '+aSite)
                    good=0
                pass
            if good: goodSites.append(aSite)
        if len(goodSites) == 0:
            common.logger.debug(3,"No selected Sites")
        return goodSites

# #######################################################################
    def checkWhiteList(self, Sites):
        """
        select sites that are defined by the user (via CE white list)
        """
        if len(self.reCEWhiteList)==0: return Sites
        goodSites = []
        for aSite in Sites:
            good=0
            for re in self.reCEWhiteList:
                if re.search(aSite):
                    common.logger.message('CE in white list, adding site '+aSite)
                    good=1
                pass
            if good: goodSites.append(aSite)
        if len(goodSites) == 0:
            common.logger.debug(3,"No selected Sites")
        return goodSites 

#######################################################################
    def uniquelist(self, old):
        """
        remove duplicates from a list
        """
        nd={}
        for e in old:
            nd[e]=0
        return nd.keys()

Revision:	1.11
Committed:	Wed Jul 26 18:51:25 2006 UTC (18 years, 9 months ago) by mkirn
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_2_1
Changes since 1.10:	+13 -2 lines
Log Message:	Raise an exception only if all blocks in the dataset are not in DLS
#	User	Rev	Content
1	gutsche	1.7	#!/usr/bin/env python
2	afanfani	1.1	import sys, os, string, re
3			import urllib, urllister
4			import urllib2
5			import common
6			from DLSInfo import *
7
8			# ####################################
9	afanfani	1.3	class DataLocationError(exceptions.Exception):
10	slacapra	1.5	def __init__(self, errorMessage):
11			args=errorMessage
12			exceptions.Exception.__init__(self, args)
13			pass
14	afanfani	1.3
15	slacapra	1.5	def getErrorMessage(self):
16			""" Return exception error """
17			return "%s" % (self.args)
18	afanfani	1.1
19			# ####################################
20			# class to extact data location
21			class DataLocation:
22			def __init__(self, Listfileblocks, cfg_params):
23
24			# Attributes
25			self.Listfileblocks = Listfileblocks # DLS input: list of fileblocks lists
26	slacapra	1.5
27	afanfani	1.1	self.cfg_params = cfg_params
28
29			self.SelectedSites = [] # DLS output: list of sites hosting all the needed fileblocks
30			# retireved using method getSites
31
32			CEBlackList = []
33			try:
34			tmpBad = string.split(self.cfg_params['EDG.ce_black_list'],',')
35			for tmp in tmpBad:
36			tmp=string.strip(tmp)
37			CEBlackList.append(tmp)
38			except KeyError:
39			pass
40			common.logger.debug(5,'CEBlackList: '+str(CEBlackList))
41			self.reCEBlackList=[]
42			for bad in CEBlackList:
43			self.reCEBlackList.append(re.compile( bad ))
44
45	afanfani	1.4	CEWhiteList = []
46			try:
47			tmpGood = string.split(self.cfg_params['EDG.ce_white_list'],',')
48			for tmp in tmpGood:
49			tmp=string.strip(tmp)
50			CEWhiteList.append(tmp)
51			except KeyError:
52			pass
53			common.logger.debug(5,'CEWhiteList: '+str(CEWhiteList))
54			self.reCEWhiteList=[]
55			for good in CEWhiteList:
56			self.reCEWhiteList.append(re.compile( good ))
57
58
59	afanfani	1.1	# #######################################################################
60			def fetchDLSInfo(self):
61			"""
62			Contact DLS
63			"""
64			countblock=0
65			Sites = []
66			allblockSites = []
67
68	afanfani	1.9
69	afanfani	1.8	try:
70			dlstype=self.cfg_params['ORCA.dls_type']
71			except KeyError:
72	afanfani	1.10	## default to DLS-DLI:
73			dlstype='dli'
74
75	afanfani	1.8	DLS_type="DLS_TYPE_%s"%dlstype.upper()
76	afanfani	1.6
77	mkirn	1.11	## find the replicas for each block
78			tryCount = 0
79			failCount = 0
80	afanfani	1.1	for fileblocks in self.Listfileblocks:
81	slacapra	1.5	for afileblock in fileblocks:
82			countblock=countblock+1
83			dbspath=string.split(afileblock,'#')[0]
84			(null,ds,tier,ow)=string.split(dbspath,'/')
85			datablock=ow+"/"+ds
86			#
87	afanfani	1.9	dls=DLSInfo(DLS_type,self.cfg_params['CRAB.jobtype'])
88	slacapra	1.5	try:
89	afanfani	1.6	replicas=dls.getReplicas(datablock)
90	mkirn	1.11	tryCount = tryCount + 1
91	slacapra	1.5	except DLSNoReplicas, ex:
92	mkirn	1.11	#raise DataLocationError(ex.getErrorMessage())
93			msg = "%s -Proceeding without this file block." % str(ex.getErrorMessage())
94			common.logger.message(msg)
95			tryCount = tryCount + 1
96			failCount = failCount + 1
97	slacapra	1.5	except:
98			raise DataLocationError('')
99	afanfani	1.1
100	slacapra	1.5	for replica in replicas:
101			Sites.append(replica)
102	afanfani	1.1
103	mkirn	1.11	if tryCount == failCount:
104			msg = "All data blocks encountered a DLS error. Quitting."
105			raise DataLocationError(msg)
106
107	afanfani	1.1	## select only sites that contains _all_ the fileblocks
108			allblockSites=self.SelectSites(countblock,Sites)
109			#for as in allblockSites:
110			# print " site is "+as
111	afanfani	1.4	## select sites that are not in a BlackList , if any
112			self.SelectedNotBlackSites=self.checkBlackList(allblockSites)
113			## select sites there are in the white list , if any
114			self.SelectedSites=self.checkWhiteList(self.SelectedNotBlackSites)
115	afanfani	1.1
116			# #######################################################################
117			def getSites(self):
118			"""
119	slacapra	1.5	get the sites hosting all the needed data
120	afanfani	1.1	"""
121			return self.SelectedSites
122
123			# #######################################################################
124			def SelectSites(self, countblock, Sites ):
125			"""
126	slacapra	1.5	select only sites that contains _all_ the fileblocks
127	afanfani	1.1	"""
128			goodSites=[]
129			for aSite in Sites :
130	slacapra	1.5	if ( Sites.count(aSite)==countblock ) :
131			goodSites.append(aSite)
132	afanfani	1.1
133			return self.uniquelist(goodSites)
134
135			# #######################################################################
136			def checkBlackList(self, Sites):
137			"""
138	afanfani	1.4	select sites that are not excluded by the user (via CE black list)
139	afanfani	1.1	"""
140			goodSites = []
141			for aSite in Sites:
142			good=1
143			for re in self.reCEBlackList:
144			if re.search(aSite):
145			common.logger.message('CE in black list, skipping site '+aSite)
146			good=0
147			pass
148			if good: goodSites.append(aSite)
149			if len(goodSites) == 0:
150			common.logger.debug(3,"No selected Sites")
151			return goodSites
152
153	afanfani	1.4	# #######################################################################
154			def checkWhiteList(self, Sites):
155			"""
156			select sites that are defined by the user (via CE white list)
157			"""
158			if len(self.reCEWhiteList)==0: return Sites
159			goodSites = []
160			for aSite in Sites:
161			good=0
162			for re in self.reCEWhiteList:
163			if re.search(aSite):
164			common.logger.message('CE in white list, adding site '+aSite)
165			good=1
166			pass
167			if good: goodSites.append(aSite)
168			if len(goodSites) == 0:
169			common.logger.debug(3,"No selected Sites")
170			return goodSites
171
172	afanfani	1.1	#######################################################################
173			def uniquelist(self, old):
174			"""
175	slacapra	1.5	remove duplicates from a list
176	afanfani	1.1	"""
177			nd={}
178			for e in old:
179			nd[e]=0
180			return nd.keys()
181	slacapra	1.5