CRAB/python/BlackWhiteListParser.py

#!/usr/bin/env python
"""
_BlackWhiteListParser_

Parsing for black and white lists, both SE and CE

Large parts of the July 2008 re-write come from Brian Bockelman

"""

__revision__ = "$Id: BlackWhiteListParser.py,v 1.6 2008/07/09 19:55:56 ewv Exp $"
__version__ = "$Revision: 1.6 $"


import os
import sys
import sets
import time
import types
import fnmatch

from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
import common
from ProdCommon.SiteDB.SiteDB import SiteDBJSON

class BlackWhiteListParser(object):

    """
    A class which applies blacklist and whitelist; designed to allow the user
    to filter out sites.  Allows users to specify only the CMS name from SiteDB
    (and simple wildcards), but internally filters only on the CE/SE name.
    """

    def __init__(self, cfg_params):
        self.kind = 'se'
        self.mapper = None # Defined by Super class
        self.siteDBAPI = SiteDBJSON()

    def configure(self, cfg_params):
        """
        Load up the black and white list from the configuation parameters
           * EDG.%s_black_list
           * EDG.%s_white_list
        and expand things that SiteDB knows the CMS names for
        """ % (self.kind, self.kind)

        self.blacklist = []
        if cfg_params.has_key('EDG.%s_black_list' % self.kind):
            user_input = cfg_params['EDG.%s_black_list' % self.kind]
            self.blacklist = self.expandList(user_input)
        common.logger.debug(5,'Converted %s blacklist: %s' % (self.kind, ', '.join(self.blacklist)))

        self.whitelist = []
        if cfg_params.has_key('EDG.%s_white_list' % self.kind):
            user_input = cfg_params['EDG.%s_white_list' % self.kind]
            self.whitelist = self.expandList(user_input)
        common.logger.debug(5, 'Converted %s whitelist: %s' % (self.kind, ', '.join(self.whitelist)))

        self.blacklist = sets.Set(self.blacklist)
        self.whitelist = sets.Set(self.whitelist)
        #print "User's %s blacklist: %s" % (self.kind,self.blacklist)
        #print "User's %s whitelist: %s" % (self.kind,self.whitelist)

    def expandList(self, userInput):
        """
        Contact SiteDB to expand lists like T2_US into lists of
        actual SE names and CE names.
        """

        userList = userInput.split(',')
        expandedList = []
        hadErrors = False
        for item in userList:
            item = item.strip()
            try:
                expandedItem = self.mapper(item)
            except Exception: # FIXME: WMCore SiteDB re-throws particular exception
                expandedItem = None
                hadErrors = True

            if expandedItem:
                expandedList.extend(expandedItem)
            else:
                expandedList.append(item)

        if hadErrors:
            common.logger.message("Problem connecting to SiteDB. " \
                                + "%s " % self.kind.upper() \
                                + "white/blacklist may be incomplete.")
            common.logger.message("List is %s" % expandedList)

        return expandedList

    def checkBlackList(self, Sites, fileblocks=''):
        """
        Select sites that are not excluded by the user (via blacklist)

        The sites returned are the input sites minus the contents of the
        self.blacklist

        @param Sites: The sites which will be filtered
        @keyword fileblocks: The block this is used for; only used in a pretty
           debug message.
        @returns: The input sites minus the blacklist.
        """
        Sites = sets.Set(Sites)
        #print "Sites:",Sites
        blacklist = self.blacklist
        blacklist = sets.Set(self.match_list(Sites, self.blacklist))
        #print "Black list:",blacklist
        goodSites = Sites.difference(blacklist)
        #print "Good Sites:",goodSites,"\n"
        goodSites = list(goodSites)
        if not goodSites and fileblocks:
            msg = "No sites hosting the block %s after blackList" % fileblocks
            common.logger.debug(5, msg)
            common.logger.debug(5, "Proceeding without this block.\n")
        elif fileblocks:
            common.logger.debug(5, "Selected sites for block %s via blacklist " \
                "are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
        return goodSites

    def checkWhiteList(self, Sites, fileblocks=''):
        """
        Select sites that are defined by the user (via white list).

        The sites returned are the intersection of the input sites and the
        contents of self.whitelist

        @param Sites: The sites which will be filtered
        @keyword fileblocks: The block this is applied for; only used for a
           pretty debug message
        @returns: The intersection of the input Sites and self.whitelist.
        """
        if not self.whitelist:
            return Sites
        whitelist = self.whitelist
        whitelist = self.match_list(Sites, self.whitelist)
        #print "White list:",whitelist
        Sites = sets.Set(Sites)
        goodSites = Sites.intersection(whitelist)
        #print "Good Sites:",goodSites,"\n"
        goodSites = list(goodSites)
        if not goodSites and fileblocks:
            msg = "No sites hosting the block %s after whiteList" % fileblocks
            common.logger.debug(5, msg)
            common.logger.debug(5, "Proceeding without this block.\n")
        elif fileblocks:
            common.logger.debug(5, "Selected sites for block %s via whitelist "\
                " are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))

        return goodSites

    def cleanForBlackWhiteList(self, destinations, list=False):
        """
        Clean for black/white lists using parser.

        Take the input list and apply the blacklist, then the whitelist that
        the user specified.

        @param destinations: A list of all the input sites
        @keyword list: Set to True or the string 'list' to return a list
           object.  Set to False or the string '' to return a string object.
           The default is False.
        @returns: The list of all input sites, first filtered by the blacklist,
           then filtered by the whitelist.  If list=True, returns a list; if
           list=False, return a string.
        """
        if list:
            return self.checkWhiteList(self.checkBlackList(destinations))
        else:
            return ','.join(self.checkWhiteList(self.checkBlackList( \
                destinations)))


    def match_list(self, names, match_list):
        """
        Filter a list of names against a comma-separated list of expressions.

        This uses the `match` function to do the heavy lifting

        @param names: A list of input names to filter
        @type names: list
        @param match_list: A comma-separated list of expressions
        @type match_list: str
        @returns: A list, filtered from `names`, of all entries which match an
          expression in match_list
        @rtype: list
        """
        results = []
        if isinstance(match_list, types.StringType):
            match_list = match_list.split(',')

        for expr in match_list:
            expr = expr.strip()
            matching = self.match(names, expr)
            if matching:
                results.extend(matching)
            else:
                results.append(expr)
        return results


    def match(self, names, expr):
        """
        Return all the entries in `names` which match `expr`

        First, try to apply wildcard-based filters, then look at substrings,
        then interpret expr as a regex.

        @param names: An input list of strings to match
        @param expr: A string expression to use for matching
        @returns: All entries in the list `names` which match `expr`
        """

        results = fnmatch.filter(names, expr)
        results.extend([i for i in names if i.find(expr) >= 0])
        try:
            my_re = re.compile(expr)
        except:
            my_re = None
        if not my_re:
            return results
        results.extend([i for i in names if my_re.search(i)])
        return results


class SEBlackWhiteListParser(BlackWhiteListParser):
    """
    Use the BlackWhiteListParser to filter out the possible list of SEs
    from the user's input; see the documentation for BlackWhiteListParser.
    """

    def __init__(self, cfg_params):
        super(SEBlackWhiteListParser, self).__init__(cfg_params)
        self.kind = 'se'
        self.mapper = self.siteDBAPI.CMSNametoSE
        self.configure(cfg_params)


class CEBlackWhiteListParser(BlackWhiteListParser):
    """
    Use the BlackWhiteListParser to filter out the possible list of CEs
    from the user's input; see the documentation for BlackWhiteListParser.
    """

    def __init__(self, cfg_params):
        super(CEBlackWhiteListParser, self).__init__(cfg_params)
        self.kind = 'ce'
        self.mapper = self.siteDBAPI.CMSNametoCE
        self.configure(cfg_params)
Revision:	1.7
Committed:	Thu Sep 11 14:44:02 2008 UTC (16 years, 7 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_4_1_pre4, CRAB_2_4_1_pre3, CRAB_2_4_1_pre2, CRAB_2_4_1_pre1, CRAB_2_4_0_Tutorial, CRAB_2_4_0_Tutorial_pre1, CRAB_2_4_0, CRAB_2_4_0_pre9, CRAB_2_4_0_pre8, CRAB_2_4_0_pre7, CRAB_2_4_0_pre6, CRAB_2_4_0_pre5, CRAB_2_4_0_pre4, CRAB_2_4_0_pre3, CRAB_2_4_0_pre2, CRAB_2_4_0_pre1
Changes since 1.6:	+38 -20 lines
Log Message:	Trap SiteDB errors, warn user
#	User	Rev	Content
1	ewv	1.6	#!/usr/bin/env python
2			"""
3			_BlackWhiteListParser_
4
5			Parsing for black and white lists, both SE and CE
6
7			Large parts of the July 2008 re-write come from Brian Bockelman
8
9			"""
10
11	ewv	1.7	__revision__ = "$Id: BlackWhiteListParser.py,v 1.6 2008/07/09 19:55:56 ewv Exp $"
12			__version__ = "$Revision: 1.6 $"
13	ewv	1.6
14
15			import os
16			import sys
17			import sets
18			import time
19			import types
20			import fnmatch
21
22	gutsche	1.1	from crab_logger import Logger
23			from crab_exceptions import *
24			from crab_util import *
25			import common
26	ewv	1.6	from ProdCommon.SiteDB.SiteDB import SiteDBJSON
27
28			class BlackWhiteListParser(object):
29	gutsche	1.1
30	ewv	1.6	"""
31			A class which applies blacklist and whitelist; designed to allow the user
32			to filter out sites. Allows users to specify only the CMS name from SiteDB
33			(and simple wildcards), but internally filters only on the CE/SE name.
34			"""
35
36			def __init__(self, cfg_params):
37			self.kind = 'se'
38	ewv	1.7	self.mapper = None # Defined by Super class
39	ewv	1.6	self.siteDBAPI = SiteDBJSON()
40	gutsche	1.1
41	ewv	1.6	def configure(self, cfg_params):
42			"""
43			Load up the black and white list from the configuation parameters
44			* EDG.%s_black_list
45			* EDG.%s_white_list
46			and expand things that SiteDB knows the CMS names for
47			""" % (self.kind, self.kind)
48
49			self.blacklist = []
50			if cfg_params.has_key('EDG.%s_black_list' % self.kind):
51			user_input = cfg_params['EDG.%s_black_list' % self.kind]
52			self.blacklist = self.expandList(user_input)
53			common.logger.debug(5,'Converted %s blacklist: %s' % (self.kind, ', '.join(self.blacklist)))
54
55			self.whitelist = []
56			if cfg_params.has_key('EDG.%s_white_list' % self.kind):
57			user_input = cfg_params['EDG.%s_white_list' % self.kind]
58			self.whitelist = self.expandList(user_input)
59			common.logger.debug(5, 'Converted %s whitelist: %s' % (self.kind, ', '.join(self.whitelist)))
60
61			self.blacklist = sets.Set(self.blacklist)
62			self.whitelist = sets.Set(self.whitelist)
63			#print "User's %s blacklist: %s" % (self.kind,self.blacklist)
64			#print "User's %s whitelist: %s" % (self.kind,self.whitelist)
65
66			def expandList(self, userInput):
67	ewv	1.7	"""
68			Contact SiteDB to expand lists like T2_US into lists of
69			actual SE names and CE names.
70			"""
71
72			userList = userInput.split(',')
73			expandedList = []
74			hadErrors = False
75			for item in userList:
76			item = item.strip()
77			try:
78			expandedItem = self.mapper(item)
79			except Exception: # FIXME: WMCore SiteDB re-throws particular exception
80			expandedItem = None
81			hadErrors = True
82
83			if expandedItem:
84			expandedList.extend(expandedItem)
85			else:
86			expandedList.append(item)
87
88			if hadErrors:
89			common.logger.message("Problem connecting to SiteDB. " \
90			+ "%s " % self.kind.upper() \
91			+ "white/blacklist may be incomplete.")
92			common.logger.message("List is %s" % expandedList)
93	ewv	1.5
94	ewv	1.7	return expandedList
95	gutsche	1.1
96	ewv	1.6	def checkBlackList(self, Sites, fileblocks=''):
97			"""
98			Select sites that are not excluded by the user (via blacklist)
99	gutsche	1.1
100	ewv	1.6	The sites returned are the input sites minus the contents of the
101			self.blacklist
102
103			@param Sites: The sites which will be filtered
104			@keyword fileblocks: The block this is used for; only used in a pretty
105			debug message.
106			@returns: The input sites minus the blacklist.
107			"""
108			Sites = sets.Set(Sites)
109			#print "Sites:",Sites
110			blacklist = self.blacklist
111			blacklist = sets.Set(self.match_list(Sites, self.blacklist))
112			#print "Black list:",blacklist
113			goodSites = Sites.difference(blacklist)
114			#print "Good Sites:",goodSites,"\n"
115			goodSites = list(goodSites)
116			if not goodSites and fileblocks:
117			msg = "No sites hosting the block %s after blackList" % fileblocks
118	ewv	1.7	common.logger.debug(5, msg)
119			common.logger.debug(5, "Proceeding without this block.\n")
120	ewv	1.6	elif fileblocks:
121	ewv	1.7	common.logger.debug(5, "Selected sites for block %s via blacklist " \
122	ewv	1.6	"are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
123	gutsche	1.1	return goodSites
124
125	ewv	1.6	def checkWhiteList(self, Sites, fileblocks=''):
126	gutsche	1.1	"""
127	ewv	1.6	Select sites that are defined by the user (via white list).
128
129			The sites returned are the intersection of the input sites and the
130			contents of self.whitelist
131	ewv	1.5
132	ewv	1.6	@param Sites: The sites which will be filtered
133			@keyword fileblocks: The block this is applied for; only used for a
134			pretty debug message
135			@returns: The intersection of the input Sites and self.whitelist.
136			"""
137			if not self.whitelist:
138			return Sites
139			whitelist = self.whitelist
140			whitelist = self.match_list(Sites, self.whitelist)
141			#print "White list:",whitelist
142			Sites = sets.Set(Sites)
143			goodSites = Sites.intersection(whitelist)
144			#print "Good Sites:",goodSites,"\n"
145			goodSites = list(goodSites)
146			if not goodSites and fileblocks:
147			msg = "No sites hosting the block %s after whiteList" % fileblocks
148	ewv	1.7	common.logger.debug(5, msg)
149			common.logger.debug(5, "Proceeding without this block.\n")
150	ewv	1.6	elif fileblocks:
151	ewv	1.7	common.logger.debug(5, "Selected sites for block %s via whitelist "\
152	ewv	1.6	" are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
153	ewv	1.5
154			return goodSites
155
156	ewv	1.7	def cleanForBlackWhiteList(self, destinations, list=False):
157	gutsche	1.2	"""
158	ewv	1.6	Clean for black/white lists using parser.
159
160			Take the input list and apply the blacklist, then the whitelist that
161			the user specified.
162
163			@param destinations: A list of all the input sites
164			@keyword list: Set to True or the string 'list' to return a list
165			object. Set to False or the string '' to return a string object.
166			The default is False.
167			@returns: The list of all input sites, first filtered by the blacklist,
168			then filtered by the whitelist. If list=True, returns a list; if
169			list=False, return a string.
170	gutsche	1.2	"""
171	ewv	1.6	if list:
172			return self.checkWhiteList(self.checkBlackList(destinations))
173	spiga	1.3	else:
174	ewv	1.6	return ','.join(self.checkWhiteList(self.checkBlackList( \
175			destinations)))
176
177
178			def match_list(self, names, match_list):
179			"""
180			Filter a list of names against a comma-separated list of expressions.
181
182			This uses the `match` function to do the heavy lifting
183
184			@param names: A list of input names to filter
185			@type names: list
186			@param match_list: A comma-separated list of expressions
187			@type match_list: str
188			@returns: A list, filtered from `names`, of all entries which match an
189			expression in match_list
190			@rtype: list
191			"""
192			results = []
193			if isinstance(match_list, types.StringType):
194			match_list = match_list.split(',')
195
196			for expr in match_list:
197			expr = expr.strip()
198			matching = self.match(names, expr)
199			if matching:
200			results.extend(matching)
201			else:
202			results.append(expr)
203			return results
204
205
206			def match(self, names, expr):
207			"""
208			Return all the entries in `names` which match `expr`
209
210			First, try to apply wildcard-based filters, then look at substrings,
211			then interpret expr as a regex.
212
213			@param names: An input list of strings to match
214			@param expr: A string expression to use for matching
215			@returns: All entries in the list `names` which match `expr`
216			"""
217
218			results = fnmatch.filter(names, expr)
219			results.extend([i for i in names if i.find(expr) >= 0])
220			try:
221			my_re = re.compile(expr)
222			except:
223			my_re = None
224			if not my_re:
225			return results
226			results.extend([i for i in names if my_re.search(i)])
227			return results
228
229
230
231			class SEBlackWhiteListParser(BlackWhiteListParser):
232			"""
233			Use the BlackWhiteListParser to filter out the possible list of SEs
234			from the user's input; see the documentation for BlackWhiteListParser.
235			"""
236
237			def __init__(self, cfg_params):
238			super(SEBlackWhiteListParser, self).__init__(cfg_params)
239			self.kind = 'se'
240			self.mapper = self.siteDBAPI.CMSNametoSE
241			self.configure(cfg_params)
242
243
244	ewv	1.5
245			class CEBlackWhiteListParser(BlackWhiteListParser):
246	ewv	1.6	"""
247			Use the BlackWhiteListParser to filter out the possible list of CEs
248			from the user's input; see the documentation for BlackWhiteListParser.
249			"""
250
251	ewv	1.7	def __init__(self, cfg_params):
252	ewv	1.6	super(CEBlackWhiteListParser, self).__init__(cfg_params)
253			self.kind = 'ce'
254			self.mapper = self.siteDBAPI.CMSNametoCE
255			self.configure(cfg_params)