CRAB/python/BlackWhiteListParser.py

#!/usr/bin/env python
"""
_BlackWhiteListParser_

Parsing for black and white lists, both SE and CE

Large parts of the July 2008 re-write come from Brian Bockelman

"""

__revision__ = "$Id: BlackWhiteListParser.py,v 1.7 2008/09/11 14:44:02 ewv Exp $"
__version__ = "$Revision: 1.7 $"


import os
import sys
import sets
import time
import types
import fnmatch

from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
import common
from ProdCommon.SiteDB.SiteDB import SiteDBJSON

class BlackWhiteListParser(object):

    """
    A class which applies blacklist and whitelist; designed to allow the user
    to filter out sites.  Allows users to specify only the CMS name from SiteDB
    (and simple wildcards), but internally filters only on the CE/SE name.
    """

    def __init__(self, cfg_params):
        self.kind = 'se'
        self.mapper = None # Defined by Super class
        self.siteDBAPI = SiteDBJSON()

    def configure(self, cfg_params):
        """
        Load up the black and white list from the configuation parameters
           * EDG.%s_black_list
           * EDG.%s_white_list
        and expand things that SiteDB knows the CMS names for
        """ % (self.kind, self.kind)

        self.blacklist = []
        if cfg_params.has_key('EDG.%s_black_list' % self.kind):
            user_input = cfg_params['EDG.%s_black_list' % self.kind]
            self.blacklist = self.expandList(user_input)
        common.logger.debug(5,'Converted %s blacklist: %s' % (self.kind, ', '.join(self.blacklist)))

        self.whitelist = []
        if cfg_params.has_key('EDG.%s_white_list' % self.kind):
            user_input = cfg_params['EDG.%s_white_list' % self.kind]
            self.whitelist = self.expandList(user_input)
        common.logger.debug(5, 'Converted %s whitelist: %s' % (self.kind, ', '.join(self.whitelist)))

        self.blacklist = sets.Set(self.blacklist)
        self.whitelist = sets.Set(self.whitelist)
        #print "User's %s blacklist: %s" % (self.kind,self.blacklist)
        #print "User's %s whitelist: %s" % (self.kind,self.whitelist)

    def expandList(self, userInput):
        """
        Contact SiteDB to expand lists like T2_US into lists of
        actual SE names and CE names.
        """

        userList = userInput.split(',')
        expandedList = []
        hadErrors = False
        for item in userList:
            item = item.strip()
            try:
                expandedItem = self.mapper(item)
            except Exception: # FIXME: WMCore SiteDB re-throws particular exception
                expandedItem = None
                hadErrors = True

            if expandedItem:
                expandedList.extend(expandedItem)
            else:
                expandedList.append(item)

        if hadErrors:
            common.logger.message("Problem connecting to SiteDB. " \
                                + "%s " % self.kind.upper() \
                                + "white/blacklist may be incomplete.")
            common.logger.message("List is %s" % expandedList)

        return expandedList

    def checkBlackList(self, Sites, fileblocks=''):
        """
        Select sites that are not excluded by the user (via blacklist)

        The sites returned are the input sites minus the contents of the
        self.blacklist

        @param Sites: The sites which will be filtered
        @keyword fileblocks: The block this is used for; only used in a pretty
           debug message.
        @returns: The input sites minus the blacklist.
        """
        Sites = sets.Set(Sites)
        #print "Sites:",Sites
        blacklist = self.blacklist
        blacklist = sets.Set(self.match_list(Sites, self.blacklist))
        #print "Black list:",blacklist
        goodSites = Sites.difference(blacklist)
        #print "Good Sites:",goodSites,"\n"
        goodSites = list(goodSites)
        if not goodSites and fileblocks:
            msg = "No sites hosting the block %s after blackList" % fileblocks
            common.logger.debug(5, msg)
            common.logger.debug(5, "Proceeding without this block.\n")
        elif fileblocks:
            common.logger.debug(5, "Selected sites for block %s via blacklist " \
                "are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
        return goodSites

    def checkWhiteList(self, Sites, fileblocks=''):
        """
        Select sites that are defined by the user (via white list).

        The sites returned are the intersection of the input sites and the
        contents of self.whitelist

        @param Sites: The sites which will be filtered
        @keyword fileblocks: The block this is applied for; only used for a
           pretty debug message
        @returns: The intersection of the input Sites and self.whitelist.
        """
        if not self.whitelist:
            return Sites
        whitelist = self.whitelist
        whitelist = self.match_list(Sites, self.whitelist)
        #print "White list:",whitelist
        Sites = sets.Set(Sites)
        goodSites = Sites.intersection(whitelist)
        #print "Good Sites:",goodSites,"\n"
        goodSites = list(goodSites)
        if not goodSites and fileblocks:
            msg = "No sites hosting the block %s after whiteList" % fileblocks
            common.logger.debug(5, msg)
            common.logger.debug(5, "Proceeding without this block.\n")
        elif fileblocks:
            common.logger.debug(5, "Selected sites for block %s via whitelist "\
                " are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))

        return goodSites

    def cleanForBlackWhiteList(self, destinations, list=False):
        """
        Clean for black/white lists using parser.

        Take the input list and apply the blacklist, then the whitelist that
        the user specified.

        @param destinations: A list of all the input sites
        @keyword list: Set to True or the string 'list' to return a list
           object.  Set to False or the string '' to return a string object.
           The default is False.
        @returns: The list of all input sites, first filtered by the blacklist,
           then filtered by the whitelist.  If list=True, returns a list; if
           list=False, return a string.
        """
        if list:
            return self.checkWhiteList(self.checkBlackList(destinations))
        else:
            return ','.join(self.checkWhiteList(self.checkBlackList( \
                destinations)))


    def match_list(self, names, match_list):
        """
        Filter a list of names against a comma-separated list of expressions.

        This uses the `match` function to do the heavy lifting

        @param names: A list of input names to filter
        @type names: list
        @param match_list: A comma-separated list of expressions
        @type match_list: str
        @returns: A list, filtered from `names`, of all entries which match an
          expression in match_list
        @rtype: list
        """
        results = []
        if isinstance(match_list, types.StringType):
            match_list = match_list.split(',')

        for expr in match_list:
            expr = expr.strip()
            matching = self.match(names, expr)
            if matching:
                results.extend(matching)
            else:
                results.append(expr)
        return results


    def match(self, names, expr):
        """
        Return all the entries in `names` which match `expr`

        First, try to apply wildcard-based filters, then look at substrings,
        then interpret expr as a regex.

        @param names: An input list of strings to match
        @param expr: A string expression to use for matching
        @returns: All entries in the list `names` which match `expr`
        """

        results = fnmatch.filter(names, expr)
        results.extend([i for i in names if i.find(expr) >= 0])
        try:
            my_re = re.compile(expr)
        except:
            my_re = None
        if not my_re:
            return results
        results.extend([i for i in names if my_re.search(i)])
        return results


class SEBlackWhiteListParser(BlackWhiteListParser):
    """
    Use the BlackWhiteListParser to filter out the possible list of SEs
    from the user's input; see the documentation for BlackWhiteListParser.
    """

    def __init__(self, cfg_params):
        super(SEBlackWhiteListParser, self).__init__(cfg_params)
        self.kind = 'se'
        self.mapper = self.siteDBAPI.CMSNametoSE
        self.configure(cfg_params)


class CEBlackWhiteListParser(BlackWhiteListParser):
    """
    Use the BlackWhiteListParser to filter out the possible list of CEs
    from the user's input; see the documentation for BlackWhiteListParser.
    """

    def __init__(self, cfg_params):
        super(CEBlackWhiteListParser, self).__init__(cfg_params)
        self.kind = 'ce'
        self.mapper = self.siteDBAPI.CMSNametoCE
        self.configure(cfg_params)
Revision:	1.8
Committed:	Fri Oct 24 17:13:22 2008 UTC (16 years, 6 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.7:	+2 -2 lines
State:	*FILE REMOVED*
Error occurred while calculating annotation data.
Log Message:	Remove code moved to WMCore
#	Content
1	#!/usr/bin/env python
2	"""
3	_BlackWhiteListParser_
4
5	Parsing for black and white lists, both SE and CE
6
7	Large parts of the July 2008 re-write come from Brian Bockelman
8
9	"""
10
11	__revision__ = "$Id: BlackWhiteListParser.py,v 1.7 2008/09/11 14:44:02 ewv Exp $"
12	__version__ = "$Revision: 1.7 $"
13
14
15	import os
16	import sys
17	import sets
18	import time
19	import types
20	import fnmatch
21
22	from crab_logger import Logger
23	from crab_exceptions import *
24	from crab_util import *
25	import common
26	from ProdCommon.SiteDB.SiteDB import SiteDBJSON
27
28	class BlackWhiteListParser(object):
29
30	"""
31	A class which applies blacklist and whitelist; designed to allow the user
32	to filter out sites. Allows users to specify only the CMS name from SiteDB
33	(and simple wildcards), but internally filters only on the CE/SE name.
34	"""
35
36	def __init__(self, cfg_params):
37	self.kind = 'se'
38	self.mapper = None # Defined by Super class
39	self.siteDBAPI = SiteDBJSON()
40
41	def configure(self, cfg_params):
42	"""
43	Load up the black and white list from the configuation parameters
44	* EDG.%s_black_list
45	* EDG.%s_white_list
46	and expand things that SiteDB knows the CMS names for
47	""" % (self.kind, self.kind)
48
49	self.blacklist = []
50	if cfg_params.has_key('EDG.%s_black_list' % self.kind):
51	user_input = cfg_params['EDG.%s_black_list' % self.kind]
52	self.blacklist = self.expandList(user_input)
53	common.logger.debug(5,'Converted %s blacklist: %s' % (self.kind, ', '.join(self.blacklist)))
54
55	self.whitelist = []
56	if cfg_params.has_key('EDG.%s_white_list' % self.kind):
57	user_input = cfg_params['EDG.%s_white_list' % self.kind]
58	self.whitelist = self.expandList(user_input)
59	common.logger.debug(5, 'Converted %s whitelist: %s' % (self.kind, ', '.join(self.whitelist)))
60
61	self.blacklist = sets.Set(self.blacklist)
62	self.whitelist = sets.Set(self.whitelist)
63	#print "User's %s blacklist: %s" % (self.kind,self.blacklist)
64	#print "User's %s whitelist: %s" % (self.kind,self.whitelist)
65
66	def expandList(self, userInput):
67	"""
68	Contact SiteDB to expand lists like T2_US into lists of
69	actual SE names and CE names.
70	"""
71
72	userList = userInput.split(',')
73	expandedList = []
74	hadErrors = False
75	for item in userList:
76	item = item.strip()
77	try:
78	expandedItem = self.mapper(item)
79	except Exception: # FIXME: WMCore SiteDB re-throws particular exception
80	expandedItem = None
81	hadErrors = True
82
83	if expandedItem:
84	expandedList.extend(expandedItem)
85	else:
86	expandedList.append(item)
87
88	if hadErrors:
89	common.logger.message("Problem connecting to SiteDB. " \
90	+ "%s " % self.kind.upper() \
91	+ "white/blacklist may be incomplete.")
92	common.logger.message("List is %s" % expandedList)
93
94	return expandedList
95
96	def checkBlackList(self, Sites, fileblocks=''):
97	"""
98	Select sites that are not excluded by the user (via blacklist)
99
100	The sites returned are the input sites minus the contents of the
101	self.blacklist
102
103	@param Sites: The sites which will be filtered
104	@keyword fileblocks: The block this is used for; only used in a pretty
105	debug message.
106	@returns: The input sites minus the blacklist.
107	"""
108	Sites = sets.Set(Sites)
109	#print "Sites:",Sites
110	blacklist = self.blacklist
111	blacklist = sets.Set(self.match_list(Sites, self.blacklist))
112	#print "Black list:",blacklist
113	goodSites = Sites.difference(blacklist)
114	#print "Good Sites:",goodSites,"\n"
115	goodSites = list(goodSites)
116	if not goodSites and fileblocks:
117	msg = "No sites hosting the block %s after blackList" % fileblocks
118	common.logger.debug(5, msg)
119	common.logger.debug(5, "Proceeding without this block.\n")
120	elif fileblocks:
121	common.logger.debug(5, "Selected sites for block %s via blacklist " \
122	"are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
123	return goodSites
124
125	def checkWhiteList(self, Sites, fileblocks=''):
126	"""
127	Select sites that are defined by the user (via white list).
128
129	The sites returned are the intersection of the input sites and the
130	contents of self.whitelist
131
132	@param Sites: The sites which will be filtered
133	@keyword fileblocks: The block this is applied for; only used for a
134	pretty debug message
135	@returns: The intersection of the input Sites and self.whitelist.
136	"""
137	if not self.whitelist:
138	return Sites
139	whitelist = self.whitelist
140	whitelist = self.match_list(Sites, self.whitelist)
141	#print "White list:",whitelist
142	Sites = sets.Set(Sites)
143	goodSites = Sites.intersection(whitelist)
144	#print "Good Sites:",goodSites,"\n"
145	goodSites = list(goodSites)
146	if not goodSites and fileblocks:
147	msg = "No sites hosting the block %s after whiteList" % fileblocks
148	common.logger.debug(5, msg)
149	common.logger.debug(5, "Proceeding without this block.\n")
150	elif fileblocks:
151	common.logger.debug(5, "Selected sites for block %s via whitelist "\
152	" are %s.\n" % (', '.join(fileblocks), ', '.join(goodSites)))
153
154	return goodSites
155
156	def cleanForBlackWhiteList(self, destinations, list=False):
157	"""
158	Clean for black/white lists using parser.
159
160	Take the input list and apply the blacklist, then the whitelist that
161	the user specified.
162
163	@param destinations: A list of all the input sites
164	@keyword list: Set to True or the string 'list' to return a list
165	object. Set to False or the string '' to return a string object.
166	The default is False.
167	@returns: The list of all input sites, first filtered by the blacklist,
168	then filtered by the whitelist. If list=True, returns a list; if
169	list=False, return a string.
170	"""
171	if list:
172	return self.checkWhiteList(self.checkBlackList(destinations))
173	else:
174	return ','.join(self.checkWhiteList(self.checkBlackList( \
175	destinations)))
176
177
178	def match_list(self, names, match_list):
179	"""
180	Filter a list of names against a comma-separated list of expressions.
181
182	This uses the `match` function to do the heavy lifting
183
184	@param names: A list of input names to filter
185	@type names: list
186	@param match_list: A comma-separated list of expressions
187	@type match_list: str
188	@returns: A list, filtered from `names`, of all entries which match an
189	expression in match_list
190	@rtype: list
191	"""
192	results = []
193	if isinstance(match_list, types.StringType):
194	match_list = match_list.split(',')
195
196	for expr in match_list:
197	expr = expr.strip()
198	matching = self.match(names, expr)
199	if matching:
200	results.extend(matching)
201	else:
202	results.append(expr)
203	return results
204
205
206	def match(self, names, expr):
207	"""
208	Return all the entries in `names` which match `expr`
209
210	First, try to apply wildcard-based filters, then look at substrings,
211	then interpret expr as a regex.
212
213	@param names: An input list of strings to match
214	@param expr: A string expression to use for matching
215	@returns: All entries in the list `names` which match `expr`
216	"""
217
218	results = fnmatch.filter(names, expr)
219	results.extend([i for i in names if i.find(expr) >= 0])
220	try:
221	my_re = re.compile(expr)
222	except:
223	my_re = None
224	if not my_re:
225	return results
226	results.extend([i for i in names if my_re.search(i)])
227	return results
228
229
230
231	class SEBlackWhiteListParser(BlackWhiteListParser):
232	"""
233	Use the BlackWhiteListParser to filter out the possible list of SEs
234	from the user's input; see the documentation for BlackWhiteListParser.
235	"""
236
237	def __init__(self, cfg_params):
238	super(SEBlackWhiteListParser, self).__init__(cfg_params)
239	self.kind = 'se'
240	self.mapper = self.siteDBAPI.CMSNametoSE
241	self.configure(cfg_params)
242
243
244
245	class CEBlackWhiteListParser(BlackWhiteListParser):
246	"""
247	Use the BlackWhiteListParser to filter out the possible list of CEs
248	from the user's input; see the documentation for BlackWhiteListParser.
249	"""
250
251	def __init__(self, cfg_params):
252	super(CEBlackWhiteListParser, self).__init__(cfg_params)
253	self.kind = 'ce'
254	self.mapper = self.siteDBAPI.CMSNametoCE
255	self.configure(cfg_params)