CRAB/python/LumiList.py

#!/usr/bin/env python

"""
This code is now deprecated. Changes should be made to CMSSW/FWCore/PythonUtilities/python/LumiList.py instead

Handle lists of lumi sections. Constuct in several different formats and filter
(mask) a secondary list of lumis.
This class can also handle ranges of events as the structure is identical
or could be subclassed renaming a function or two.
"""

__revision__ = "$Id: LumiList.py,v 1.4 2010/05/26 19:46:12 ewv Exp $"
__version__ = "$Revision: 1.4 $"

try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
    import json
except ImportError:
    import simplejson as json

class LumiList(object):
    """
    Deal with lists of lumis in several different forms:
    Compact list:
        {
        '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
        '2':[[1,45],[50,80]]
        }
        where the first key is the run number, subsequent pairs are
        ranges of lumis within that run that are desired
    Runs and lumis:
        {
        '1': [1,2,3,4,6,7,8,9,10],
        '2': [1,4,5,20]
        }
        where the first key is the run number and the list is a list of
        individual lumi sections
    Run  lumi pairs:
        [[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
        where each pair in the list is an individual run&lumi
    CMSSW representation:
        '1:1-1:33,1:35,1:37-1:47,2:1-2:45,2:50-2:80'
        The string used by CMSSW in lumisToProcess or lumisToSkip
        is a subset of the compactList example above
    """


    def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None):
        """
        Constructor takes filename (JSON), a list of run/lumi pairs,
        or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
        """
        self.compactList = {}
        if filename:
            self.filename = filename
            jsonFile = open(self.filename,'r')
            self.compactList = json.load(jsonFile)
        elif lumis:
            runsAndLumis = {}
            for (run, lumi) in lumis:
                run = str(run)
                if not runsAndLumis.has_key(run):
                    runsAndLumis[run] = []
                runsAndLumis[run].append(lumi)

        if runsAndLumis:
            for run in runsAndLumis.keys():
                runString = str(run)
                lastLumi = -1000
                self.compactList[runString] = []
                lumiList = runsAndLumis[run]
                for lumi in sorted(lumiList):
                    if lumi == lastLumi:
                        pass # Skip duplicates
                    elif lumi != lastLumi + 1: # Break in lumi sequence
                        self.compactList[runString].append([lumi, lumi])
                    else:
                        nRange =  len(self.compactList[runString])
                        self.compactList[runString][nRange-1][1] = lumi
                    lastLumi = lumi
        if runs:
            for run in runs:
                runString = str(run)
                self.compactList[runString] = [[1,0xFFFFFFF]]

    def filterLumis(self, lumiList):
        """
        Return a list of lumis that are in compactList.
        lumilist is of the simple form
        [(run1,lumi1),(run1,lumi2),(run2,lumi1)]
        """
        filteredList = []
        for (run, lumi) in lumiList:
            runsInLumi = self.compactList.get(str(run), [[0,-1]])
            for (first, last) in runsInLumi:
                if lumi >= first and lumi <= last:
                    filteredList.append((run, lumi))
                    break
        return filteredList


    def getCompactList(self):
        """
        Return the compact list representation
        """
        return self.compactList


    def getLumis(self):
        """
        Return the list of pairs representation
        """
        theList = []
        runs = self.compactList.keys()
        runs.sort(key=int)
        for run in runs:
            lumis = self.compactList[run]
            for lumiPair in sorted(lumis):
                for lumi in range(lumiPair[0], lumiPair[1]+1):
                    theList.append((int(run), lumi))

        return theList


    def getCMSSWString(self):
        """
        Turn compactList into a list of the format
        R1:L1,R2:L2-R2:L3 which is acceptable to CMSSW LumiBlockRange variable
        """

        parts = []
        runs = self.compactList.keys()
        runs.sort(key=int)
        for run in runs:
            lumis = self.compactList[run]
            for lumiPair in sorted(lumis):
                if lumiPair[0] == lumiPair[1]:
                    parts.append("%s:%s" % (run, lumiPair[0]))
                else:
                    parts.append("%s:%s-%s:%s" %
                                 (run, lumiPair[0], run, lumiPair[1]))

        output = ','.join(parts)
        return output


# Unit test code
'''
import unittest
from WMCore.DataStructs.LumiList import LumiList


class LumiListTest(unittest.TestCase):
    """
    _LumiListTest_

    """

    def testRead(self):
        """
        Test reading from JSON
        """
        exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
        exDict   = {'1': [[1, 33], [35, 35], [37, 47]],
                    '2': [[49, 75], [77, 130], [133, 136]]}

        jsonList = LumiList(filename = 'lumiTest.json')
        lumiString = jsonList.getCMSSWString()
        lumiList = jsonList.getCompactList()

        self.assertTrue(lumiString == exString)
        self.assertTrue(lumiList   == exDict)

    def testList(self):
        """
        Test constucting from list of pairs
        """

        listLs1 = range(1, 34) + [35] + range(37, 48)
        listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
        lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList = jsonLister.getCompactList()

        pairLister = LumiList(lumis = lumis)
        pairString = pairLister.getCMSSWString()
        pairList = pairLister.getCompactList()

        self.assertTrue(jsonString == pairString)
        self.assertTrue(jsonList   == pairList)


    def testRuns(self):
        """
        Test constucting from run and list of lumis
        """
        runsAndLumis = {
            1: range(1, 34) + [35] + range(37, 48),
            2: range(49, 76) + range(77, 131) + range(133, 137)
        }
        runsAndLumis2 = {
            '1': range(1, 34) + [35] + range(37, 48),
            '2': range(49, 76) + range(77, 131) + range(133, 137)
        }

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList   = jsonLister.getCompactList()

        runLister = LumiList(runsAndLumis = runsAndLumis)
        runString = runLister.getCMSSWString()
        runList   = runLister.getCompactList()

        runLister2 = LumiList(runsAndLumis = runsAndLumis2)
        runList2 = runLister2.getCompactList()

        self.assertTrue(jsonString == runString)
        self.assertTrue(jsonList   == runList)
        self.assertTrue(runList2   == runList)


    def testFilter(self):
        """
        Test filtering of a list of lumis
        """
        runsAndLumis = {
            1: range(1, 34) + [35] + range(37, 48),
            2: range(49, 76) + range(77, 131) + range(133, 137)
        }

        completeList = zip([1]*150, range(1, 150)) + \
                       zip([2]*150, range(1, 150)) + \
                       zip([3]*150, range(1, 150))

        smallList    = zip([1]*50,  range(1, 10)) + zip([2]*50, range(50, 70))
        overlapList  = zip([1]*150, range(30, 40)) + \
                       zip([2]*150, range(60, 80))
        overlapRes   = zip([1]*9,   range(30, 34)) + [(1, 35)] + \
                       zip([1]*9,   range(37, 40)) + \
                       zip([2]*30,  range(60, 76)) + \
                       zip([2]*9,   range(77, 80))

        runLister = LumiList(runsAndLumis = runsAndLumis)

        # Test a list to be filtered which is a superset of constructed list
        filterComplete = runLister.filterLumis(completeList)
        # Test a list to be filtered which is a subset of constructed list
        filterSmall    = runLister.filterLumis(smallList)
        # Test a list to be filtered which is neither
        filterOverlap  = runLister.filterLumis(overlapList)

        self.assertTrue(filterComplete == runLister.getLumis())
        self.assertTrue(filterSmall    == smallList)
        self.assertTrue(filterOverlap  == overlapRes)

    def testDuplicates(self):
        """
        Test a list with lots of duplicates
        """
        result = zip([1]*100, range(1, 34) + range(37, 48))
        lumis  = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))

        lister = LumiList(lumis = lumis)
        self.assertTrue(lister.getLumis() == result)

    def testNull(self):
        """
        Test a null list
        """

        runLister = LumiList(lumis = None)

        self.assertTrue(runLister.getCMSSWString() == '')
        self.assertTrue(runLister.getLumis() == [])
        self.assertTrue(runLister.getCompactList() == {})


if __name__ == '__main__':
    unittest.main()
'''

# Test JSON file

#{"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}
Revision:	1.5
Committed:	Tue Jun 29 17:46:42 2010 UTC (14 years, 10 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, HEAD
Changes since 1.4:	+4 -2 lines
Log Message:	Grab LumiList from CMSSW libraries for CMSSW 3.8 and higher
#	Content
1	#!/usr/bin/env python
2
3	"""
4	This code is now deprecated. Changes should be made to CMSSW/FWCore/PythonUtilities/python/LumiList.py instead
5
6	Handle lists of lumi sections. Constuct in several different formats and filter
7	(mask) a secondary list of lumis.
8	This class can also handle ranges of events as the structure is identical
9	or could be subclassed renaming a function or two.
10	"""
11
12	__revision__ = "$Id: LumiList.py,v 1.4 2010/05/26 19:46:12 ewv Exp $"
13	__version__ = "$Revision: 1.4 $"
14
15	try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
16	import json
17	except ImportError:
18	import simplejson as json
19
20	class LumiList(object):
21	"""
22	Deal with lists of lumis in several different forms:
23	Compact list:
24	{
25	'1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
26	'2':[[1,45],[50,80]]
27	}
28	where the first key is the run number, subsequent pairs are
29	ranges of lumis within that run that are desired
30	Runs and lumis:
31	{
32	'1': [1,2,3,4,6,7,8,9,10],
33	'2': [1,4,5,20]
34	}
35	where the first key is the run number and the list is a list of
36	individual lumi sections
37	Run lumi pairs:
38	[[1,1], [1,2],[1,4], [2,1], [2,5], [1,10]]
39	where each pair in the list is an individual run&lumi
40	CMSSW representation:
41	'1:1-1:33,1:35,1:37-1:47,2:1-2:45,2:50-2:80'
42	The string used by CMSSW in lumisToProcess or lumisToSkip
43	is a subset of the compactList example above
44	"""
45
46
47	def __init__(self, filename = None, lumis = None, runsAndLumis = None, runs = None):
48	"""
49	Constructor takes filename (JSON), a list of run/lumi pairs,
50	or a dict with run #'s as the keys and a list of lumis as the values, or just a list of runs
51	"""
52	self.compactList = {}
53	if filename:
54	self.filename = filename
55	jsonFile = open(self.filename,'r')
56	self.compactList = json.load(jsonFile)
57	elif lumis:
58	runsAndLumis = {}
59	for (run, lumi) in lumis:
60	run = str(run)
61	if not runsAndLumis.has_key(run):
62	runsAndLumis[run] = []
63	runsAndLumis[run].append(lumi)
64
65	if runsAndLumis:
66	for run in runsAndLumis.keys():
67	runString = str(run)
68	lastLumi = -1000
69	self.compactList[runString] = []
70	lumiList = runsAndLumis[run]
71	for lumi in sorted(lumiList):
72	if lumi == lastLumi:
73	pass # Skip duplicates
74	elif lumi != lastLumi + 1: # Break in lumi sequence
75	self.compactList[runString].append([lumi, lumi])
76	else:
77	nRange = len(self.compactList[runString])
78	self.compactList[runString][nRange-1][1] = lumi
79	lastLumi = lumi
80	if runs:
81	for run in runs:
82	runString = str(run)
83	self.compactList[runString] = [[1,0xFFFFFFF]]
84
85	def filterLumis(self, lumiList):
86	"""
87	Return a list of lumis that are in compactList.
88	lumilist is of the simple form
89	[(run1,lumi1),(run1,lumi2),(run2,lumi1)]
90	"""
91	filteredList = []
92	for (run, lumi) in lumiList:
93	runsInLumi = self.compactList.get(str(run), [[0,-1]])
94	for (first, last) in runsInLumi:
95	if lumi >= first and lumi <= last:
96	filteredList.append((run, lumi))
97	break
98	return filteredList
99
100
101	def getCompactList(self):
102	"""
103	Return the compact list representation
104	"""
105	return self.compactList
106
107
108	def getLumis(self):
109	"""
110	Return the list of pairs representation
111	"""
112	theList = []
113	runs = self.compactList.keys()
114	runs.sort(key=int)
115	for run in runs:
116	lumis = self.compactList[run]
117	for lumiPair in sorted(lumis):
118	for lumi in range(lumiPair[0], lumiPair[1]+1):
119	theList.append((int(run), lumi))
120
121	return theList
122
123
124	def getCMSSWString(self):
125	"""
126	Turn compactList into a list of the format
127	R1:L1,R2:L2-R2:L3 which is acceptable to CMSSW LumiBlockRange variable
128	"""
129
130	parts = []
131	runs = self.compactList.keys()
132	runs.sort(key=int)
133	for run in runs:
134	lumis = self.compactList[run]
135	for lumiPair in sorted(lumis):
136	if lumiPair[0] == lumiPair[1]:
137	parts.append("%s:%s" % (run, lumiPair[0]))
138	else:
139	parts.append("%s:%s-%s:%s" %
140	(run, lumiPair[0], run, lumiPair[1]))
141
142	output = ','.join(parts)
143	return output
144
145
146
147	# Unit test code
148	'''
149	import unittest
150	from WMCore.DataStructs.LumiList import LumiList
151
152
153	class LumiListTest(unittest.TestCase):
154	"""
155	_LumiListTest_
156
157	"""
158
159	def testRead(self):
160	"""
161	Test reading from JSON
162	"""
163	exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
164	exDict = {'1': [[1, 33], [35, 35], [37, 47]],
165	'2': [[49, 75], [77, 130], [133, 136]]}
166
167	jsonList = LumiList(filename = 'lumiTest.json')
168	lumiString = jsonList.getCMSSWString()
169	lumiList = jsonList.getCompactList()
170
171	self.assertTrue(lumiString == exString)
172	self.assertTrue(lumiList == exDict)
173
174	def testList(self):
175	"""
176	Test constucting from list of pairs
177	"""
178
179	listLs1 = range(1, 34) + [35] + range(37, 48)
180	listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
181	lumis = zip([1]100, listLs1) + zip([2]100, listLs2)
182
183	jsonLister = LumiList(filename = 'lumiTest.json')
184	jsonString = jsonLister.getCMSSWString()
185	jsonList = jsonLister.getCompactList()
186
187	pairLister = LumiList(lumis = lumis)
188	pairString = pairLister.getCMSSWString()
189	pairList = pairLister.getCompactList()
190
191	self.assertTrue(jsonString == pairString)
192	self.assertTrue(jsonList == pairList)
193
194
195	def testRuns(self):
196	"""
197	Test constucting from run and list of lumis
198	"""
199	runsAndLumis = {
200	1: range(1, 34) + [35] + range(37, 48),
201	2: range(49, 76) + range(77, 131) + range(133, 137)
202	}
203	runsAndLumis2 = {
204	'1': range(1, 34) + [35] + range(37, 48),
205	'2': range(49, 76) + range(77, 131) + range(133, 137)
206	}
207
208	jsonLister = LumiList(filename = 'lumiTest.json')
209	jsonString = jsonLister.getCMSSWString()
210	jsonList = jsonLister.getCompactList()
211
212	runLister = LumiList(runsAndLumis = runsAndLumis)
213	runString = runLister.getCMSSWString()
214	runList = runLister.getCompactList()
215
216	runLister2 = LumiList(runsAndLumis = runsAndLumis2)
217	runList2 = runLister2.getCompactList()
218
219	self.assertTrue(jsonString == runString)
220	self.assertTrue(jsonList == runList)
221	self.assertTrue(runList2 == runList)
222
223
224	def testFilter(self):
225	"""
226	Test filtering of a list of lumis
227	"""
228	runsAndLumis = {
229	1: range(1, 34) + [35] + range(37, 48),
230	2: range(49, 76) + range(77, 131) + range(133, 137)
231	}
232
233	completeList = zip([1]*150, range(1, 150)) + \
234	zip([2]*150, range(1, 150)) + \
235	zip([3]*150, range(1, 150))
236
237	smallList = zip([1]50, range(1, 10)) + zip([2]50, range(50, 70))
238	overlapList = zip([1]*150, range(30, 40)) + \
239	zip([2]*150, range(60, 80))
240	overlapRes = zip([1]*9, range(30, 34)) + [(1, 35)] + \
241	zip([1]*9, range(37, 40)) + \
242	zip([2]*30, range(60, 76)) + \
243	zip([2]*9, range(77, 80))
244
245	runLister = LumiList(runsAndLumis = runsAndLumis)
246
247	# Test a list to be filtered which is a superset of constructed list
248	filterComplete = runLister.filterLumis(completeList)
249	# Test a list to be filtered which is a subset of constructed list
250	filterSmall = runLister.filterLumis(smallList)
251	# Test a list to be filtered which is neither
252	filterOverlap = runLister.filterLumis(overlapList)
253
254	self.assertTrue(filterComplete == runLister.getLumis())
255	self.assertTrue(filterSmall == smallList)
256	self.assertTrue(filterOverlap == overlapRes)
257
258	def testDuplicates(self):
259	"""
260	Test a list with lots of duplicates
261	"""
262	result = zip([1]*100, range(1, 34) + range(37, 48))
263	lumis = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))
264
265	lister = LumiList(lumis = lumis)
266	self.assertTrue(lister.getLumis() == result)
267
268	def testNull(self):
269	"""
270	Test a null list
271	"""
272
273	runLister = LumiList(lumis = None)
274
275	self.assertTrue(runLister.getCMSSWString() == '')
276	self.assertTrue(runLister.getLumis() == [])
277	self.assertTrue(runLister.getCompactList() == {})
278
279
280
281	if __name__ == '__main__':
282	unittest.main()
283	'''
284
285	# Test JSON file
286
287	#{"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}