UserCode/RateMonShiftTool_dev/MenuAnalyzer.py

#!/usr/bin/env python

import os
import sys
from DatabaseParser import ConnectDB
import re
import cx_Oracle
import eventContent

try:  ## set is builtin in python 2.6.4 and sets is deprecated
    set
except NameError:
    from sets import Set

class MenuAnalyzer:
    def __init__(self,name):
        ##default ranges
        self.maxModuleNameLength = 300
        self.maxModulesPerPath   = 50
        self.maxPaths = 500
        self.maxEndPaths = 30

        ##required streams
        self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
                                  'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
        self.ExpressStreamName = 'Express'
        self.expressPDs      = { 'ExpressPhysics' : 'Collisions',
                                 'ExpressCosmics' : 'Cosmics' }

        self.expressType = ''

        self.menuName = name
        self.endPathList = set()
        self.perPathModuleList={}
        self.perModuleTypeList={}
        self.perStreamPDList={}
        self.perPDPathList={}
        self.Results={}
        self.ModuleList=[]
        self.eventContent={}

        self.AnalysisList=[]
        

        ## statically define the analysis map: new analyses must be registered here
        self.AnalysisMap = {
            'moduleLength' : self.checkModuleLength,
            'numberOfPaths' : self.checkNumPaths,
            'numberOfEndPaths' : self.checkNumEndPaths,
            'reqStreamsAndPDs' : self.reqStreamsAndPDs,
            'checkExpress' : self.checkExpress,
            'checkNameFormats' :self.checkNameFormats,
            'checkEventContent':self.checkEventContent
            }
        self.ProblemDescriptions = {
            'moduleLength':'Modules too long',
            'numberOfPaths':'Too many paths',
            'numberOfEndPaths':'Too many endpaths',
            'reqStreamsAndPDs':'Missing required stream/PD',
            'checkExpress' : 'Invalid or missing express stream/PD',
            'checkNameFormats' : 'Invalid PD or path name format',
            'checkEventContent' : 'Invalid Event Content'
            }

        self.T0REGEXP = { ## These are the regexps that T0 uses to access things
            # Regexp for save file name paths. We don't process anything else.
            'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
            # Regexp for valid dataset names.
            'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
            # Regexp for valid RelVal dataset names.
            'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
            'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
            # Regexp for online DQM files.
            'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
            # Regexp for offline DQM files.
            'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
            # Regexp for acquisition era part of the processed dataset name.
            'RXERA' : re.compile("^([A-Za-z]+\d+|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
            }

    def AddAnalysis(self,name): 
        self.AnalysisList.append(name)

    def AddAllAnalyses(self):
        for name in self.AnalysisMap.iterkeys():
            self.AddAnalysis(name)

    def Analyze(self):
        cursor = ConnectDB('hlt')
        self.GetModules(cursor)
        self.GetStreamsPathsPDs(cursor)
        self.GetEventContent(cursor)
        for analysis in self.AnalysisList: 
            if not self.AnalysisMap.has_key(analysis):
                print "ERROR: Analysis %s not defined" % (analysis,)
                continue
            self.AnalysisMap[analysis]()
            
        
    def checkModuleLength(self):
        self.Results['moduleLength'] = []
        for modName,type in self.perModuleTypeList.iteritems():
            if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
        
    def checkNumPaths(self):
        if len(self.perPathModuleList) > self.maxPaths:
            self.Results['numberOfPaths'] = len(self.perPathModuleList) 
        else:
            self.Results['numberOfPaths'] = 0
    def checkNumEndPaths(self):
        if len(self.endPathList) > self.maxEndPaths:
            self.Results['numberOfEndPaths'] = len(self.endPathList)
        else:
            self.Results['numberOfEndPaths'] = 0
    def reqStreamsAndPDs(self):
        self.Results['reqStreamsAndPDs'] = []
        for stream,PDList in self.requiredStreamsAndPDs.iteritems():
            if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
            for PD in PDList:
                if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
    
    def checkExpress(self):
        self.Results['checkExpress'] = []
        if not self.perStreamPDList.has_key(self.ExpressStreamName):
            self.Results['checkExpress'].append(self.ExpressStreamName)
            return

        if len(self.perStreamPDList[self.ExpressStreamName]) >1:
            self.Results['checkExpress'].append("MULTIPLE_PDS")
        if len(self.perStreamPDList[self.ExpressStreamName]) <1:
            self.Results['checkExpress'].append("NO_PDS")

        for PD in self.perStreamPDList[self.ExpressStreamName]:
            if not self.expressPDs.has_key(PD):
                self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
            else:
                self.expressType = self.expressPDs[PD]
                
        
    def checkNameFormats(self):
        self.Results['checkNameFormats']=[]
        for PD in self.perPDPathList.iterkeys():
            if not self.T0REGEXP['RXDATASET'].match(PD):
                self.Results['checkNameFormats'].append(PD)
        for path in self.perPathModuleList.iterkeys():
            if not self.T0REGEXP['RXSAFEPATH'].match(path):
                self.Results['checkNameFormats'].append(path)

    def checkEventContent(self):
        self.Results['checkEventContent']=[]
        for stream,content in self.eventContent.iteritems():
            #first check for a drop statement
            if not ('drop *' in content or 'drop *_hlt*_*_*' in content):
                self.Results['checkEventContent'].append(stream+'::drop *')
            if not eventContent.requiredEventContent.has_key(stream): continue
            requiredContent = eventContent.requiredEventContent[stream]
            for entry in requiredContent:
                if not entry in content:
                    self.Results['checkEventContent'].append(stream+'::'+entry)
        
    def GetModules(self,cursor):
        sqlquery ="""  
        SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
        FROM
        CMS_HLT.PARAMETERS B,
        CMS_HLT.SUPERIDPARAMETERASSOC C,
        CMS_HLT.MODULETEMPLATES D,
        CMS_HLT.MODULES E,
        CMS_HLT.PATHMODULEASSOC F,
        CMS_HLT.CONFIGURATIONPATHASSOC G,
        CMS_HLT.CONFIGURATIONS H,
        CMS_HLT.PATHS I
        WHERE
        B.PARAMID = C.PARAMID AND
        C.SUPERID = F.MODULEID AND
        E.TEMPLATEID = D.SUPERID AND
        F.MODULEID = E.SUPERID AND
        F.PATHID=G.PATHID AND
        I.PATHID=G.PATHID AND
        G.CONFIGID=H.CONFIGID AND
        H.CONFIGDESCRIPTOR='%s' 
        """ % (self.menuName,)
        
        cursor.execute(sqlquery)
        for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
            if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
            self.perPathModuleList[PathName].append(ModuleName)
            self.perModuleTypeList[ModuleName] = ModuleType
            if endPath: self.endPathList.add(PathName)


    def GetStreamsPathsPDs(self,cursor):
        sqlquery= """
        SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
        FROM
        CMS_HLT.STREAMS A,
        CMS_HLT.CONFIGURATIONS B,
        CMS_HLT.CONFIGURATIONPATHASSOC C,
        CMS_HLT.PATHSTREAMDATASETASSOC D,
        CMS_HLT.PATHS E,
        CMS_HLT.PRIMARYDATASETS F
        WHERE
        B.CONFIGDESCRIPTOR='%s' AND
        C.CONFIGID=B.CONFIGID AND
        D.PATHID=C.PATHID AND
        A.STREAMID=D.STREAMID AND
        E.PATHID = C.PATHID AND
        F.DATASETID = D.DATASETID
        """ % (self.menuName,)
        
        cursor.execute(sqlquery)
        for StreamName,PathName,PDName in cursor.fetchall():
            if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
            if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
            if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
            self.perPDPathList[PDName].append(PathName)

    def GetEventContent(self,cursor):
        sqlquery = """
        SELECT A.STREAMLABEL,H.STATEMENTTYPE,H.CLASSN,H.MODULEL,H.EXTRAN,H.PROCESSN
        FROM
        CMS_HLT.STREAMS A,
        CMS_HLT.CONFIGURATIONS B,
        CMS_HLT.CONFIGURATIONPATHASSOC C,
        CMS_HLT.PATHSTREAMDATASETASSOC D,
        CMS_HLT.PATHS E,
        CMS_HLT.ECSTREAMASSOC F,
        CMS_HLT.ECSTATEMENTASSOC G,
        CMS_HLT.EVENTCONTENTSTATEMENTS H
        WHERE
        B.CONFIGDESCRIPTOR='%s' AND
        C.CONFIGID=B.CONFIGID AND
        D.PATHID=C.PATHID AND
        A.STREAMID=D.STREAMID AND
        E.PATHID = C.PATHID AND
        F.STREAMID = D.STREAMID AND
        G.EVENTCONTENTID=F.EVENTCONTENTID AND
        H.STATEMENTID=G.STATEMENTID
        """ % (self.menuName,)

        cursor.execute(sqlquery)
        for stream,keep,Class,module,extra,process in cursor.fetchall():
            if not self.eventContent.has_key(stream): self.eventContent[stream]=[]
            statement = "%s_%s_%s_%s" % (Class,module,extra,process,)
            if statement == "*_*_*_*": statement = "*"
            if keep == 1: statement = "keep "+statement
            else: statement = "drop "+statement
            if not statement in self.eventContent[stream]:
                self.eventContent[stream].append( statement )
Revision:	1.5
Committed:	Thu Aug 2 15:54:52 2012 UTC (12 years, 9 months ago) by amott
Content type:	text/x-python
Branch:	MAIN
Changes since 1.4:	+41 -6 lines
Log Message:	Added event content check
#	User	Rev	Content
1	amott	1.1	#!/usr/bin/env python
2
3			import os
4			import sys
5			from DatabaseParser import ConnectDB
6			import re
7			import cx_Oracle
8	amott	1.5	import eventContent
9	amott	1.1
10			try: ## set is builtin in python 2.6.4 and sets is deprecated
11			set
12			except NameError:
13			from sets import Set
14
15			class MenuAnalyzer:
16			def __init__(self,name):
17			##default ranges
18			self.maxModuleNameLength = 300
19			self.maxModulesPerPath = 50
20			self.maxPaths = 500
21			self.maxEndPaths = 30
22
23			##required streams
24			self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
25	amott	1.3	'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
26	amott	1.1	self.ExpressStreamName = 'Express'
27			self.expressPDs = { 'ExpressPhysics' : 'Collisions',
28			'ExpressCosmics' : 'Cosmics' }
29
30			self.expressType = ''
31
32			self.menuName = name
33			self.endPathList = set()
34			self.perPathModuleList={}
35			self.perModuleTypeList={}
36			self.perStreamPDList={}
37			self.perPDPathList={}
38			self.Results={}
39			self.ModuleList=[]
40	amott	1.5	self.eventContent={}
41
42	amott	1.1	self.AnalysisList=[]
43
44
45			## statically define the analysis map: new analyses must be registered here
46			self.AnalysisMap = {
47			'moduleLength' : self.checkModuleLength,
48			'numberOfPaths' : self.checkNumPaths,
49			'numberOfEndPaths' : self.checkNumEndPaths,
50			'reqStreamsAndPDs' : self.reqStreamsAndPDs,
51			'checkExpress' : self.checkExpress,
52	amott	1.5	'checkNameFormats' :self.checkNameFormats,
53			'checkEventContent':self.checkEventContent
54	amott	1.1	}
55	amott	1.2	self.ProblemDescriptions = {
56			'moduleLength':'Modules too long',
57			'numberOfPaths':'Too many paths',
58			'numberOfEndPaths':'Too many endpaths',
59			'reqStreamsAndPDs':'Missing required stream/PD',
60			'checkExpress' : 'Invalid or missing express stream/PD',
61	amott	1.5	'checkNameFormats' : 'Invalid PD or path name format',
62			'checkEventContent' : 'Invalid Event Content'
63	amott	1.2	}
64	amott	1.1
65			self.T0REGEXP = { ## These are the regexps that T0 uses to access things
66			# Regexp for save file name paths. We don't process anything else.
67			'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
68			# Regexp for valid dataset names.
69			'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
70			# Regexp for valid RelVal dataset names.
71			'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
72			'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
73			# Regexp for online DQM files.
74			'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
75			# Regexp for offline DQM files.
76			'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
77			# Regexp for acquisition era part of the processed dataset name.
78			'RXERA' : re.compile("^([A-Za-z]+\d+\|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
79			}
80
81			def AddAnalysis(self,name):
82			self.AnalysisList.append(name)
83
84			def AddAllAnalyses(self):
85			for name in self.AnalysisMap.iterkeys():
86			self.AddAnalysis(name)
87
88			def Analyze(self):
89			cursor = ConnectDB('hlt')
90			self.GetModules(cursor)
91			self.GetStreamsPathsPDs(cursor)
92	amott	1.5	self.GetEventContent(cursor)
93	amott	1.1	for analysis in self.AnalysisList:
94			if not self.AnalysisMap.has_key(analysis):
95			print "ERROR: Analysis %s not defined" % (analysis,)
96			continue
97			self.AnalysisMap[analysis]()
98
99
100
101			def checkModuleLength(self):
102			self.Results['moduleLength'] = []
103			for modName,type in self.perModuleTypeList.iteritems():
104			if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
105
106			def checkNumPaths(self):
107			if len(self.perPathModuleList) > self.maxPaths:
108			self.Results['numberOfPaths'] = len(self.perPathModuleList)
109			else:
110			self.Results['numberOfPaths'] = 0
111			def checkNumEndPaths(self):
112			if len(self.endPathList) > self.maxEndPaths:
113			self.Results['numberOfEndPaths'] = len(self.endPathList)
114			else:
115			self.Results['numberOfEndPaths'] = 0
116			def reqStreamsAndPDs(self):
117			self.Results['reqStreamsAndPDs'] = []
118			for stream,PDList in self.requiredStreamsAndPDs.iteritems():
119			if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
120			for PD in PDList:
121			if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
122
123			def checkExpress(self):
124			self.Results['checkExpress'] = []
125			if not self.perStreamPDList.has_key(self.ExpressStreamName):
126			self.Results['checkExpress'].append(self.ExpressStreamName)
127			return
128
129			if len(self.perStreamPDList[self.ExpressStreamName]) >1:
130			self.Results['checkExpress'].append("MULTIPLE_PDS")
131			if len(self.perStreamPDList[self.ExpressStreamName]) <1:
132			self.Results['checkExpress'].append("NO_PDS")
133
134			for PD in self.perStreamPDList[self.ExpressStreamName]:
135			if not self.expressPDs.has_key(PD):
136			self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
137			else:
138			self.expressType = self.expressPDs[PD]
139
140
141
142			def checkNameFormats(self):
143			self.Results['checkNameFormats']=[]
144			for PD in self.perPDPathList.iterkeys():
145			if not self.T0REGEXP['RXDATASET'].match(PD):
146			self.Results['checkNameFormats'].append(PD)
147			for path in self.perPathModuleList.iterkeys():
148			if not self.T0REGEXP['RXSAFEPATH'].match(path):
149			self.Results['checkNameFormats'].append(path)
150	amott	1.5
151			def checkEventContent(self):
152			self.Results['checkEventContent']=[]
153			for stream,content in self.eventContent.iteritems():
154			#first check for a drop statement
155			if not ('drop ' in content or 'drop _hlt__*' in content):
156			self.Results['checkEventContent'].append(stream+'::drop *')
157			if not eventContent.requiredEventContent.has_key(stream): continue
158			requiredContent = eventContent.requiredEventContent[stream]
159			for entry in requiredContent:
160			if not entry in content:
161			self.Results['checkEventContent'].append(stream+'::'+entry)
162	amott	1.1
163			def GetModules(self,cursor):
164			sqlquery ="""
165			SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
166			FROM
167			CMS_HLT.PARAMETERS B,
168			CMS_HLT.SUPERIDPARAMETERASSOC C,
169			CMS_HLT.MODULETEMPLATES D,
170			CMS_HLT.MODULES E,
171			CMS_HLT.PATHMODULEASSOC F,
172			CMS_HLT.CONFIGURATIONPATHASSOC G,
173			CMS_HLT.CONFIGURATIONS H,
174			CMS_HLT.PATHS I
175			WHERE
176			B.PARAMID = C.PARAMID AND
177			C.SUPERID = F.MODULEID AND
178			E.TEMPLATEID = D.SUPERID AND
179			F.MODULEID = E.SUPERID AND
180			F.PATHID=G.PATHID AND
181			I.PATHID=G.PATHID AND
182			G.CONFIGID=H.CONFIGID AND
183			H.CONFIGDESCRIPTOR='%s'
184			""" % (self.menuName,)
185
186			cursor.execute(sqlquery)
187			for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
188			if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
189			self.perPathModuleList[PathName].append(ModuleName)
190			self.perModuleTypeList[ModuleName] = ModuleType
191			if endPath: self.endPathList.add(PathName)
192
193
194			def GetStreamsPathsPDs(self,cursor):
195			sqlquery= """
196			SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
197			FROM
198			CMS_HLT.STREAMS A,
199			CMS_HLT.CONFIGURATIONS B,
200			CMS_HLT.CONFIGURATIONPATHASSOC C,
201			CMS_HLT.PATHSTREAMDATASETASSOC D,
202			CMS_HLT.PATHS E,
203			CMS_HLT.PRIMARYDATASETS F
204			WHERE
205			B.CONFIGDESCRIPTOR='%s' AND
206			C.CONFIGID=B.CONFIGID AND
207			D.PATHID=C.PATHID AND
208			A.STREAMID=D.STREAMID AND
209			E.PATHID = C.PATHID AND
210			F.DATASETID = D.DATASETID
211			""" % (self.menuName,)
212
213			cursor.execute(sqlquery)
214			for StreamName,PathName,PDName in cursor.fetchall():
215			if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
216			if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
217			if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
218			self.perPDPathList[PDName].append(PathName)
219
220	amott	1.4	def GetEventContent(self,cursor):
221	amott	1.5	sqlquery = """
222			SELECT A.STREAMLABEL,H.STATEMENTTYPE,H.CLASSN,H.MODULEL,H.EXTRAN,H.PROCESSN
223	amott	1.4	FROM
224			CMS_HLT.STREAMS A,
225			CMS_HLT.CONFIGURATIONS B,
226	amott	1.5	CMS_HLT.CONFIGURATIONPATHASSOC C,
227			CMS_HLT.PATHSTREAMDATASETASSOC D,
228			CMS_HLT.PATHS E,
229			CMS_HLT.ECSTREAMASSOC F,
230			CMS_HLT.ECSTATEMENTASSOC G,
231			CMS_HLT.EVENTCONTENTSTATEMENTS H
232	amott	1.4	WHERE
233			B.CONFIGDESCRIPTOR='%s' AND
234			C.CONFIGID=B.CONFIGID AND
235			D.PATHID=C.PATHID AND
236			A.STREAMID=D.STREAMID AND
237			E.PATHID = C.PATHID AND
238	amott	1.5	F.STREAMID = D.STREAMID AND
239			G.EVENTCONTENTID=F.EVENTCONTENTID AND
240			H.STATEMENTID=G.STATEMENTID
241	amott	1.4	""" % (self.menuName,)
242	amott	1.5
243			cursor.execute(sqlquery)
244			for stream,keep,Class,module,extra,process in cursor.fetchall():
245			if not self.eventContent.has_key(stream): self.eventContent[stream]=[]
246			statement = "%s_%s_%s_%s" % (Class,module,extra,process,)
247			if statement == "___": statement = "*"
248			if keep == 1: statement = "keep "+statement
249			else: statement = "drop "+statement
250			if not statement in self.eventContent[stream]:
251			self.eventContent[stream].append( statement )