UserCode/RateMonShiftTool_dev/MenuAnalyzer.py

#!/usr/bin/env python

import os
import sys
from DatabaseParser import ConnectDB
import re
import cx_Oracle

try:  ## set is builtin in python 2.6.4 and sets is deprecated
    set
except NameError:
    from sets import Set

class MenuAnalyzer:
    def __init__(self,name):
        ##default ranges
        self.maxModuleNameLength = 300
        self.maxModulesPerPath   = 50
        self.maxPaths = 500
        self.maxEndPaths = 30

        ##required streams
        self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
                                  'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
        self.ExpressStreamName = 'Express'
        self.expressPDs      = { 'ExpressPhysics' : 'Collisions',
                                 'ExpressCosmics' : 'Cosmics' }

        self.expressType = ''

        self.menuName = name
        self.endPathList = set()
        self.perPathModuleList={}
        self.perModuleTypeList={}
        self.perStreamPDList={}
        self.perPDPathList={}
        self.Results={}
        self.ModuleList=[]
        self.AnalysisList=[]
        

        ## statically define the analysis map: new analyses must be registered here
        self.AnalysisMap = {
            'moduleLength' : self.checkModuleLength,
            'numberOfPaths' : self.checkNumPaths,
            'numberOfEndPaths' : self.checkNumEndPaths,
            'reqStreamsAndPDs' : self.reqStreamsAndPDs,
            'checkExpress' : self.checkExpress,
            'checkNameFormats' :self.checkNameFormats
            }
        self.ProblemDescriptions = {
            'moduleLength':'Modules too long',
            'numberOfPaths':'Too many paths',
            'numberOfEndPaths':'Too many endpaths',
            'reqStreamsAndPDs':'Missing required stream/PD',
            'checkExpress' : 'Invalid or missing express stream/PD',
            'checkNameFormats' : 'Invalid PD or path name format'
            }

        self.T0REGEXP = { ## These are the regexps that T0 uses to access things
            # Regexp for save file name paths. We don't process anything else.
            'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
            # Regexp for valid dataset names.
            'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
            # Regexp for valid RelVal dataset names.
            'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
            'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
            # Regexp for online DQM files.
            'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
            # Regexp for offline DQM files.
            'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
            # Regexp for acquisition era part of the processed dataset name.
            'RXERA' : re.compile("^([A-Za-z]+\d+|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
            }

    def AddAnalysis(self,name): 
        self.AnalysisList.append(name)

    def AddAllAnalyses(self):
        for name in self.AnalysisMap.iterkeys():
            self.AddAnalysis(name)

    def Analyze(self):
        cursor = ConnectDB('hlt')
        self.GetModules(cursor)
        self.GetStreamsPathsPDs(cursor)
        for analysis in self.AnalysisList: 
            if not self.AnalysisMap.has_key(analysis):
                print "ERROR: Analysis %s not defined" % (analysis,)
                continue
            self.AnalysisMap[analysis]()
            
        
    def checkModuleLength(self):
        self.Results['moduleLength'] = []
        for modName,type in self.perModuleTypeList.iteritems():
            if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
        
    def checkNumPaths(self):
        if len(self.perPathModuleList) > self.maxPaths:
            self.Results['numberOfPaths'] = len(self.perPathModuleList) 
        else:
            self.Results['numberOfPaths'] = 0
    def checkNumEndPaths(self):
        if len(self.endPathList) > self.maxEndPaths:
            self.Results['numberOfEndPaths'] = len(self.endPathList)
        else:
            self.Results['numberOfEndPaths'] = 0
    def reqStreamsAndPDs(self):
        self.Results['reqStreamsAndPDs'] = []
        for stream,PDList in self.requiredStreamsAndPDs.iteritems():
            if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
            for PD in PDList:
                if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
    
    def checkExpress(self):
        self.Results['checkExpress'] = []
        if not self.perStreamPDList.has_key(self.ExpressStreamName):
            self.Results['checkExpress'].append(self.ExpressStreamName)
            return

        if len(self.perStreamPDList[self.ExpressStreamName]) >1:
            self.Results['checkExpress'].append("MULTIPLE_PDS")
        if len(self.perStreamPDList[self.ExpressStreamName]) <1:
            self.Results['checkExpress'].append("NO_PDS")

        for PD in self.perStreamPDList[self.ExpressStreamName]:
            if not self.expressPDs.has_key(PD):
                self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
            else:
                self.expressType = self.expressPDs[PD]
                
        
    def checkNameFormats(self):
        self.Results['checkNameFormats']=[]
        for PD in self.perPDPathList.iterkeys():
            if not self.T0REGEXP['RXDATASET'].match(PD):
                self.Results['checkNameFormats'].append(PD)
        for path in self.perPathModuleList.iterkeys():
            if not self.T0REGEXP['RXSAFEPATH'].match(path):
                self.Results['checkNameFormats'].append(path)
        
    def GetModules(self,cursor):
        sqlquery ="""  
        SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
        FROM
        CMS_HLT.PARAMETERS B,
        CMS_HLT.SUPERIDPARAMETERASSOC C,
        CMS_HLT.MODULETEMPLATES D,
        CMS_HLT.MODULES E,
        CMS_HLT.PATHMODULEASSOC F,
        CMS_HLT.CONFIGURATIONPATHASSOC G,
        CMS_HLT.CONFIGURATIONS H,
        CMS_HLT.PATHS I
        WHERE
        B.PARAMID = C.PARAMID AND
        C.SUPERID = F.MODULEID AND
        E.TEMPLATEID = D.SUPERID AND
        F.MODULEID = E.SUPERID AND
        F.PATHID=G.PATHID AND
        I.PATHID=G.PATHID AND
        G.CONFIGID=H.CONFIGID AND
        H.CONFIGDESCRIPTOR='%s' 
        """ % (self.menuName,)
        
        cursor.execute(sqlquery)
        for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
            if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
            self.perPathModuleList[PathName].append(ModuleName)
            self.perModuleTypeList[ModuleName] = ModuleType
            if endPath: self.endPathList.add(PathName)


    def GetStreamsPathsPDs(self,cursor):
        sqlquery= """
        SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
        FROM
        CMS_HLT.STREAMS A,
        CMS_HLT.CONFIGURATIONS B,
        CMS_HLT.CONFIGURATIONPATHASSOC C,
        CMS_HLT.PATHSTREAMDATASETASSOC D,
        CMS_HLT.PATHS E,
        CMS_HLT.PRIMARYDATASETS F
        WHERE
        B.CONFIGDESCRIPTOR='%s' AND
        C.CONFIGID=B.CONFIGID AND
        D.PATHID=C.PATHID AND
        A.STREAMID=D.STREAMID AND
        E.PATHID = C.PATHID AND
        F.DATASETID = D.DATASETID
        """ % (self.menuName,)
        
        cursor.execute(sqlquery)
        for StreamName,PathName,PDName in cursor.fetchall():
            if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
            if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
            if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
            self.perPDPathList[PDName].append(PathName)

Revision:	1.3
Committed:	Thu Jul 12 16:41:14 2012 UTC (12 years, 9 months ago) by amott
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	MenuAnalyzer_V00-00-01, MenuAnalyzer_V1
Changes since 1.2:	+1 -1 lines
Log Message:	added DQM to required streams
#	Content
1	#!/usr/bin/env python
2
3	import os
4	import sys
5	from DatabaseParser import ConnectDB
6	import re
7	import cx_Oracle
8
9	try: ## set is builtin in python 2.6.4 and sets is deprecated
10	set
11	except NameError:
12	from sets import Set
13
14	class MenuAnalyzer:
15	def __init__(self,name):
16	##default ranges
17	self.maxModuleNameLength = 300
18	self.maxModulesPerPath = 50
19	self.maxPaths = 500
20	self.maxEndPaths = 30
21
22	##required streams
23	self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
24	'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
25	self.ExpressStreamName = 'Express'
26	self.expressPDs = { 'ExpressPhysics' : 'Collisions',
27	'ExpressCosmics' : 'Cosmics' }
28
29	self.expressType = ''
30
31	self.menuName = name
32	self.endPathList = set()
33	self.perPathModuleList={}
34	self.perModuleTypeList={}
35	self.perStreamPDList={}
36	self.perPDPathList={}
37	self.Results={}
38	self.ModuleList=[]
39	self.AnalysisList=[]
40
41
42	## statically define the analysis map: new analyses must be registered here
43	self.AnalysisMap = {
44	'moduleLength' : self.checkModuleLength,
45	'numberOfPaths' : self.checkNumPaths,
46	'numberOfEndPaths' : self.checkNumEndPaths,
47	'reqStreamsAndPDs' : self.reqStreamsAndPDs,
48	'checkExpress' : self.checkExpress,
49	'checkNameFormats' :self.checkNameFormats
50	}
51	self.ProblemDescriptions = {
52	'moduleLength':'Modules too long',
53	'numberOfPaths':'Too many paths',
54	'numberOfEndPaths':'Too many endpaths',
55	'reqStreamsAndPDs':'Missing required stream/PD',
56	'checkExpress' : 'Invalid or missing express stream/PD',
57	'checkNameFormats' : 'Invalid PD or path name format'
58	}
59
60	self.T0REGEXP = { ## These are the regexps that T0 uses to access things
61	# Regexp for save file name paths. We don't process anything else.
62	'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
63	# Regexp for valid dataset names.
64	'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
65	# Regexp for valid RelVal dataset names.
66	'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
67	'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
68	# Regexp for online DQM files.
69	'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
70	# Regexp for offline DQM files.
71	'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
72	# Regexp for acquisition era part of the processed dataset name.
73	'RXERA' : re.compile("^([A-Za-z]+\d+\|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
74	}
75
76	def AddAnalysis(self,name):
77	self.AnalysisList.append(name)
78
79	def AddAllAnalyses(self):
80	for name in self.AnalysisMap.iterkeys():
81	self.AddAnalysis(name)
82
83	def Analyze(self):
84	cursor = ConnectDB('hlt')
85	self.GetModules(cursor)
86	self.GetStreamsPathsPDs(cursor)
87	for analysis in self.AnalysisList:
88	if not self.AnalysisMap.has_key(analysis):
89	print "ERROR: Analysis %s not defined" % (analysis,)
90	continue
91	self.AnalysisMap[analysis]()
92
93
94
95	def checkModuleLength(self):
96	self.Results['moduleLength'] = []
97	for modName,type in self.perModuleTypeList.iteritems():
98	if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
99
100	def checkNumPaths(self):
101	if len(self.perPathModuleList) > self.maxPaths:
102	self.Results['numberOfPaths'] = len(self.perPathModuleList)
103	else:
104	self.Results['numberOfPaths'] = 0
105	def checkNumEndPaths(self):
106	if len(self.endPathList) > self.maxEndPaths:
107	self.Results['numberOfEndPaths'] = len(self.endPathList)
108	else:
109	self.Results['numberOfEndPaths'] = 0
110	def reqStreamsAndPDs(self):
111	self.Results['reqStreamsAndPDs'] = []
112	for stream,PDList in self.requiredStreamsAndPDs.iteritems():
113	if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
114	for PD in PDList:
115	if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
116
117	def checkExpress(self):
118	self.Results['checkExpress'] = []
119	if not self.perStreamPDList.has_key(self.ExpressStreamName):
120	self.Results['checkExpress'].append(self.ExpressStreamName)
121	return
122
123	if len(self.perStreamPDList[self.ExpressStreamName]) >1:
124	self.Results['checkExpress'].append("MULTIPLE_PDS")
125	if len(self.perStreamPDList[self.ExpressStreamName]) <1:
126	self.Results['checkExpress'].append("NO_PDS")
127
128	for PD in self.perStreamPDList[self.ExpressStreamName]:
129	if not self.expressPDs.has_key(PD):
130	self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
131	else:
132	self.expressType = self.expressPDs[PD]
133
134
135
136	def checkNameFormats(self):
137	self.Results['checkNameFormats']=[]
138	for PD in self.perPDPathList.iterkeys():
139	if not self.T0REGEXP['RXDATASET'].match(PD):
140	self.Results['checkNameFormats'].append(PD)
141	for path in self.perPathModuleList.iterkeys():
142	if not self.T0REGEXP['RXSAFEPATH'].match(path):
143	self.Results['checkNameFormats'].append(path)
144
145	def GetModules(self,cursor):
146	sqlquery ="""
147	SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
148	FROM
149	CMS_HLT.PARAMETERS B,
150	CMS_HLT.SUPERIDPARAMETERASSOC C,
151	CMS_HLT.MODULETEMPLATES D,
152	CMS_HLT.MODULES E,
153	CMS_HLT.PATHMODULEASSOC F,
154	CMS_HLT.CONFIGURATIONPATHASSOC G,
155	CMS_HLT.CONFIGURATIONS H,
156	CMS_HLT.PATHS I
157	WHERE
158	B.PARAMID = C.PARAMID AND
159	C.SUPERID = F.MODULEID AND
160	E.TEMPLATEID = D.SUPERID AND
161	F.MODULEID = E.SUPERID AND
162	F.PATHID=G.PATHID AND
163	I.PATHID=G.PATHID AND
164	G.CONFIGID=H.CONFIGID AND
165	H.CONFIGDESCRIPTOR='%s'
166	""" % (self.menuName,)
167
168	cursor.execute(sqlquery)
169	for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
170	if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
171	self.perPathModuleList[PathName].append(ModuleName)
172	self.perModuleTypeList[ModuleName] = ModuleType
173	if endPath: self.endPathList.add(PathName)
174
175
176	def GetStreamsPathsPDs(self,cursor):
177	sqlquery= """
178	SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
179	FROM
180	CMS_HLT.STREAMS A,
181	CMS_HLT.CONFIGURATIONS B,
182	CMS_HLT.CONFIGURATIONPATHASSOC C,
183	CMS_HLT.PATHSTREAMDATASETASSOC D,
184	CMS_HLT.PATHS E,
185	CMS_HLT.PRIMARYDATASETS F
186	WHERE
187	B.CONFIGDESCRIPTOR='%s' AND
188	C.CONFIGID=B.CONFIGID AND
189	D.PATHID=C.PATHID AND
190	A.STREAMID=D.STREAMID AND
191	E.PATHID = C.PATHID AND
192	F.DATASETID = D.DATASETID
193	""" % (self.menuName,)
194
195	cursor.execute(sqlquery)
196	for StreamName,PathName,PDName in cursor.fetchall():
197	if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
198	if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
199	if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
200	self.perPDPathList[PDName].append(PathName)
201