ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/RateMonShiftTool_dev/MenuAnalyzer.py
Revision: 1.6
Committed: Tue Aug 7 09:36:35 2012 UTC (12 years, 8 months ago) by amott
Content type: text/x-python
Branch: MAIN
CVS Tags: V00-01-07, V00-01-06
Changes since 1.5: +32 -2 lines
Log Message:
Added check for L1 unmask module

File Contents

# User Rev Content
1 amott 1.1 #!/usr/bin/env python
2    
3     import os
4     import sys
5     from DatabaseParser import ConnectDB
6     import re
7     import cx_Oracle
8 amott 1.5 import eventContent
9 amott 1.1
10     try: ## set is builtin in python 2.6.4 and sets is deprecated
11     set
12     except NameError:
13     from sets import Set
14    
15     class MenuAnalyzer:
16     def __init__(self,name):
17     ##default ranges
18     self.maxModuleNameLength = 300
19     self.maxModulesPerPath = 50
20     self.maxPaths = 500
21     self.maxEndPaths = 30
22    
23     ##required streams
24     self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
25 amott 1.3 'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
26 amott 1.1 self.ExpressStreamName = 'Express'
27     self.expressPDs = { 'ExpressPhysics' : 'Collisions',
28     'ExpressCosmics' : 'Cosmics' }
29    
30     self.expressType = ''
31    
32     self.menuName = name
33     self.endPathList = set()
34     self.perPathModuleList={}
35     self.perModuleTypeList={}
36     self.perStreamPDList={}
37     self.perPDPathList={}
38     self.Results={}
39     self.ModuleList=[]
40 amott 1.6 self.ESModuleList=[]
41 amott 1.5 self.eventContent={}
42    
43 amott 1.1 self.AnalysisList=[]
44    
45    
46     ## statically define the analysis map: new analyses must be registered here
47     self.AnalysisMap = {
48     'moduleLength' : self.checkModuleLength,
49     'numberOfPaths' : self.checkNumPaths,
50     'numberOfEndPaths' : self.checkNumEndPaths,
51     'reqStreamsAndPDs' : self.reqStreamsAndPDs,
52     'checkExpress' : self.checkExpress,
53 amott 1.5 'checkNameFormats' :self.checkNameFormats,
54 amott 1.6 'checkEventContent':self.checkEventContent,
55     'checkL1Unmask':self.checkL1Unmask
56 amott 1.1 }
57 amott 1.2 self.ProblemDescriptions = {
58     'moduleLength':'Modules too long',
59     'numberOfPaths':'Too many paths',
60     'numberOfEndPaths':'Too many endpaths',
61     'reqStreamsAndPDs':'Missing required stream/PD',
62     'checkExpress' : 'Invalid or missing express stream/PD',
63 amott 1.5 'checkNameFormats' : 'Invalid PD or path name format',
64 amott 1.6 'checkEventContent' : 'Invalid Event Content',
65     'checkL1Unmask' : 'L1 Unmask Module in Menu'
66 amott 1.2 }
67 amott 1.1
68     self.T0REGEXP = { ## These are the regexps that T0 uses to access things
69     # Regexp for save file name paths. We don't process anything else.
70     'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
71     # Regexp for valid dataset names.
72     'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
73     # Regexp for valid RelVal dataset names.
74     'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
75     'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
76     # Regexp for online DQM files.
77     'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
78     # Regexp for offline DQM files.
79     'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
80     # Regexp for acquisition era part of the processed dataset name.
81     'RXERA' : re.compile("^([A-Za-z]+\d+|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
82     }
83    
84     def AddAnalysis(self,name):
85     self.AnalysisList.append(name)
86    
87     def AddAllAnalyses(self):
88     for name in self.AnalysisMap.iterkeys():
89     self.AddAnalysis(name)
90    
91     def Analyze(self):
92     cursor = ConnectDB('hlt')
93     self.GetModules(cursor)
94 amott 1.6 self.GetESModules(cursor)
95 amott 1.1 self.GetStreamsPathsPDs(cursor)
96 amott 1.5 self.GetEventContent(cursor)
97 amott 1.1 for analysis in self.AnalysisList:
98     if not self.AnalysisMap.has_key(analysis):
99     print "ERROR: Analysis %s not defined" % (analysis,)
100     continue
101     self.AnalysisMap[analysis]()
102    
103    
104    
105     def checkModuleLength(self):
106     self.Results['moduleLength'] = []
107     for modName,type in self.perModuleTypeList.iteritems():
108     if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
109    
110     def checkNumPaths(self):
111     if len(self.perPathModuleList) > self.maxPaths:
112     self.Results['numberOfPaths'] = len(self.perPathModuleList)
113     else:
114     self.Results['numberOfPaths'] = 0
115     def checkNumEndPaths(self):
116     if len(self.endPathList) > self.maxEndPaths:
117     self.Results['numberOfEndPaths'] = len(self.endPathList)
118     else:
119     self.Results['numberOfEndPaths'] = 0
120     def reqStreamsAndPDs(self):
121     self.Results['reqStreamsAndPDs'] = []
122     for stream,PDList in self.requiredStreamsAndPDs.iteritems():
123     if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
124     for PD in PDList:
125     if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
126    
127     def checkExpress(self):
128     self.Results['checkExpress'] = []
129     if not self.perStreamPDList.has_key(self.ExpressStreamName):
130     self.Results['checkExpress'].append(self.ExpressStreamName)
131     return
132    
133     if len(self.perStreamPDList[self.ExpressStreamName]) >1:
134     self.Results['checkExpress'].append("MULTIPLE_PDS")
135     if len(self.perStreamPDList[self.ExpressStreamName]) <1:
136     self.Results['checkExpress'].append("NO_PDS")
137    
138     for PD in self.perStreamPDList[self.ExpressStreamName]:
139     if not self.expressPDs.has_key(PD):
140     self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
141     else:
142     self.expressType = self.expressPDs[PD]
143    
144    
145    
146     def checkNameFormats(self):
147     self.Results['checkNameFormats']=[]
148     for PD in self.perPDPathList.iterkeys():
149     if not self.T0REGEXP['RXDATASET'].match(PD):
150     self.Results['checkNameFormats'].append(PD)
151     for path in self.perPathModuleList.iterkeys():
152     if not self.T0REGEXP['RXSAFEPATH'].match(path):
153     self.Results['checkNameFormats'].append(path)
154 amott 1.5
155     def checkEventContent(self):
156     self.Results['checkEventContent']=[]
157     for stream,content in self.eventContent.iteritems():
158     #first check for a drop statement
159     if not ('drop *' in content or 'drop *_hlt*_*_*' in content):
160     self.Results['checkEventContent'].append(stream+'::drop *')
161     if not eventContent.requiredEventContent.has_key(stream): continue
162     requiredContent = eventContent.requiredEventContent[stream]
163     for entry in requiredContent:
164     if not entry in content:
165     self.Results['checkEventContent'].append(stream+'::'+entry)
166 amott 1.6
167     def checkL1Unmask(self):
168     self.Results['checkL1Unmask']=[]
169     if 'L1GtTriggerMaskAlgoTrigTrivialProducer' in self.ESModuleList:
170     self.Results['checkL1Unmask'].append('L1GtTriggerMaskAlgoTrigTrivialProducer')
171     if 'L1GtTriggerMaskTechTrigTrivialProducer' in self.ESModuleList:
172     self.Results['checkL1Unmask'].append('L1GtTriggerMaskTechTrigTrivialProducer')
173    
174 amott 1.1
175     def GetModules(self,cursor):
176     sqlquery ="""
177     SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
178     FROM
179     CMS_HLT.PARAMETERS B,
180     CMS_HLT.SUPERIDPARAMETERASSOC C,
181     CMS_HLT.MODULETEMPLATES D,
182     CMS_HLT.MODULES E,
183     CMS_HLT.PATHMODULEASSOC F,
184     CMS_HLT.CONFIGURATIONPATHASSOC G,
185     CMS_HLT.CONFIGURATIONS H,
186     CMS_HLT.PATHS I
187     WHERE
188     B.PARAMID = C.PARAMID AND
189     C.SUPERID = F.MODULEID AND
190     E.TEMPLATEID = D.SUPERID AND
191     F.MODULEID = E.SUPERID AND
192     F.PATHID=G.PATHID AND
193     I.PATHID=G.PATHID AND
194     G.CONFIGID=H.CONFIGID AND
195     H.CONFIGDESCRIPTOR='%s'
196     """ % (self.menuName,)
197    
198     cursor.execute(sqlquery)
199     for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
200     if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
201     self.perPathModuleList[PathName].append(ModuleName)
202     self.perModuleTypeList[ModuleName] = ModuleType
203 amott 1.6 if not ModuleName in self.ModuleList: self.ModuleList.append(ModuleName)
204 amott 1.1 if endPath: self.endPathList.add(PathName)
205    
206    
207     def GetStreamsPathsPDs(self,cursor):
208     sqlquery= """
209     SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
210     FROM
211     CMS_HLT.STREAMS A,
212     CMS_HLT.CONFIGURATIONS B,
213     CMS_HLT.CONFIGURATIONPATHASSOC C,
214     CMS_HLT.PATHSTREAMDATASETASSOC D,
215     CMS_HLT.PATHS E,
216     CMS_HLT.PRIMARYDATASETS F
217     WHERE
218     B.CONFIGDESCRIPTOR='%s' AND
219     C.CONFIGID=B.CONFIGID AND
220     D.PATHID=C.PATHID AND
221     A.STREAMID=D.STREAMID AND
222     E.PATHID = C.PATHID AND
223     F.DATASETID = D.DATASETID
224     """ % (self.menuName,)
225    
226     cursor.execute(sqlquery)
227     for StreamName,PathName,PDName in cursor.fetchall():
228     if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
229     if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
230     if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
231     self.perPDPathList[PDName].append(PathName)
232    
233 amott 1.6 def GetESModules(self,cursor):
234     sqlquery = """
235     SELECT UNIQUE(F.NAME)
236     FROM
237     CMS_HLT.ESMODULES F,
238     CMS_HLT.CONFIGURATIONESMODULEASSOC G,
239     CMS_HLT.CONFIGURATIONS H
240     WHERE
241     G.ESMODULEID = F.SUPERID AND
242     G.CONFIGID=H.CONFIGID AND
243     H.CONFIGDESCRIPTOR='%s'
244     """ % (self.menuName,)
245    
246     cursor.execute(sqlquery)
247     for ModuleName, in cursor.fetchall():
248     if not ModuleName in self.ESModuleList: self.ESModuleList.append(ModuleName)
249    
250 amott 1.4 def GetEventContent(self,cursor):
251 amott 1.5 sqlquery = """
252     SELECT A.STREAMLABEL,H.STATEMENTTYPE,H.CLASSN,H.MODULEL,H.EXTRAN,H.PROCESSN
253 amott 1.4 FROM
254     CMS_HLT.STREAMS A,
255     CMS_HLT.CONFIGURATIONS B,
256 amott 1.5 CMS_HLT.CONFIGURATIONPATHASSOC C,
257     CMS_HLT.PATHSTREAMDATASETASSOC D,
258     CMS_HLT.PATHS E,
259     CMS_HLT.ECSTREAMASSOC F,
260     CMS_HLT.ECSTATEMENTASSOC G,
261     CMS_HLT.EVENTCONTENTSTATEMENTS H
262 amott 1.4 WHERE
263     B.CONFIGDESCRIPTOR='%s' AND
264     C.CONFIGID=B.CONFIGID AND
265     D.PATHID=C.PATHID AND
266     A.STREAMID=D.STREAMID AND
267     E.PATHID = C.PATHID AND
268 amott 1.5 F.STREAMID = D.STREAMID AND
269     G.EVENTCONTENTID=F.EVENTCONTENTID AND
270     H.STATEMENTID=G.STATEMENTID
271 amott 1.4 """ % (self.menuName,)
272 amott 1.5
273     cursor.execute(sqlquery)
274     for stream,keep,Class,module,extra,process in cursor.fetchall():
275     if not self.eventContent.has_key(stream): self.eventContent[stream]=[]
276     statement = "%s_%s_%s_%s" % (Class,module,extra,process,)
277     if statement == "*_*_*_*": statement = "*"
278     if keep == 1: statement = "keep "+statement
279     else: statement = "drop "+statement
280     if not statement in self.eventContent[stream]:
281     self.eventContent[stream].append( statement )