ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/RateMonShiftTool_dev/MenuAnalyzer.py
Revision: 1.7
Committed: Sun Aug 12 14:23:55 2012 UTC (12 years, 8 months ago) by amott
Content type: text/x-python
Branch: MAIN
CVS Tags: V00-02-01, V00-01-10, V-00-01-10, V00-01-09, V00-01-08
Changes since 1.6: +6 -1 lines
Log Message:
added explicit stream listing for invalid PD names

File Contents

# Content
1 #!/usr/bin/env python
2
3 import os
4 import sys
5 from DatabaseParser import ConnectDB
6 import re
7 import cx_Oracle
8 import eventContent
9
10 try: ## set is builtin in python 2.6.4 and sets is deprecated
11 set
12 except NameError:
13 from sets import Set
14
15 class MenuAnalyzer:
16 def __init__(self,name):
17 ##default ranges
18 self.maxModuleNameLength = 300
19 self.maxModulesPerPath = 50
20 self.maxPaths = 500
21 self.maxEndPaths = 30
22
23 ##required streams
24 self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
25 'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
26 self.ExpressStreamName = 'Express'
27 self.expressPDs = { 'ExpressPhysics' : 'Collisions',
28 'ExpressCosmics' : 'Cosmics' }
29
30 self.expressType = ''
31
32 self.menuName = name
33 self.endPathList = set()
34 self.perPathModuleList={}
35 self.perModuleTypeList={}
36 self.perStreamPDList={}
37 self.perPDPathList={}
38 self.Results={}
39 self.ModuleList=[]
40 self.ESModuleList=[]
41 self.eventContent={}
42
43 self.AnalysisList=[]
44
45
46 ## statically define the analysis map: new analyses must be registered here
47 self.AnalysisMap = {
48 'moduleLength' : self.checkModuleLength,
49 'numberOfPaths' : self.checkNumPaths,
50 'numberOfEndPaths' : self.checkNumEndPaths,
51 'reqStreamsAndPDs' : self.reqStreamsAndPDs,
52 'checkExpress' : self.checkExpress,
53 'checkNameFormats' :self.checkNameFormats,
54 'checkEventContent':self.checkEventContent,
55 'checkL1Unmask':self.checkL1Unmask
56 }
57 self.ProblemDescriptions = {
58 'moduleLength':'Modules too long',
59 'numberOfPaths':'Too many paths',
60 'numberOfEndPaths':'Too many endpaths',
61 'reqStreamsAndPDs':'Missing required stream/PD',
62 'checkExpress' : 'Invalid or missing express stream/PD',
63 'checkNameFormats' : 'Invalid PD or path name format',
64 'checkEventContent' : 'Invalid Event Content',
65 'checkL1Unmask' : 'L1 Unmask Module in Menu'
66 }
67
68 self.T0REGEXP = { ## These are the regexps that T0 uses to access things
69 # Regexp for save file name paths. We don't process anything else.
70 'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
71 # Regexp for valid dataset names.
72 'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
73 # Regexp for valid RelVal dataset names.
74 'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
75 'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
76 # Regexp for online DQM files.
77 'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
78 # Regexp for offline DQM files.
79 'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
80 # Regexp for acquisition era part of the processed dataset name.
81 'RXERA' : re.compile("^([A-Za-z]+\d+|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
82 }
83
84 def AddAnalysis(self,name):
85 self.AnalysisList.append(name)
86
87 def AddAllAnalyses(self):
88 for name in self.AnalysisMap.iterkeys():
89 self.AddAnalysis(name)
90
91 def Analyze(self):
92 cursor = ConnectDB('hlt')
93 self.GetModules(cursor)
94 self.GetESModules(cursor)
95 self.GetStreamsPathsPDs(cursor)
96 self.GetEventContent(cursor)
97 for analysis in self.AnalysisList:
98 if not self.AnalysisMap.has_key(analysis):
99 print "ERROR: Analysis %s not defined" % (analysis,)
100 continue
101 self.AnalysisMap[analysis]()
102
103
104
105 def checkModuleLength(self):
106 self.Results['moduleLength'] = []
107 for modName,type in self.perModuleTypeList.iteritems():
108 if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
109
110 def checkNumPaths(self):
111 if len(self.perPathModuleList) > self.maxPaths:
112 self.Results['numberOfPaths'] = len(self.perPathModuleList)
113 else:
114 self.Results['numberOfPaths'] = 0
115 def checkNumEndPaths(self):
116 if len(self.endPathList) > self.maxEndPaths:
117 self.Results['numberOfEndPaths'] = len(self.endPathList)
118 else:
119 self.Results['numberOfEndPaths'] = 0
120 def reqStreamsAndPDs(self):
121 self.Results['reqStreamsAndPDs'] = []
122 for stream,PDList in self.requiredStreamsAndPDs.iteritems():
123 if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
124 for PD in PDList:
125 if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
126
127 def checkExpress(self):
128 self.Results['checkExpress'] = []
129 if not self.perStreamPDList.has_key(self.ExpressStreamName):
130 self.Results['checkExpress'].append(self.ExpressStreamName)
131 return
132
133 if len(self.perStreamPDList[self.ExpressStreamName]) >1:
134 self.Results['checkExpress'].append("MULTIPLE_PDS")
135 if len(self.perStreamPDList[self.ExpressStreamName]) <1:
136 self.Results['checkExpress'].append("NO_PDS")
137
138 for PD in self.perStreamPDList[self.ExpressStreamName]:
139 if not self.expressPDs.has_key(PD):
140 self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
141 else:
142 self.expressType = self.expressPDs[PD]
143
144
145
146 def checkNameFormats(self):
147 self.Results['checkNameFormats']=[]
148 for PD in self.perPDPathList.iterkeys():
149 if not self.T0REGEXP['RXDATASET'].match(PD):
150 for k,v in self.perStreamPDList.iteritems():
151 if PD in v:
152 self.Results['checkNameFormats'].append(k+"::"+PD)
153 break
154
155 self.Results['checkNameFormats'].append('NO STREAM::'+PD)
156 for path in self.perPathModuleList.iterkeys():
157 if not self.T0REGEXP['RXSAFEPATH'].match(path):
158 self.Results['checkNameFormats'].append(path)
159
160 def checkEventContent(self):
161 self.Results['checkEventContent']=[]
162 for stream,content in self.eventContent.iteritems():
163 #first check for a drop statement
164 if not ('drop *' in content or 'drop *_hlt*_*_*' in content):
165 self.Results['checkEventContent'].append(stream+'::drop *')
166 if not eventContent.requiredEventContent.has_key(stream): continue
167 requiredContent = eventContent.requiredEventContent[stream]
168 for entry in requiredContent:
169 if not entry in content:
170 self.Results['checkEventContent'].append(stream+'::'+entry)
171
172 def checkL1Unmask(self):
173 self.Results['checkL1Unmask']=[]
174 if 'L1GtTriggerMaskAlgoTrigTrivialProducer' in self.ESModuleList:
175 self.Results['checkL1Unmask'].append('L1GtTriggerMaskAlgoTrigTrivialProducer')
176 if 'L1GtTriggerMaskTechTrigTrivialProducer' in self.ESModuleList:
177 self.Results['checkL1Unmask'].append('L1GtTriggerMaskTechTrigTrivialProducer')
178
179
180 def GetModules(self,cursor):
181 sqlquery ="""
182 SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
183 FROM
184 CMS_HLT.PARAMETERS B,
185 CMS_HLT.SUPERIDPARAMETERASSOC C,
186 CMS_HLT.MODULETEMPLATES D,
187 CMS_HLT.MODULES E,
188 CMS_HLT.PATHMODULEASSOC F,
189 CMS_HLT.CONFIGURATIONPATHASSOC G,
190 CMS_HLT.CONFIGURATIONS H,
191 CMS_HLT.PATHS I
192 WHERE
193 B.PARAMID = C.PARAMID AND
194 C.SUPERID = F.MODULEID AND
195 E.TEMPLATEID = D.SUPERID AND
196 F.MODULEID = E.SUPERID AND
197 F.PATHID=G.PATHID AND
198 I.PATHID=G.PATHID AND
199 G.CONFIGID=H.CONFIGID AND
200 H.CONFIGDESCRIPTOR='%s'
201 """ % (self.menuName,)
202
203 cursor.execute(sqlquery)
204 for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
205 if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
206 self.perPathModuleList[PathName].append(ModuleName)
207 self.perModuleTypeList[ModuleName] = ModuleType
208 if not ModuleName in self.ModuleList: self.ModuleList.append(ModuleName)
209 if endPath: self.endPathList.add(PathName)
210
211
212 def GetStreamsPathsPDs(self,cursor):
213 sqlquery= """
214 SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
215 FROM
216 CMS_HLT.STREAMS A,
217 CMS_HLT.CONFIGURATIONS B,
218 CMS_HLT.CONFIGURATIONPATHASSOC C,
219 CMS_HLT.PATHSTREAMDATASETASSOC D,
220 CMS_HLT.PATHS E,
221 CMS_HLT.PRIMARYDATASETS F
222 WHERE
223 B.CONFIGDESCRIPTOR='%s' AND
224 C.CONFIGID=B.CONFIGID AND
225 D.PATHID=C.PATHID AND
226 A.STREAMID=D.STREAMID AND
227 E.PATHID = C.PATHID AND
228 F.DATASETID = D.DATASETID
229 """ % (self.menuName,)
230
231 cursor.execute(sqlquery)
232 for StreamName,PathName,PDName in cursor.fetchall():
233 if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
234 if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
235 if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
236 self.perPDPathList[PDName].append(PathName)
237
238 def GetESModules(self,cursor):
239 sqlquery = """
240 SELECT UNIQUE(F.NAME)
241 FROM
242 CMS_HLT.ESMODULES F,
243 CMS_HLT.CONFIGURATIONESMODULEASSOC G,
244 CMS_HLT.CONFIGURATIONS H
245 WHERE
246 G.ESMODULEID = F.SUPERID AND
247 G.CONFIGID=H.CONFIGID AND
248 H.CONFIGDESCRIPTOR='%s'
249 """ % (self.menuName,)
250
251 cursor.execute(sqlquery)
252 for ModuleName, in cursor.fetchall():
253 if not ModuleName in self.ESModuleList: self.ESModuleList.append(ModuleName)
254
255 def GetEventContent(self,cursor):
256 sqlquery = """
257 SELECT A.STREAMLABEL,H.STATEMENTTYPE,H.CLASSN,H.MODULEL,H.EXTRAN,H.PROCESSN
258 FROM
259 CMS_HLT.STREAMS A,
260 CMS_HLT.CONFIGURATIONS B,
261 CMS_HLT.CONFIGURATIONPATHASSOC C,
262 CMS_HLT.PATHSTREAMDATASETASSOC D,
263 CMS_HLT.PATHS E,
264 CMS_HLT.ECSTREAMASSOC F,
265 CMS_HLT.ECSTATEMENTASSOC G,
266 CMS_HLT.EVENTCONTENTSTATEMENTS H
267 WHERE
268 B.CONFIGDESCRIPTOR='%s' AND
269 C.CONFIGID=B.CONFIGID AND
270 D.PATHID=C.PATHID AND
271 A.STREAMID=D.STREAMID AND
272 E.PATHID = C.PATHID AND
273 F.STREAMID = D.STREAMID AND
274 G.EVENTCONTENTID=F.EVENTCONTENTID AND
275 H.STATEMENTID=G.STATEMENTID
276 """ % (self.menuName,)
277
278 cursor.execute(sqlquery)
279 for stream,keep,Class,module,extra,process in cursor.fetchall():
280 if not self.eventContent.has_key(stream): self.eventContent[stream]=[]
281 statement = "%s_%s_%s_%s" % (Class,module,extra,process,)
282 if statement == "*_*_*_*": statement = "*"
283 if keep == 1: statement = "keep "+statement
284 else: statement = "drop "+statement
285 if not statement in self.eventContent[stream]:
286 self.eventContent[stream].append( statement )