1 |
amott |
1.1 |
#!/usr/bin/env python
|
2 |
|
|
|
3 |
|
|
import os
|
4 |
|
|
import sys
|
5 |
|
|
from DatabaseParser import ConnectDB
|
6 |
|
|
import re
|
7 |
|
|
import cx_Oracle
|
8 |
amott |
1.5 |
import eventContent
|
9 |
amott |
1.1 |
|
10 |
|
|
try: ## set is builtin in python 2.6.4 and sets is deprecated
|
11 |
|
|
set
|
12 |
|
|
except NameError:
|
13 |
|
|
from sets import Set
|
14 |
|
|
|
15 |
|
|
class MenuAnalyzer:
|
16 |
|
|
def __init__(self,name):
|
17 |
|
|
##default ranges
|
18 |
|
|
self.maxModuleNameLength = 300
|
19 |
|
|
self.maxModulesPerPath = 50
|
20 |
|
|
self.maxPaths = 500
|
21 |
|
|
self.maxEndPaths = 30
|
22 |
|
|
|
23 |
|
|
##required streams
|
24 |
|
|
self.requiredStreamsAndPDs = { 'Calibration' : ['TestEnablesEcalHcalDT'],'EcalCalibration' : ['EcalLaser'],
|
25 |
amott |
1.3 |
'TrackerCalibration' : ['TestEnablesTracker'], 'DQM' : 'OnlineMonitor'}
|
26 |
amott |
1.1 |
self.ExpressStreamName = 'Express'
|
27 |
|
|
self.expressPDs = { 'ExpressPhysics' : 'Collisions',
|
28 |
|
|
'ExpressCosmics' : 'Cosmics' }
|
29 |
|
|
|
30 |
|
|
self.expressType = ''
|
31 |
|
|
|
32 |
|
|
self.menuName = name
|
33 |
|
|
self.endPathList = set()
|
34 |
|
|
self.perPathModuleList={}
|
35 |
|
|
self.perModuleTypeList={}
|
36 |
|
|
self.perStreamPDList={}
|
37 |
|
|
self.perPDPathList={}
|
38 |
|
|
self.Results={}
|
39 |
|
|
self.ModuleList=[]
|
40 |
amott |
1.6 |
self.ESModuleList=[]
|
41 |
amott |
1.5 |
self.eventContent={}
|
42 |
|
|
|
43 |
amott |
1.1 |
self.AnalysisList=[]
|
44 |
|
|
|
45 |
|
|
|
46 |
|
|
## statically define the analysis map: new analyses must be registered here
|
47 |
|
|
self.AnalysisMap = {
|
48 |
|
|
'moduleLength' : self.checkModuleLength,
|
49 |
|
|
'numberOfPaths' : self.checkNumPaths,
|
50 |
|
|
'numberOfEndPaths' : self.checkNumEndPaths,
|
51 |
|
|
'reqStreamsAndPDs' : self.reqStreamsAndPDs,
|
52 |
|
|
'checkExpress' : self.checkExpress,
|
53 |
amott |
1.5 |
'checkNameFormats' :self.checkNameFormats,
|
54 |
amott |
1.6 |
'checkEventContent':self.checkEventContent,
|
55 |
|
|
'checkL1Unmask':self.checkL1Unmask
|
56 |
amott |
1.1 |
}
|
57 |
amott |
1.2 |
self.ProblemDescriptions = {
|
58 |
|
|
'moduleLength':'Modules too long',
|
59 |
|
|
'numberOfPaths':'Too many paths',
|
60 |
|
|
'numberOfEndPaths':'Too many endpaths',
|
61 |
|
|
'reqStreamsAndPDs':'Missing required stream/PD',
|
62 |
|
|
'checkExpress' : 'Invalid or missing express stream/PD',
|
63 |
amott |
1.5 |
'checkNameFormats' : 'Invalid PD or path name format',
|
64 |
amott |
1.6 |
'checkEventContent' : 'Invalid Event Content',
|
65 |
|
|
'checkL1Unmask' : 'L1 Unmask Module in Menu'
|
66 |
amott |
1.2 |
}
|
67 |
amott |
1.1 |
|
68 |
|
|
self.T0REGEXP = { ## These are the regexps that T0 uses to access things
|
69 |
|
|
# Regexp for save file name paths. We don't process anything else.
|
70 |
|
|
'RXSAFEPATH' : re.compile("^[-A-Za-z0-9_]+$"),
|
71 |
|
|
# Regexp for valid dataset names.
|
72 |
|
|
'RXDATASET' : re.compile("^[-A-Za-z0-9_]+$"),
|
73 |
|
|
# Regexp for valid RelVal dataset names.
|
74 |
|
|
'RXRELVALMC' : re.compile("^/RelVal[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
|
75 |
|
|
'RXRELVALDATA' : re.compile("^/[^/]+/(CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)[-_].*$"),
|
76 |
|
|
# Regexp for online DQM files.
|
77 |
|
|
'RXONLINE' : re.compile("^(?:.*/)?DQM_V(\d+)(_[A-Za-z0-9]+)?_R(\d+)\.root$"),
|
78 |
|
|
# Regexp for offline DQM files.
|
79 |
|
|
'RXOFFLINE' : re.compile("^(?:.*/)?DQM_V(\d+)_R(\d+)((?:__[-A-Za-z0-9_]+){3})\.root$"),
|
80 |
|
|
# Regexp for acquisition era part of the processed dataset name.
|
81 |
|
|
'RXERA' : re.compile("^([A-Za-z]+\d+|CMSSW(?:_[0-9]+)+(?:_pre[0-9]+)?)")
|
82 |
|
|
}
|
83 |
|
|
|
84 |
|
|
def AddAnalysis(self,name):
|
85 |
|
|
self.AnalysisList.append(name)
|
86 |
|
|
|
87 |
|
|
def AddAllAnalyses(self):
|
88 |
|
|
for name in self.AnalysisMap.iterkeys():
|
89 |
|
|
self.AddAnalysis(name)
|
90 |
|
|
|
91 |
|
|
def Analyze(self):
|
92 |
|
|
cursor = ConnectDB('hlt')
|
93 |
|
|
self.GetModules(cursor)
|
94 |
amott |
1.6 |
self.GetESModules(cursor)
|
95 |
amott |
1.1 |
self.GetStreamsPathsPDs(cursor)
|
96 |
amott |
1.5 |
self.GetEventContent(cursor)
|
97 |
amott |
1.1 |
for analysis in self.AnalysisList:
|
98 |
|
|
if not self.AnalysisMap.has_key(analysis):
|
99 |
|
|
print "ERROR: Analysis %s not defined" % (analysis,)
|
100 |
|
|
continue
|
101 |
|
|
self.AnalysisMap[analysis]()
|
102 |
|
|
|
103 |
|
|
|
104 |
|
|
|
105 |
|
|
def checkModuleLength(self):
|
106 |
|
|
self.Results['moduleLength'] = []
|
107 |
|
|
for modName,type in self.perModuleTypeList.iteritems():
|
108 |
|
|
if len(modName) > self.maxModuleNameLength: self.Results['moduleLength'].append(modName)
|
109 |
|
|
|
110 |
|
|
def checkNumPaths(self):
|
111 |
|
|
if len(self.perPathModuleList) > self.maxPaths:
|
112 |
|
|
self.Results['numberOfPaths'] = len(self.perPathModuleList)
|
113 |
|
|
else:
|
114 |
|
|
self.Results['numberOfPaths'] = 0
|
115 |
|
|
def checkNumEndPaths(self):
|
116 |
|
|
if len(self.endPathList) > self.maxEndPaths:
|
117 |
|
|
self.Results['numberOfEndPaths'] = len(self.endPathList)
|
118 |
|
|
else:
|
119 |
|
|
self.Results['numberOfEndPaths'] = 0
|
120 |
|
|
def reqStreamsAndPDs(self):
|
121 |
|
|
self.Results['reqStreamsAndPDs'] = []
|
122 |
|
|
for stream,PDList in self.requiredStreamsAndPDs.iteritems():
|
123 |
|
|
if not self.perStreamPDList.has_key(stream): self.Results['reqStreamsAndPDs'].append(stream)
|
124 |
|
|
for PD in PDList:
|
125 |
|
|
if not PD in self.requiredStreamsAndPDs[stream]: self.Results['reqStreamsAndPDs'].append(stream+'::'+PD)
|
126 |
|
|
|
127 |
|
|
def checkExpress(self):
|
128 |
|
|
self.Results['checkExpress'] = []
|
129 |
|
|
if not self.perStreamPDList.has_key(self.ExpressStreamName):
|
130 |
|
|
self.Results['checkExpress'].append(self.ExpressStreamName)
|
131 |
|
|
return
|
132 |
|
|
|
133 |
|
|
if len(self.perStreamPDList[self.ExpressStreamName]) >1:
|
134 |
|
|
self.Results['checkExpress'].append("MULTIPLE_PDS")
|
135 |
|
|
if len(self.perStreamPDList[self.ExpressStreamName]) <1:
|
136 |
|
|
self.Results['checkExpress'].append("NO_PDS")
|
137 |
|
|
|
138 |
|
|
for PD in self.perStreamPDList[self.ExpressStreamName]:
|
139 |
|
|
if not self.expressPDs.has_key(PD):
|
140 |
|
|
self.Results['checkExpress'].append(self.ExpressStreamName+"::"+PD)
|
141 |
|
|
else:
|
142 |
|
|
self.expressType = self.expressPDs[PD]
|
143 |
|
|
|
144 |
|
|
|
145 |
|
|
|
146 |
|
|
def checkNameFormats(self):
|
147 |
|
|
self.Results['checkNameFormats']=[]
|
148 |
|
|
for PD in self.perPDPathList.iterkeys():
|
149 |
|
|
if not self.T0REGEXP['RXDATASET'].match(PD):
|
150 |
|
|
self.Results['checkNameFormats'].append(PD)
|
151 |
|
|
for path in self.perPathModuleList.iterkeys():
|
152 |
|
|
if not self.T0REGEXP['RXSAFEPATH'].match(path):
|
153 |
|
|
self.Results['checkNameFormats'].append(path)
|
154 |
amott |
1.5 |
|
155 |
|
|
def checkEventContent(self):
|
156 |
|
|
self.Results['checkEventContent']=[]
|
157 |
|
|
for stream,content in self.eventContent.iteritems():
|
158 |
|
|
#first check for a drop statement
|
159 |
|
|
if not ('drop *' in content or 'drop *_hlt*_*_*' in content):
|
160 |
|
|
self.Results['checkEventContent'].append(stream+'::drop *')
|
161 |
|
|
if not eventContent.requiredEventContent.has_key(stream): continue
|
162 |
|
|
requiredContent = eventContent.requiredEventContent[stream]
|
163 |
|
|
for entry in requiredContent:
|
164 |
|
|
if not entry in content:
|
165 |
|
|
self.Results['checkEventContent'].append(stream+'::'+entry)
|
166 |
amott |
1.6 |
|
167 |
|
|
def checkL1Unmask(self):
|
168 |
|
|
self.Results['checkL1Unmask']=[]
|
169 |
|
|
if 'L1GtTriggerMaskAlgoTrigTrivialProducer' in self.ESModuleList:
|
170 |
|
|
self.Results['checkL1Unmask'].append('L1GtTriggerMaskAlgoTrigTrivialProducer')
|
171 |
|
|
if 'L1GtTriggerMaskTechTrigTrivialProducer' in self.ESModuleList:
|
172 |
|
|
self.Results['checkL1Unmask'].append('L1GtTriggerMaskTechTrigTrivialProducer')
|
173 |
|
|
|
174 |
amott |
1.1 |
|
175 |
|
|
def GetModules(self,cursor):
|
176 |
|
|
sqlquery ="""
|
177 |
|
|
SELECT I.NAME,E.NAME,D.NAME,I.ISENDPATH
|
178 |
|
|
FROM
|
179 |
|
|
CMS_HLT.PARAMETERS B,
|
180 |
|
|
CMS_HLT.SUPERIDPARAMETERASSOC C,
|
181 |
|
|
CMS_HLT.MODULETEMPLATES D,
|
182 |
|
|
CMS_HLT.MODULES E,
|
183 |
|
|
CMS_HLT.PATHMODULEASSOC F,
|
184 |
|
|
CMS_HLT.CONFIGURATIONPATHASSOC G,
|
185 |
|
|
CMS_HLT.CONFIGURATIONS H,
|
186 |
|
|
CMS_HLT.PATHS I
|
187 |
|
|
WHERE
|
188 |
|
|
B.PARAMID = C.PARAMID AND
|
189 |
|
|
C.SUPERID = F.MODULEID AND
|
190 |
|
|
E.TEMPLATEID = D.SUPERID AND
|
191 |
|
|
F.MODULEID = E.SUPERID AND
|
192 |
|
|
F.PATHID=G.PATHID AND
|
193 |
|
|
I.PATHID=G.PATHID AND
|
194 |
|
|
G.CONFIGID=H.CONFIGID AND
|
195 |
|
|
H.CONFIGDESCRIPTOR='%s'
|
196 |
|
|
""" % (self.menuName,)
|
197 |
|
|
|
198 |
|
|
cursor.execute(sqlquery)
|
199 |
|
|
for PathName,ModuleName,ModuleType,endPath in cursor.fetchall():
|
200 |
|
|
if not self.perPathModuleList.has_key(PathName): self.perPathModuleList[PathName] = []
|
201 |
|
|
self.perPathModuleList[PathName].append(ModuleName)
|
202 |
|
|
self.perModuleTypeList[ModuleName] = ModuleType
|
203 |
amott |
1.6 |
if not ModuleName in self.ModuleList: self.ModuleList.append(ModuleName)
|
204 |
amott |
1.1 |
if endPath: self.endPathList.add(PathName)
|
205 |
|
|
|
206 |
|
|
|
207 |
|
|
def GetStreamsPathsPDs(self,cursor):
|
208 |
|
|
sqlquery= """
|
209 |
|
|
SELECT A.STREAMLABEL,E.NAME,F.DATASETLABEL
|
210 |
|
|
FROM
|
211 |
|
|
CMS_HLT.STREAMS A,
|
212 |
|
|
CMS_HLT.CONFIGURATIONS B,
|
213 |
|
|
CMS_HLT.CONFIGURATIONPATHASSOC C,
|
214 |
|
|
CMS_HLT.PATHSTREAMDATASETASSOC D,
|
215 |
|
|
CMS_HLT.PATHS E,
|
216 |
|
|
CMS_HLT.PRIMARYDATASETS F
|
217 |
|
|
WHERE
|
218 |
|
|
B.CONFIGDESCRIPTOR='%s' AND
|
219 |
|
|
C.CONFIGID=B.CONFIGID AND
|
220 |
|
|
D.PATHID=C.PATHID AND
|
221 |
|
|
A.STREAMID=D.STREAMID AND
|
222 |
|
|
E.PATHID = C.PATHID AND
|
223 |
|
|
F.DATASETID = D.DATASETID
|
224 |
|
|
""" % (self.menuName,)
|
225 |
|
|
|
226 |
|
|
cursor.execute(sqlquery)
|
227 |
|
|
for StreamName,PathName,PDName in cursor.fetchall():
|
228 |
|
|
if not self.perStreamPDList.has_key(StreamName): self.perStreamPDList[StreamName] = []
|
229 |
|
|
if not PDName in self.perStreamPDList[StreamName]: self.perStreamPDList[StreamName].append(PDName)
|
230 |
|
|
if not self.perPDPathList.has_key(PDName): self.perPDPathList[PDName] = []
|
231 |
|
|
self.perPDPathList[PDName].append(PathName)
|
232 |
|
|
|
233 |
amott |
1.6 |
def GetESModules(self,cursor):
|
234 |
|
|
sqlquery = """
|
235 |
|
|
SELECT UNIQUE(F.NAME)
|
236 |
|
|
FROM
|
237 |
|
|
CMS_HLT.ESMODULES F,
|
238 |
|
|
CMS_HLT.CONFIGURATIONESMODULEASSOC G,
|
239 |
|
|
CMS_HLT.CONFIGURATIONS H
|
240 |
|
|
WHERE
|
241 |
|
|
G.ESMODULEID = F.SUPERID AND
|
242 |
|
|
G.CONFIGID=H.CONFIGID AND
|
243 |
|
|
H.CONFIGDESCRIPTOR='%s'
|
244 |
|
|
""" % (self.menuName,)
|
245 |
|
|
|
246 |
|
|
cursor.execute(sqlquery)
|
247 |
|
|
for ModuleName, in cursor.fetchall():
|
248 |
|
|
if not ModuleName in self.ESModuleList: self.ESModuleList.append(ModuleName)
|
249 |
|
|
|
250 |
amott |
1.4 |
def GetEventContent(self,cursor):
|
251 |
amott |
1.5 |
sqlquery = """
|
252 |
|
|
SELECT A.STREAMLABEL,H.STATEMENTTYPE,H.CLASSN,H.MODULEL,H.EXTRAN,H.PROCESSN
|
253 |
amott |
1.4 |
FROM
|
254 |
|
|
CMS_HLT.STREAMS A,
|
255 |
|
|
CMS_HLT.CONFIGURATIONS B,
|
256 |
amott |
1.5 |
CMS_HLT.CONFIGURATIONPATHASSOC C,
|
257 |
|
|
CMS_HLT.PATHSTREAMDATASETASSOC D,
|
258 |
|
|
CMS_HLT.PATHS E,
|
259 |
|
|
CMS_HLT.ECSTREAMASSOC F,
|
260 |
|
|
CMS_HLT.ECSTATEMENTASSOC G,
|
261 |
|
|
CMS_HLT.EVENTCONTENTSTATEMENTS H
|
262 |
amott |
1.4 |
WHERE
|
263 |
|
|
B.CONFIGDESCRIPTOR='%s' AND
|
264 |
|
|
C.CONFIGID=B.CONFIGID AND
|
265 |
|
|
D.PATHID=C.PATHID AND
|
266 |
|
|
A.STREAMID=D.STREAMID AND
|
267 |
|
|
E.PATHID = C.PATHID AND
|
268 |
amott |
1.5 |
F.STREAMID = D.STREAMID AND
|
269 |
|
|
G.EVENTCONTENTID=F.EVENTCONTENTID AND
|
270 |
|
|
H.STATEMENTID=G.STATEMENTID
|
271 |
amott |
1.4 |
""" % (self.menuName,)
|
272 |
amott |
1.5 |
|
273 |
|
|
cursor.execute(sqlquery)
|
274 |
|
|
for stream,keep,Class,module,extra,process in cursor.fetchall():
|
275 |
|
|
if not self.eventContent.has_key(stream): self.eventContent[stream]=[]
|
276 |
|
|
statement = "%s_%s_%s_%s" % (Class,module,extra,process,)
|
277 |
|
|
if statement == "*_*_*_*": statement = "*"
|
278 |
|
|
if keep == 1: statement = "keep "+statement
|
279 |
|
|
else: statement = "drop "+statement
|
280 |
|
|
if not statement in self.eventContent[stream]:
|
281 |
|
|
self.eventContent[stream].append( statement )
|