1 |
amott |
1.1 |
#!/usr/bin/env python
|
2 |
|
|
|
3 |
amott |
1.15 |
#from AndrewGetRun import GetRun
|
4 |
amott |
1.1 |
from ReadConfig import RateMonConfig
|
5 |
|
|
import sys
|
6 |
|
|
import os
|
7 |
|
|
import cPickle as pickle
|
8 |
|
|
import getopt
|
9 |
|
|
import time
|
10 |
awoodard |
1.49 |
from StreamMonitor import *
|
11 |
amott |
1.1 |
from colors import *
|
12 |
|
|
from TablePrint import *
|
13 |
|
|
from AddTableInfo_db import MoreTableInfo
|
14 |
|
|
from math import *
|
15 |
grchrist |
1.43 |
from DatabaseParser import *
|
16 |
muell149 |
1.48 |
from TablePrint import *
|
17 |
amott |
1.1 |
|
18 |
|
|
WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
|
19 |
|
|
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
|
20 |
grchrist |
1.34 |
RefRunNameTemplate = "RefRuns/%s/Run_%s.pk"
|
21 |
amott |
1.1 |
|
22 |
|
|
# define a function that clears the terminal screen
|
23 |
|
|
def clear():
|
24 |
|
|
print("\x1B[2J")
|
25 |
|
|
|
26 |
|
|
|
27 |
|
|
def usage():
|
28 |
|
|
print sys.argv[0]+" [Options]"
|
29 |
awoodard |
1.44 |
print "This script gets the current HLT trigger rates and compares them to a reference run or a fit to multiple runs"
|
30 |
amott |
1.1 |
print "Options: "
|
31 |
awoodard |
1.44 |
print "--AllowedPercDiff=<diff> Report only if difference in trigger rate is greater than <diff>%"
|
32 |
|
|
print "--AllowedSigmaDiff=<diff> Report only if difference in trigger rate is greater than <diff> standard deviations"
|
33 |
amott |
1.1 |
print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
|
34 |
|
|
print "--FindL1Zeros Look for physics paths with 0 L1 rate"
|
35 |
|
|
print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
|
36 |
|
|
print "--NumberLS=<#> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
|
37 |
|
|
print " or set LSSlidingWindow = -1"
|
38 |
|
|
print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate>"
|
39 |
|
|
print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
|
40 |
|
|
print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
|
41 |
|
|
print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
|
42 |
|
|
print "--ShowPSTriggers Show prescaled triggers in rate comparison"
|
43 |
amott |
1.33 |
print "--sortBy=<field> Sort the triggers by field. Valid fields are: name, rate, rateDiff"
|
44 |
amott |
1.2 |
print "--force Override the check for collisions run"
|
45 |
muell149 |
1.50 |
print "--write Writes rates to .csv file"
|
46 |
amott |
1.1 |
print "--help Print this help"
|
47 |
grchrist |
1.20 |
|
48 |
|
|
def pickYear():
|
49 |
|
|
global thisyear
|
50 |
grchrist |
1.23 |
thisyear="2012"
|
51 |
|
|
##print "Year set to ",thisyear
|
52 |
grchrist |
1.20 |
|
53 |
amott |
1.1 |
def main():
|
54 |
grchrist |
1.21 |
pickYear()
|
55 |
amott |
1.1 |
try:
|
56 |
awoodard |
1.44 |
opt, args = getopt.getopt(sys.argv[1:],"",["AllowedPercDiff=","AllowedSigmaDiff=","CompareRun=","FindL1Zeros",\
|
57 |
amott |
1.1 |
"FirstLS=","NumberLS=","IgnoreLowRate=","ListIgnoredPaths",\
|
58 |
muell149 |
1.50 |
"PrintLumi","RefRun=","ShowPSTriggers","force","sortBy=","write","help"])
|
59 |
amott |
1.1 |
except getopt.GetoptError, err:
|
60 |
|
|
print str(err)
|
61 |
|
|
usage()
|
62 |
|
|
sys.exit(2)
|
63 |
|
|
|
64 |
|
|
Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
|
65 |
|
|
for o,a in opt:
|
66 |
|
|
if o=="--ConfigFile":
|
67 |
|
|
Config.CFGfile=a
|
68 |
|
|
Config.ReadCFG()
|
69 |
grchrist |
1.35 |
|
70 |
|
|
|
71 |
|
|
if "NoV" in Config.FitFileName:
|
72 |
|
|
Config.NoVersion=True
|
73 |
grchrist |
1.43 |
#print "NoVersion=",Config.NoVersion
|
74 |
grchrist |
1.23 |
|
75 |
awoodard |
1.47 |
ShowSigmaAndPercDiff = Config.DefShowSigmaAndPercDiff
|
76 |
awoodard |
1.46 |
WarnOnSigmaDiff = Config.DefWarnOnSigmaDiff
|
77 |
awoodard |
1.44 |
AllowedRatePercDiff = Config.DefAllowRatePercDiff
|
78 |
|
|
AllowedRateSigmaDiff = Config.DefAllowRateSigmaDiff
|
79 |
amott |
1.1 |
CompareRunNum = ""
|
80 |
|
|
FindL1Zeros = False
|
81 |
|
|
FirstLS = 9999
|
82 |
amott |
1.2 |
NumLS = -10
|
83 |
amott |
1.1 |
IgnoreThreshold = Config.DefAllowIgnoreThresh
|
84 |
|
|
ListIgnoredPaths = False
|
85 |
|
|
PrintLumi = False
|
86 |
|
|
RefRunNum = int(Config.ReferenceRun)
|
87 |
|
|
ShowPSTriggers = True
|
88 |
amott |
1.2 |
Force = False
|
89 |
muell149 |
1.50 |
writeb = False
|
90 |
amott |
1.33 |
SortBy = ""
|
91 |
|
|
ShifterMode = int(Config.ShifterMode) # get this from the config, but can be overridden by other options
|
92 |
grchrist |
1.23 |
|
93 |
|
|
## if int(Config.ShifterMode):
|
94 |
|
|
## print "ShifterMode!!"
|
95 |
|
|
## else:
|
96 |
|
|
## print "ExpertMode"
|
97 |
amott |
1.1 |
|
98 |
|
|
if Config.LSWindow > 0:
|
99 |
|
|
NumLS = -1*Config.LSWindow
|
100 |
|
|
|
101 |
|
|
for o,a in opt: # get options passed on the command line
|
102 |
awoodard |
1.44 |
if o=="--AllowedPercDiff":
|
103 |
|
|
AllowedRatePercDiff = float(a)
|
104 |
|
|
elif o=="--AllowedSigmaDiff":
|
105 |
|
|
AllowedRateSigmaDiff = float(a)
|
106 |
amott |
1.1 |
elif o=="--CompareRun":
|
107 |
|
|
CompareRunNum=int(a)
|
108 |
amott |
1.33 |
ShifterMode = False
|
109 |
amott |
1.1 |
elif o=="--FindL1Zeros":
|
110 |
|
|
FindL1Zeros = True
|
111 |
|
|
elif o=="--FirstLS":
|
112 |
|
|
FirstLS = int(a)
|
113 |
amott |
1.33 |
ShifterMode = False
|
114 |
amott |
1.1 |
elif o=="--NumberLS":
|
115 |
|
|
NumLS = int(a)
|
116 |
|
|
elif o=="--IgnoreLowRate":
|
117 |
|
|
IgnoreThreshold = float(a)
|
118 |
|
|
elif o=="--ListIgnoredPaths":
|
119 |
|
|
ListIgnoredPaths=True
|
120 |
amott |
1.33 |
ShifterMode = False
|
121 |
amott |
1.1 |
elif o=="--PrintLumi":
|
122 |
|
|
PrintLumi = True
|
123 |
|
|
elif o=="--RefRun":
|
124 |
|
|
RefRunNum=int(a)
|
125 |
|
|
elif o=="--ShowPSTriggers":
|
126 |
|
|
ShowPSTriggers=True
|
127 |
amott |
1.33 |
elif o=="--sortBy":
|
128 |
|
|
SortBy = a
|
129 |
amott |
1.2 |
elif o=="--force":
|
130 |
|
|
Force = True
|
131 |
muell149 |
1.50 |
elif o=="--write":
|
132 |
|
|
writeb = True
|
133 |
amott |
1.1 |
elif o=="--help":
|
134 |
|
|
usage()
|
135 |
|
|
sys.exit(0)
|
136 |
|
|
else:
|
137 |
|
|
print "Invalid Option "+a
|
138 |
|
|
sys.exit(1)
|
139 |
|
|
|
140 |
grchrist |
1.23 |
|
141 |
amott |
1.1 |
RefLumisExists = False
|
142 |
|
|
"""
|
143 |
grchrist |
1.34 |
RefRunFile=RefRunNameTemplate % str(RefRunNum)
|
144 |
amott |
1.1 |
if RefRunNum > 0:
|
145 |
|
|
RefRates = {}
|
146 |
|
|
for Iterator in range(1,100):
|
147 |
|
|
if RefLumisExists: ## Quits at the end of a run
|
148 |
|
|
if max(RefLumis[0]) <= (Iterator+1)*10:
|
149 |
|
|
break
|
150 |
|
|
|
151 |
|
|
RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
|
152 |
|
|
print "RefRunFile=",RefRunFile
|
153 |
|
|
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
154 |
|
|
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
155 |
|
|
print RefRunFile[:RefRunFile.rfind('/')]
|
156 |
|
|
sys.exit(0)
|
157 |
|
|
|
158 |
|
|
if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
|
159 |
|
|
print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
|
160 |
|
|
print "Creating ..."
|
161 |
|
|
try:
|
162 |
|
|
RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
|
163 |
|
|
print "parsing"
|
164 |
|
|
except:
|
165 |
|
|
print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
166 |
|
|
continue
|
167 |
|
|
|
168 |
|
|
else: # otherwise load it from the file
|
169 |
|
|
RefParser = pickle.load( open( RefRunFile ) )
|
170 |
|
|
print "loading"
|
171 |
|
|
if not RefLumisExists:
|
172 |
|
|
RefLumis = RefParser.LumiInfo
|
173 |
|
|
RefLumisExists = True
|
174 |
|
|
|
175 |
|
|
try:
|
176 |
|
|
RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
|
177 |
|
|
LastSuccessfulIterator = Iterator
|
178 |
|
|
except:
|
179 |
|
|
print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
180 |
grchrist |
1.34 |
|
181 |
amott |
1.1 |
"""
|
182 |
grchrist |
1.34 |
RefRunFile = RefRunNameTemplate % (thisyear,RefRunNum)
|
183 |
amott |
1.1 |
RefParser = DatabaseParser()
|
184 |
grchrist |
1.23 |
##print "Reference Run: "+str(RefRunNum)
|
185 |
amott |
1.1 |
if RefRunNum > 0:
|
186 |
awoodard |
1.49 |
print "Getting RefRunFile",RefRunFile
|
187 |
amott |
1.1 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
188 |
|
|
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
189 |
|
|
print RefRunFile[:RefRunFile.rfind('/')]
|
190 |
grchrist |
1.27 |
sys.exit(0)
|
191 |
grchrist |
1.34 |
s
|
192 |
amott |
1.1 |
return
|
193 |
|
|
if not os.path.exists(RefRunFile):
|
194 |
grchrist |
1.34 |
print "RefRunFile does not exist"
|
195 |
amott |
1.1 |
# create the reference run file
|
196 |
|
|
try:
|
197 |
|
|
RefParser.RunNumber = RefRunNum
|
198 |
|
|
RefParser.ParseRunSetup()
|
199 |
grchrist |
1.34 |
print "RefParser is setup"
|
200 |
amott |
1.1 |
#RefParser.GetAllTriggerRatesByLS()
|
201 |
|
|
#RefParser.Save( RefRunFile )
|
202 |
|
|
except e:
|
203 |
|
|
print "PROBLEM GETTING REFERNCE RUN"
|
204 |
|
|
raise
|
205 |
|
|
else:
|
206 |
|
|
RefParser = pickle.load( open( RefRunFile ) )
|
207 |
|
|
|
208 |
|
|
# OK, Got the Reference Run
|
209 |
|
|
# Now get the most recent run
|
210 |
|
|
|
211 |
|
|
SaveRun = False
|
212 |
|
|
if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
|
213 |
grchrist |
1.22 |
CompareRunNum,isCol,isGood = GetLatestRunNumber()
|
214 |
grchrist |
1.26 |
|
215 |
|
|
|
216 |
grchrist |
1.22 |
if not isGood:
|
217 |
grchrist |
1.23 |
print "NO TRIGGER KEY FOUND for run ",CompareRunNum
|
218 |
grchrist |
1.22 |
|
219 |
grchrist |
1.23 |
##sys.exit(0)
|
220 |
amott |
1.1 |
|
221 |
|
|
if not isCol:
|
222 |
|
|
print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
|
223 |
amott |
1.16 |
print "Monitoring only stream A and Express"
|
224 |
|
|
#if not Force:
|
225 |
|
|
# sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
|
226 |
grchrist |
1.23 |
|
227 |
|
|
else:
|
228 |
|
|
print "Most Recent run is "+str(CompareRunNum)
|
229 |
amott |
1.16 |
else:
|
230 |
grchrist |
1.22 |
CompareRunNum,isCol,isGood = GetLatestRunNumber(CompareRunNum)
|
231 |
|
|
if not isGood:
|
232 |
grchrist |
1.23 |
print "NO TRIGGER KEY FOUND for run ", CompareRunNum
|
233 |
|
|
##sys.exit(0)
|
234 |
amott |
1.17 |
|
235 |
grchrist |
1.23 |
|
236 |
amott |
1.17 |
HeadParser = DatabaseParser()
|
237 |
|
|
HeadParser.RunNumber = CompareRunNum
|
238 |
grchrist |
1.23 |
|
239 |
|
|
try:
|
240 |
|
|
HeadParser.ParseRunSetup()
|
241 |
|
|
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
242 |
grchrist |
1.43 |
LastGoodLS=HeadParser.GetLastLS(isCol)+1
|
243 |
|
|
tempLastGoodLS=LastGoodLS
|
244 |
grchrist |
1.23 |
CurrRun=CompareRunNum
|
245 |
grchrist |
1.43 |
#print "done good"
|
246 |
grchrist |
1.23 |
except:
|
247 |
grchrist |
1.43 |
#print "exception"
|
248 |
grchrist |
1.23 |
HeadLumiRange=[]
|
249 |
|
|
LastGoodLS=-1
|
250 |
grchrist |
1.43 |
tempLastGoodLS=LastGoodLS-1
|
251 |
grchrist |
1.23 |
CurrRun=CompareRunNum
|
252 |
|
|
isGood=0
|
253 |
|
|
|
254 |
|
|
if len(HeadLumiRange) is 0:
|
255 |
grchrist |
1.36 |
print "No lumisections that are taking physics data 0"
|
256 |
grchrist |
1.23 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
257 |
|
|
if len(HeadLumiRange)>0:
|
258 |
|
|
isGood=1
|
259 |
|
|
isCol=0
|
260 |
|
|
##sys.exit(0)
|
261 |
awoodard |
1.44 |
|
262 |
awoodard |
1.45 |
## This reduces the sensitivity for a rate measurement to cause a warning during the beginning of a run
|
263 |
awoodard |
1.49 |
if len(HeadLumiRange) > 0 and len(HeadLumiRange) < 10:
|
264 |
awoodard |
1.44 |
AllowedRateSigmaDiff = AllowedRateSigmaDiff*10 / len(HeadLumiRange)
|
265 |
grchrist |
1.23 |
|
266 |
amott |
1.1 |
if PrintLumi:
|
267 |
|
|
for LS in HeadParser.LumiInfo[0]:
|
268 |
|
|
try:
|
269 |
|
|
if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
|
270 |
|
|
continue
|
271 |
|
|
print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
|
272 |
|
|
except:
|
273 |
|
|
print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
|
274 |
|
|
|
275 |
grchrist |
1.23 |
sys.exit(0)
|
276 |
amott |
1.1 |
|
277 |
|
|
if RefRunNum == 0:
|
278 |
|
|
RefRates = 0
|
279 |
|
|
RefLumis = 0
|
280 |
|
|
LastSuccessfulIterator = 0
|
281 |
|
|
|
282 |
|
|
### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
|
283 |
grchrist |
1.23 |
###isGood=1##if there is a trigger key
|
284 |
amott |
1.1 |
try:
|
285 |
|
|
while True:
|
286 |
grchrist |
1.23 |
|
287 |
|
|
if isGood:
|
288 |
grchrist |
1.43 |
tempLastGoodLS=LastGoodLS
|
289 |
grchrist |
1.23 |
LastGoodLS=HeadParser.GetLastLS(isCol)
|
290 |
grchrist |
1.43 |
##print "Last Good=",LastGoodLS, tempLastGoodLS
|
291 |
|
|
if LastGoodLS==tempLastGoodLS:
|
292 |
|
|
write(bcolors.FAIL)
|
293 |
|
|
print "Trying to get new Run"
|
294 |
|
|
write(bcolors.ENDC+"\n")
|
295 |
|
|
else:
|
296 |
|
|
RefMoreLumiArray = HeadParser.GetMoreLumiInfo()
|
297 |
|
|
isBeams=True
|
298 |
|
|
for lumisection in HeadLumiRange:
|
299 |
|
|
try:
|
300 |
|
|
if not (RefMoreLumiArray["b1pres"][lumisection] and RefMoreLumiArray["b2pres"][lumisection] and RefMoreLumiArray["b1stab"][lumisection] and RefMoreLumiArray["b2stab"][lumisection]):
|
301 |
|
|
isBeams=False
|
302 |
|
|
except:
|
303 |
grchrist |
1.41 |
isBeams=False
|
304 |
grchrist |
1.40 |
|
305 |
grchrist |
1.43 |
if not (isCol and isBeams):
|
306 |
grchrist |
1.23 |
##clear()
|
307 |
grchrist |
1.43 |
MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
|
308 |
grchrist |
1.25 |
else:
|
309 |
grchrist |
1.43 |
if (len(HeadLumiRange)>0):
|
310 |
muell149 |
1.50 |
RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,ListIgnoredPaths,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb)
|
311 |
grchrist |
1.43 |
if FindL1Zeros:
|
312 |
awoodard |
1.44 |
CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config)
|
313 |
grchrist |
1.43 |
else:
|
314 |
|
|
print "No lumisections that are taking physics data 1"
|
315 |
amott |
1.33 |
if ShifterMode:
|
316 |
grchrist |
1.23 |
#print "Shifter Mode. Continuing"
|
317 |
|
|
pass
|
318 |
amott |
1.2 |
else:
|
319 |
grchrist |
1.23 |
print "Expert Mode. Quitting."
|
320 |
|
|
sys.exit(0)
|
321 |
amott |
1.2 |
|
322 |
amott |
1.1 |
|
323 |
|
|
print "Sleeping for 1 minute before repeating "
|
324 |
amott |
1.16 |
for iSleep in range(20):
|
325 |
amott |
1.15 |
write(".")
|
326 |
|
|
sys.stdout.flush()
|
327 |
amott |
1.16 |
time.sleep(3)
|
328 |
grchrist |
1.23 |
write(" Updating\n")
|
329 |
amott |
1.15 |
sys.stdout.flush()
|
330 |
grchrist |
1.23 |
|
331 |
|
|
##print "\nminLS=",min(HeadLumiRange),"Last LS=",HeadParser.GetLastLS(isCol),"run=",HeadParser.RunNumber
|
332 |
|
|
###Get a new run if DAQ stops
|
333 |
|
|
##print "\nLastGoodLS=",LastGoodLS
|
334 |
|
|
|
335 |
|
|
##### NEED PLACEHOLDER TO COMPARE CURRENT RUN TO LATEST RUN #####
|
336 |
|
|
|
337 |
|
|
NewRun,isCol,isGood = GetLatestRunNumber(9999999) ## update to the latest run and lumi range
|
338 |
|
|
|
339 |
|
|
try:
|
340 |
|
|
maxLumi=max(HeadLumiRange)
|
341 |
|
|
except:
|
342 |
|
|
maxLumi=0
|
343 |
|
|
|
344 |
|
|
##### THESE ARE CONDITIONS TO GET NEW RUN #####
|
345 |
|
|
if maxLumi>(LastGoodLS+1) or not isGood or NewRun!=CurrRun:
|
346 |
|
|
print "Trying to get new Run"
|
347 |
|
|
try:
|
348 |
|
|
HeadParser = DatabaseParser()
|
349 |
|
|
HeadParser.RunNumber = NewRun
|
350 |
|
|
HeadParser.ParseRunSetup()
|
351 |
|
|
CurrRun,isCol,isGood=GetLatestRunNumber(9999999)
|
352 |
|
|
FirstLS=9999
|
353 |
|
|
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
354 |
|
|
if len(HeadLumiRange) is 0:
|
355 |
|
|
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
356 |
grchrist |
1.36 |
print "No lumisections that are taking physics data 2"
|
357 |
grchrist |
1.23 |
if len(HeadLumiRange)>0:
|
358 |
|
|
isGood=1
|
359 |
|
|
isCol=0
|
360 |
|
|
|
361 |
grchrist |
1.43 |
#tempLastGoodLS=LastGoodLS
|
362 |
|
|
#LastGoodLS=HeadParser.GetLastLS(isCol)
|
363 |
grchrist |
1.25 |
##print CurrRun, isCol, isGood
|
364 |
grchrist |
1.23 |
except:
|
365 |
|
|
isGood=0
|
366 |
|
|
isCol=0
|
367 |
|
|
print "failed"
|
368 |
|
|
|
369 |
|
|
|
370 |
awoodard |
1.49 |
|
371 |
grchrist |
1.23 |
else:
|
372 |
|
|
try:
|
373 |
|
|
HeadParser.ParseRunSetup()
|
374 |
|
|
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
375 |
|
|
if len(HeadLumiRange) is 0:
|
376 |
|
|
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
377 |
grchrist |
1.26 |
print "No lumisections that are taking physics data"
|
378 |
grchrist |
1.23 |
if len(HeadLumiRange)>0:
|
379 |
|
|
isGood=1
|
380 |
|
|
isCol=0
|
381 |
grchrist |
1.43 |
#LastGoodLS=HeadParser.GetLastLS(isCol)
|
382 |
grchrist |
1.23 |
|
383 |
|
|
except:
|
384 |
|
|
isGood=0
|
385 |
|
|
isCol=0
|
386 |
|
|
clear()
|
387 |
|
|
print "NO TRIGGER KEY FOUND YET for run", NewRun ,"repeating search"
|
388 |
|
|
|
389 |
grchrist |
1.22 |
|
390 |
amott |
1.1 |
except KeyboardInterrupt:
|
391 |
|
|
print "Quitting. Peace Out."
|
392 |
|
|
|
393 |
|
|
|
394 |
muell149 |
1.50 |
def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,ListIgnoredPaths,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb):
|
395 |
amott |
1.1 |
Data = []
|
396 |
|
|
Warn = []
|
397 |
|
|
IgnoredRates=[]
|
398 |
grchrist |
1.23 |
|
399 |
abrinke1 |
1.9 |
[HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
|
400 |
abrinke1 |
1.3 |
##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
|
401 |
|
|
HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
|
402 |
abrinke1 |
1.9 |
[PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo
|
403 |
grchrist |
1.30 |
deadtimebeamactive=HeadParser.GetDeadTimeBeamActive(HeadLumiRange)
|
404 |
grchrist |
1.18 |
try:
|
405 |
|
|
pkl_file = open(Config.FitFileName, 'rb')
|
406 |
|
|
FitInput = pickle.load(pkl_file)
|
407 |
|
|
pkl_file.close()
|
408 |
grchrist |
1.29 |
##print "fit file name=",Config.FitFileName
|
409 |
grchrist |
1.31 |
|
410 |
grchrist |
1.18 |
except:
|
411 |
grchrist |
1.27 |
print "No fit file specified"
|
412 |
grchrist |
1.20 |
sys.exit(2)
|
413 |
|
|
|
414 |
grchrist |
1.23 |
try:
|
415 |
|
|
refrunfile="RefRuns/%s/Rates_HLT_10LS_JPAP.pkl" % (thisyear)
|
416 |
|
|
pkl_file = open(refrunfile, 'rb')
|
417 |
|
|
RefRatesInput = pickle.load(pkl_file)
|
418 |
|
|
pkl_file.close()
|
419 |
|
|
except:
|
420 |
grchrist |
1.24 |
RefRatesInput={}
|
421 |
grchrist |
1.34 |
print "Didn't open ref file"
|
422 |
amott |
1.1 |
|
423 |
grchrist |
1.30 |
|
424 |
|
|
trig_list=Config.MonitorList
|
425 |
|
|
|
426 |
|
|
if Config.NoVersion:
|
427 |
|
|
trig_list=[]
|
428 |
|
|
FitInputNoV={}
|
429 |
|
|
|
430 |
|
|
for trigger in Config.MonitorList:
|
431 |
|
|
trig_list.append(StripVersion(trigger))
|
432 |
|
|
for trigger in FitInput.iterkeys():
|
433 |
|
|
FitInputNoV[StripVersion(trigger)]=FitInput[trigger]
|
434 |
|
|
FitInput=FitInputNoV
|
435 |
awoodard |
1.49 |
else:
|
436 |
|
|
trig_list=Config.MonitorList
|
437 |
|
|
|
438 |
abrinke1 |
1.3 |
for HeadName in HeadUnprescaledRates:
|
439 |
grchrist |
1.30 |
HeadNameNoV=StripVersion(HeadName)
|
440 |
grchrist |
1.34 |
|
441 |
awoodard |
1.49 |
if RefParser.RunNumber == 0: ## If not ref run then just use trigger list
|
442 |
grchrist |
1.42 |
pass
|
443 |
|
|
if Config.NoVersion:
|
444 |
|
|
if HeadNameNoV not in trig_list and not ListIgnoredPaths:
|
445 |
|
|
continue
|
446 |
|
|
if HeadNameNoV not in FitInput.keys() and not ListIgnoredPaths:
|
447 |
|
|
continue
|
448 |
|
|
else:
|
449 |
|
|
if HeadName not in trig_list and not ListIgnoredPaths:
|
450 |
|
|
continue
|
451 |
|
|
if HeadName not in FitInput.keys() and not ListIgnoredPaths:
|
452 |
|
|
continue
|
453 |
awoodard |
1.49 |
|
454 |
grchrist |
1.28 |
masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_Zero"]
|
455 |
abrinke1 |
1.9 |
masked_trig = False
|
456 |
|
|
for mask in masked_triggers:
|
457 |
|
|
if str(mask) in HeadName:
|
458 |
|
|
masked_trig = True
|
459 |
|
|
if masked_trig:
|
460 |
|
|
continue
|
461 |
|
|
|
462 |
amott |
1.1 |
skipTrig=False
|
463 |
abrinke1 |
1.3 |
TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)
|
464 |
awoodard |
1.49 |
|
465 |
amott |
1.1 |
if RefParser.RunNumber == 0: ## Use rate prediction functions
|
466 |
awoodard |
1.44 |
|
467 |
grchrist |
1.30 |
PSCorrectedExpectedRate = Config.GetExpectedRate(HeadName,FitInput,RefRatesInput,HeadAvLiveLumi,HeadAvDeliveredLumi,deadtimebeamactive)
|
468 |
abrinke1 |
1.9 |
|
469 |
grchrist |
1.32 |
try:
|
470 |
|
|
ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
|
471 |
awoodard |
1.44 |
sigma = round((sqrt(PSCorrectedExpectedRate[1]) / HeadUnprescaledRates[HeadName][1]),2)
|
472 |
|
|
|
473 |
grchrist |
1.32 |
except:
|
474 |
awoodard |
1.49 |
ExpectedRate=0. ##This means we don't have a prediction for this trigger
|
475 |
grchrist |
1.30 |
|
476 |
amott |
1.1 |
PerDiff=0
|
477 |
awoodard |
1.44 |
SigmaDiff=0
|
478 |
grchrist |
1.37 |
VC = ""
|
479 |
awoodard |
1.49 |
if ExpectedRate > 0 and sigma > 0:
|
480 |
amott |
1.1 |
PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
|
481 |
muell149 |
1.48 |
try:
|
482 |
|
|
SigmaDiff = round( (TriggerRate - ExpectedRate)/sigma, 2)
|
483 |
|
|
except:
|
484 |
|
|
SigmaDiff = 0#solves nasty zero division errors
|
485 |
|
|
|
486 |
grchrist |
1.37 |
else:
|
487 |
|
|
PerDiff=-999.
|
488 |
awoodard |
1.44 |
SigmaDiff=-999
|
489 |
grchrist |
1.37 |
if ExpectedRate==0:
|
490 |
|
|
VC="0 expected rate"
|
491 |
|
|
else:
|
492 |
|
|
VC="lt 0 expected rate"
|
493 |
amott |
1.1 |
|
494 |
amott |
1.33 |
if TriggerRate < IgnoreThreshold and (ExpectedRate < IgnoreThreshold and ExpectedRate!=0):
|
495 |
amott |
1.1 |
continue
|
496 |
|
|
|
497 |
awoodard |
1.44 |
Data.append([HeadName, TriggerRate, ExpectedRate, PerDiff, SigmaDiff, round(HeadUnprescaledRates[HeadName][1],1),VC])
|
498 |
amott |
1.1 |
|
499 |
|
|
else: ## Use a reference run
|
500 |
|
|
## cheap trick to only get triggers in list when in shifter mode
|
501 |
|
|
#print "shifter mode=",int(Config.ShifterMode)
|
502 |
grchrist |
1.34 |
## continue
|
503 |
amott |
1.1 |
|
504 |
|
|
RefInstLumi = 0
|
505 |
|
|
RefIterator = 0
|
506 |
|
|
RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
|
507 |
|
|
RefLen = -10
|
508 |
grchrist |
1.34 |
|
509 |
|
|
|
510 |
abrinke1 |
1.3 |
RefUnprescaledRates = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
|
511 |
amott |
1.1 |
[RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefParser.GetLSRange(RefStartIndex,RefLen))
|
512 |
grchrist |
1.34 |
deadtimebeamactive=RefParser.GetDeadTimeBeamActive(RefParser.GetLSRange(RefStartIndex,RefLen))
|
513 |
|
|
|
514 |
amott |
1.1 |
RefRate = -1
|
515 |
|
|
for k,v in RefUnprescaledRates.iteritems():
|
516 |
grchrist |
1.19 |
#if StripVersion(HeadName) == StripVersion(k): # versions may not match
|
517 |
grchrist |
1.34 |
if HeadName==k:
|
518 |
|
|
RefRate = RefUnprescaledRates[k][2]
|
519 |
|
|
|
520 |
|
|
try:
|
521 |
|
|
ScaledRefRate = round( (RefRate*HeadAvLiveLumi/RefAvLiveLumi*(1-deadtimebeamactive)), 2 )
|
522 |
|
|
|
523 |
|
|
except ZeroDivisionError:
|
524 |
|
|
ScaledRefRate=0
|
525 |
|
|
|
526 |
|
|
##print HeadName,"ScaledRefRate=",ScaledRefRate
|
527 |
awoodard |
1.44 |
SigmaDiff = 0
|
528 |
amott |
1.1 |
if ScaledRefRate == 0:
|
529 |
awoodard |
1.44 |
PerDiff = -999
|
530 |
amott |
1.1 |
else:
|
531 |
|
|
PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
|
532 |
awoodard |
1.44 |
|
533 |
amott |
1.1 |
if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
|
534 |
|
|
continue
|
535 |
|
|
|
536 |
|
|
VC = ""
|
537 |
awoodard |
1.44 |
Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,SigmaDiff,round((HeadUnprescaledRates[HeadName][1]),1),VC])
|
538 |
amott |
1.1 |
|
539 |
amott |
1.33 |
SortedData = []
|
540 |
|
|
if SortBy == "":
|
541 |
|
|
SortedData = Data # don't do any sorting
|
542 |
grchrist |
1.35 |
if RefParser.RunNumber>0:
|
543 |
grchrist |
1.34 |
SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
|
544 |
amott |
1.33 |
elif SortBy == "name":
|
545 |
|
|
SortedData=sorted(Data, key=lambda entry: entry[0])
|
546 |
|
|
elif SortBy == "rate":
|
547 |
|
|
SortedData=sorted(Data, key=lambda entry: entry[1],reverse=True)
|
548 |
awoodard |
1.44 |
elif SortBy == "ratePercDiff":
|
549 |
amott |
1.33 |
SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
|
550 |
awoodard |
1.44 |
elif SortBy == "rateSigmaDiff":
|
551 |
|
|
SortedData=sorted(Data, key=lambda entry: abs(entry[4]),reverse=True)
|
552 |
amott |
1.33 |
else:
|
553 |
|
|
print "Invalid sorting option %s\n"%SortBy
|
554 |
|
|
SortedData = Data
|
555 |
|
|
|
556 |
|
|
#check for triggers above the warning threshold
|
557 |
|
|
Warn=[]
|
558 |
|
|
for entry in SortedData:
|
559 |
awoodard |
1.49 |
if abs(entry[4]) > AllowedRateSigmaDiff and WarnOnSigmaDiff:
|
560 |
awoodard |
1.46 |
Warn.append(True)
|
561 |
awoodard |
1.49 |
elif abs(entry[3]) > AllowedRatePercDiff and not WarnOnSigmaDiff:
|
562 |
amott |
1.33 |
Warn.append(True)
|
563 |
|
|
else:
|
564 |
|
|
Warn.append(False)
|
565 |
|
|
|
566 |
awoodard |
1.47 |
if ShowSigmaAndPercDiff == 1:
|
567 |
|
|
Header = ["Trigger Name", "Actual", "Expected","% Diff","Deviation", "Cur PS", "Comments"]
|
568 |
|
|
table_data=SortedData
|
569 |
|
|
PrettyPrintTable(Header,table_data,[80,10,10,10,10,10,20],Warn)
|
570 |
awoodard |
1.49 |
print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
|
571 |
awoodard |
1.47 |
elif WarnOnSigmaDiff == 1:
|
572 |
awoodard |
1.45 |
Header = ["Trigger Name", "Actual", "Expected","Deviation", "Cur PS", "Comments"]
|
573 |
|
|
table_data = [[col[0], col[1], col[2], col[4], col[5], col[6]] for col in SortedData]
|
574 |
awoodard |
1.47 |
PrettyPrintTable(Header,table_data,[80,10,10,10,10,10,20],Warn)
|
575 |
awoodard |
1.49 |
print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
|
576 |
awoodard |
1.44 |
else:
|
577 |
awoodard |
1.47 |
Header = ["Trigger Name", "Actual", "Expected", "% Diff", "Cur PS", "Comments"]
|
578 |
awoodard |
1.44 |
table_data = [[col[0], col[1], col[2], col[3], col[5], col[6]] for col in SortedData]
|
579 |
awoodard |
1.49 |
PrettyPrintTable(Header,table_data,[80,10,10,10,10,20],Warn)
|
580 |
awoodard |
1.44 |
|
581 |
muell149 |
1.50 |
if writeb:
|
582 |
|
|
prettyCSVwriter("rateMon_newmenu.csv",[80,10,10,10,10,20,20],Header,SortedData,Warn)
|
583 |
awoodard |
1.45 |
|
584 |
awoodard |
1.49 |
MoreTableInfo(HeadParser,HeadLumiRange,Config,True)
|
585 |
amott |
1.1 |
|
586 |
grchrist |
1.39 |
for warning in Warn:
|
587 |
|
|
if warning==True:
|
588 |
|
|
write(bcolors.WARNING)
|
589 |
|
|
print "If any trigger remains red for 3 minutes, CALL HLT DOC"
|
590 |
|
|
print "More instructions at https://twiki.cern.ch/twiki/bin/view/CMS/TriggerShiftHLTGuide"
|
591 |
|
|
write(bcolors.ENDC+"\n")
|
592 |
|
|
break
|
593 |
|
|
|
594 |
awoodard |
1.44 |
def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedPercRateDiff,IgnoreThreshold,Config):
|
595 |
amott |
1.1 |
L1Zeros=[]
|
596 |
|
|
IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
|
597 |
|
|
for key in HeadParser.TriggerRates:
|
598 |
|
|
## Skip events in the skip list
|
599 |
|
|
skipTrig=False
|
600 |
|
|
##for trig in Config.ExcludeList:
|
601 |
|
|
##if not trigN.find(trig) == -1:
|
602 |
|
|
##skipTrig=True
|
603 |
|
|
##break
|
604 |
|
|
if skipTrig:
|
605 |
|
|
continue
|
606 |
|
|
## if no events pass the L1, add it to the L1Zeros list if not already there
|
607 |
|
|
if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
|
608 |
|
|
if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
|
609 |
|
|
|
610 |
|
|
L1Zeros.append(HeadParser.TriggerRates[key][4])
|
611 |
|
|
print "L1Zeros=", L1Zeros
|
612 |
|
|
|
613 |
|
|
if len(L1Zeros) == 0:
|
614 |
|
|
#print "It looks like no masked L1 bits seed trigger paths"
|
615 |
|
|
pass
|
616 |
|
|
else:
|
617 |
|
|
print "The following seeds are used to seed HLT bits but accept 0 events:"
|
618 |
|
|
#print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
|
619 |
|
|
for Seed in L1Zeros:
|
620 |
|
|
print Seed
|
621 |
|
|
|
622 |
|
|
if __name__=='__main__':
|
623 |
grchrist |
1.20 |
global thisyear
|
624 |
amott |
1.1 |
main()
|