ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/RateMonShiftTool_dev/DatabaseRateMonitor.py
Revision: 1.53
Committed: Thu Aug 23 17:18:46 2012 UTC (12 years, 8 months ago) by awoodard
Content type: text/x-python
Branch: MAIN
Changes since 1.52: +2 -1 lines
Log Message:
bug fix

File Contents

# User Rev Content
1 amott 1.1 #!/usr/bin/env python
2    
3 amott 1.15 #from AndrewGetRun import GetRun
4 amott 1.1 from ReadConfig import RateMonConfig
5     import sys
6     import os
7     import cPickle as pickle
8     import getopt
9     import time
10 awoodard 1.49 from StreamMonitor import *
11 amott 1.1 from colors import *
12     from TablePrint import *
13     from AddTableInfo_db import MoreTableInfo
14     from math import *
15 grchrist 1.43 from DatabaseParser import *
16 muell149 1.48 from TablePrint import *
17 amott 1.1
18     WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
19     WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
20 grchrist 1.34 RefRunNameTemplate = "RefRuns/%s/Run_%s.pk"
21 amott 1.1
22     # define a function that clears the terminal screen
23     def clear():
24     print("\x1B[2J")
25    
26    
27     def usage():
28     print sys.argv[0]+" [Options]"
29 awoodard 1.44 print "This script gets the current HLT trigger rates and compares them to a reference run or a fit to multiple runs"
30 amott 1.1 print "Options: "
31 awoodard 1.51 print "--AllowedPercDiff=<diff> Warn only if difference in trigger rate is greater than <diff>%"
32     print "--AllowedSigmaDiff=<diff> Warn only if difference in trigger rate is greater than <diff> standard deviations"
33 amott 1.1 print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
34     print "--FindL1Zeros Look for physics paths with 0 L1 rate"
35     print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
36     print "--NumberLS=<#> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
37     print " or set LSSlidingWindow = -1"
38     print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate>"
39     print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
40     print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
41     print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
42     print "--ShowPSTriggers Show prescaled triggers in rate comparison"
43 amott 1.33 print "--sortBy=<field> Sort the triggers by field. Valid fields are: name, rate, rateDiff"
44 amott 1.2 print "--force Override the check for collisions run"
45 muell149 1.50 print "--write Writes rates to .csv file"
46 awoodard 1.51 print "--ShowAllBadRates Show a list of all triggers (not just those in the monitor list) with bad rates"
47 amott 1.1 print "--help Print this help"
48 grchrist 1.20
49     def pickYear():
50     global thisyear
51 grchrist 1.23 thisyear="2012"
52     ##print "Year set to ",thisyear
53 grchrist 1.20
54 amott 1.1 def main():
55 grchrist 1.21 pickYear()
56 amott 1.1 try:
57 awoodard 1.44 opt, args = getopt.getopt(sys.argv[1:],"",["AllowedPercDiff=","AllowedSigmaDiff=","CompareRun=","FindL1Zeros",\
58 amott 1.1 "FirstLS=","NumberLS=","IgnoreLowRate=","ListIgnoredPaths",\
59 awoodard 1.51 "PrintLumi","RefRun=","ShowPSTriggers","force","sortBy=","write","ShowAllBadRates","help"])
60 amott 1.1 except getopt.GetoptError, err:
61     print str(err)
62     usage()
63     sys.exit(2)
64    
65     Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
66     for o,a in opt:
67     if o=="--ConfigFile":
68     Config.CFGfile=a
69     Config.ReadCFG()
70 grchrist 1.35
71    
72     if "NoV" in Config.FitFileName:
73     Config.NoVersion=True
74 grchrist 1.43 #print "NoVersion=",Config.NoVersion
75 grchrist 1.23
76 awoodard 1.47 ShowSigmaAndPercDiff = Config.DefShowSigmaAndPercDiff
77 awoodard 1.46 WarnOnSigmaDiff = Config.DefWarnOnSigmaDiff
78 awoodard 1.44 AllowedRatePercDiff = Config.DefAllowRatePercDiff
79     AllowedRateSigmaDiff = Config.DefAllowRateSigmaDiff
80 amott 1.1 CompareRunNum = ""
81     FindL1Zeros = False
82     FirstLS = 9999
83 amott 1.2 NumLS = -10
84 amott 1.1 IgnoreThreshold = Config.DefAllowIgnoreThresh
85     ListIgnoredPaths = False
86     PrintLumi = False
87     RefRunNum = int(Config.ReferenceRun)
88     ShowPSTriggers = True
89 amott 1.2 Force = False
90 muell149 1.50 writeb = False
91 awoodard 1.51 SortBy = "rate"
92 amott 1.33 ShifterMode = int(Config.ShifterMode) # get this from the config, but can be overridden by other options
93 awoodard 1.51 ShowAllBadRates = False
94 grchrist 1.23
95     ## if int(Config.ShifterMode):
96     ## print "ShifterMode!!"
97     ## else:
98     ## print "ExpertMode"
99 amott 1.1
100     if Config.LSWindow > 0:
101     NumLS = -1*Config.LSWindow
102    
103     for o,a in opt: # get options passed on the command line
104 awoodard 1.44 if o=="--AllowedPercDiff":
105     AllowedRatePercDiff = float(a)
106     elif o=="--AllowedSigmaDiff":
107     AllowedRateSigmaDiff = float(a)
108 amott 1.1 elif o=="--CompareRun":
109     CompareRunNum=int(a)
110 amott 1.33 ShifterMode = False
111 amott 1.1 elif o=="--FindL1Zeros":
112     FindL1Zeros = True
113     elif o=="--FirstLS":
114     FirstLS = int(a)
115 amott 1.33 ShifterMode = False
116 amott 1.1 elif o=="--NumberLS":
117     NumLS = int(a)
118     elif o=="--IgnoreLowRate":
119     IgnoreThreshold = float(a)
120     elif o=="--ListIgnoredPaths":
121     ListIgnoredPaths=True
122 amott 1.33 ShifterMode = False
123 amott 1.1 elif o=="--PrintLumi":
124     PrintLumi = True
125     elif o=="--RefRun":
126     RefRunNum=int(a)
127     elif o=="--ShowPSTriggers":
128     ShowPSTriggers=True
129 amott 1.33 elif o=="--sortBy":
130     SortBy = a
131 amott 1.2 elif o=="--force":
132     Force = True
133 muell149 1.50 elif o=="--write":
134     writeb = True
135 awoodard 1.51 elif o=="--ShowAllBadRates":
136     ShowAllBadRates=True
137 amott 1.1 elif o=="--help":
138     usage()
139     sys.exit(0)
140     else:
141     print "Invalid Option "+a
142     sys.exit(1)
143    
144 grchrist 1.23
145 amott 1.1 RefLumisExists = False
146     """
147 grchrist 1.34 RefRunFile=RefRunNameTemplate % str(RefRunNum)
148 amott 1.1 if RefRunNum > 0:
149     RefRates = {}
150     for Iterator in range(1,100):
151     if RefLumisExists: ## Quits at the end of a run
152     if max(RefLumis[0]) <= (Iterator+1)*10:
153     break
154    
155     RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
156     print "RefRunFile=",RefRunFile
157     if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
158     print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
159     print RefRunFile[:RefRunFile.rfind('/')]
160     sys.exit(0)
161    
162     if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
163     print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
164     print "Creating ..."
165     try:
166     RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
167     print "parsing"
168     except:
169     print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
170     continue
171    
172     else: # otherwise load it from the file
173     RefParser = pickle.load( open( RefRunFile ) )
174     print "loading"
175     if not RefLumisExists:
176     RefLumis = RefParser.LumiInfo
177     RefLumisExists = True
178    
179     try:
180     RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
181     LastSuccessfulIterator = Iterator
182     except:
183     print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
184 grchrist 1.34
185 amott 1.1 """
186 grchrist 1.34 RefRunFile = RefRunNameTemplate % (thisyear,RefRunNum)
187 amott 1.1 RefParser = DatabaseParser()
188 grchrist 1.23 ##print "Reference Run: "+str(RefRunNum)
189 amott 1.1 if RefRunNum > 0:
190 awoodard 1.49 print "Getting RefRunFile",RefRunFile
191 amott 1.1 if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
192     print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
193     print RefRunFile[:RefRunFile.rfind('/')]
194 grchrist 1.27 sys.exit(0)
195 grchrist 1.34 s
196 amott 1.1 return
197     if not os.path.exists(RefRunFile):
198 grchrist 1.34 print "RefRunFile does not exist"
199 amott 1.1 # create the reference run file
200     try:
201     RefParser.RunNumber = RefRunNum
202     RefParser.ParseRunSetup()
203 grchrist 1.34 print "RefParser is setup"
204 amott 1.1 #RefParser.GetAllTriggerRatesByLS()
205     #RefParser.Save( RefRunFile )
206     except e:
207     print "PROBLEM GETTING REFERNCE RUN"
208     raise
209     else:
210     RefParser = pickle.load( open( RefRunFile ) )
211    
212     # OK, Got the Reference Run
213     # Now get the most recent run
214    
215     SaveRun = False
216     if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
217 grchrist 1.22 CompareRunNum,isCol,isGood = GetLatestRunNumber()
218 grchrist 1.26
219    
220 grchrist 1.22 if not isGood:
221 grchrist 1.23 print "NO TRIGGER KEY FOUND for run ",CompareRunNum
222 grchrist 1.22
223 grchrist 1.23 ##sys.exit(0)
224 amott 1.1
225     if not isCol:
226     print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
227 amott 1.16 print "Monitoring only stream A and Express"
228     #if not Force:
229     # sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
230 grchrist 1.23
231     else:
232     print "Most Recent run is "+str(CompareRunNum)
233 amott 1.16 else:
234 grchrist 1.22 CompareRunNum,isCol,isGood = GetLatestRunNumber(CompareRunNum)
235     if not isGood:
236 grchrist 1.23 print "NO TRIGGER KEY FOUND for run ", CompareRunNum
237     ##sys.exit(0)
238 amott 1.17
239 grchrist 1.23
240 amott 1.17 HeadParser = DatabaseParser()
241     HeadParser.RunNumber = CompareRunNum
242 grchrist 1.23
243     try:
244     HeadParser.ParseRunSetup()
245     HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
246 grchrist 1.43 LastGoodLS=HeadParser.GetLastLS(isCol)+1
247     tempLastGoodLS=LastGoodLS
248 grchrist 1.23 CurrRun=CompareRunNum
249 grchrist 1.43 #print "done good"
250 grchrist 1.23 except:
251 grchrist 1.43 #print "exception"
252 grchrist 1.23 HeadLumiRange=[]
253     LastGoodLS=-1
254 grchrist 1.43 tempLastGoodLS=LastGoodLS-1
255 grchrist 1.23 CurrRun=CompareRunNum
256     isGood=0
257    
258     if len(HeadLumiRange) is 0:
259 grchrist 1.36 print "No lumisections that are taking physics data 0"
260 grchrist 1.23 HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
261     if len(HeadLumiRange)>0:
262     isGood=1
263     isCol=0
264     ##sys.exit(0)
265 awoodard 1.44
266 awoodard 1.45 ## This reduces the sensitivity for a rate measurement to cause a warning during the beginning of a run
267 awoodard 1.49 if len(HeadLumiRange) > 0 and len(HeadLumiRange) < 10:
268 awoodard 1.44 AllowedRateSigmaDiff = AllowedRateSigmaDiff*10 / len(HeadLumiRange)
269 grchrist 1.23
270 amott 1.1 if PrintLumi:
271     for LS in HeadParser.LumiInfo[0]:
272     try:
273     if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
274     continue
275     print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
276     except:
277     print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
278    
279 grchrist 1.23 sys.exit(0)
280 amott 1.1
281     if RefRunNum == 0:
282     RefRates = 0
283     RefLumis = 0
284     LastSuccessfulIterator = 0
285    
286     ### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
287 grchrist 1.23 ###isGood=1##if there is a trigger key
288 amott 1.1 try:
289     while True:
290 grchrist 1.23
291     if isGood:
292 grchrist 1.43 tempLastGoodLS=LastGoodLS
293 grchrist 1.23 LastGoodLS=HeadParser.GetLastLS(isCol)
294 grchrist 1.43 ##print "Last Good=",LastGoodLS, tempLastGoodLS
295     if LastGoodLS==tempLastGoodLS:
296     write(bcolors.FAIL)
297     print "Trying to get new Run"
298     write(bcolors.ENDC+"\n")
299     else:
300     RefMoreLumiArray = HeadParser.GetMoreLumiInfo()
301     isBeams=True
302     for lumisection in HeadLumiRange:
303     try:
304     if not (RefMoreLumiArray["b1pres"][lumisection] and RefMoreLumiArray["b2pres"][lumisection] and RefMoreLumiArray["b1stab"][lumisection] and RefMoreLumiArray["b2stab"][lumisection]):
305     isBeams=False
306     except:
307 grchrist 1.41 isBeams=False
308 grchrist 1.40
309 grchrist 1.43 if not (isCol and isBeams):
310 grchrist 1.23 ##clear()
311 grchrist 1.43 MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
312 grchrist 1.25 else:
313 grchrist 1.43 if (len(HeadLumiRange)>0):
314 awoodard 1.51 RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,ListIgnoredPaths,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb,ShowAllBadRates)
315 grchrist 1.43 if FindL1Zeros:
316 awoodard 1.44 CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config)
317 grchrist 1.43 else:
318     print "No lumisections that are taking physics data 1"
319 amott 1.33 if ShifterMode:
320 grchrist 1.23 #print "Shifter Mode. Continuing"
321     pass
322 amott 1.2 else:
323 grchrist 1.23 print "Expert Mode. Quitting."
324     sys.exit(0)
325 amott 1.2
326 amott 1.1
327     print "Sleeping for 1 minute before repeating "
328 amott 1.16 for iSleep in range(20):
329 amott 1.15 write(".")
330     sys.stdout.flush()
331 amott 1.16 time.sleep(3)
332 grchrist 1.23 write(" Updating\n")
333 amott 1.15 sys.stdout.flush()
334 grchrist 1.23
335     ##print "\nminLS=",min(HeadLumiRange),"Last LS=",HeadParser.GetLastLS(isCol),"run=",HeadParser.RunNumber
336     ###Get a new run if DAQ stops
337     ##print "\nLastGoodLS=",LastGoodLS
338    
339     ##### NEED PLACEHOLDER TO COMPARE CURRENT RUN TO LATEST RUN #####
340    
341     NewRun,isCol,isGood = GetLatestRunNumber(9999999) ## update to the latest run and lumi range
342    
343     try:
344     maxLumi=max(HeadLumiRange)
345     except:
346     maxLumi=0
347    
348     ##### THESE ARE CONDITIONS TO GET NEW RUN #####
349     if maxLumi>(LastGoodLS+1) or not isGood or NewRun!=CurrRun:
350     print "Trying to get new Run"
351     try:
352     HeadParser = DatabaseParser()
353     HeadParser.RunNumber = NewRun
354     HeadParser.ParseRunSetup()
355     CurrRun,isCol,isGood=GetLatestRunNumber(9999999)
356     FirstLS=9999
357     HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
358     if len(HeadLumiRange) is 0:
359     HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
360 grchrist 1.36 print "No lumisections that are taking physics data 2"
361 grchrist 1.23 if len(HeadLumiRange)>0:
362     isGood=1
363     isCol=0
364    
365 grchrist 1.43 #tempLastGoodLS=LastGoodLS
366     #LastGoodLS=HeadParser.GetLastLS(isCol)
367 grchrist 1.25 ##print CurrRun, isCol, isGood
368 grchrist 1.23 except:
369     isGood=0
370     isCol=0
371     print "failed"
372    
373    
374 awoodard 1.49
375 grchrist 1.23 else:
376     try:
377     HeadParser.ParseRunSetup()
378     HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
379     if len(HeadLumiRange) is 0:
380     HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
381 grchrist 1.26 print "No lumisections that are taking physics data"
382 grchrist 1.23 if len(HeadLumiRange)>0:
383     isGood=1
384     isCol=0
385 grchrist 1.43 #LastGoodLS=HeadParser.GetLastLS(isCol)
386 grchrist 1.23
387     except:
388     isGood=0
389     isCol=0
390     clear()
391     print "NO TRIGGER KEY FOUND YET for run", NewRun ,"repeating search"
392    
393 grchrist 1.22
394 amott 1.1 except KeyboardInterrupt:
395     print "Quitting. Peace Out."
396    
397    
398 awoodard 1.51 def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,ListIgnoredPaths,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb,ShowAllBadRates):
399 amott 1.1 Data = []
400     Warn = []
401     IgnoredRates=[]
402 grchrist 1.23
403 abrinke1 1.9 [HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
404 abrinke1 1.3 ##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
405     HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
406 abrinke1 1.9 [PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo
407 grchrist 1.30 deadtimebeamactive=HeadParser.GetDeadTimeBeamActive(HeadLumiRange)
408 grchrist 1.18 try:
409     pkl_file = open(Config.FitFileName, 'rb')
410     FitInput = pickle.load(pkl_file)
411     pkl_file.close()
412 grchrist 1.29 ##print "fit file name=",Config.FitFileName
413 grchrist 1.31
414 grchrist 1.18 except:
415 grchrist 1.27 print "No fit file specified"
416 grchrist 1.20 sys.exit(2)
417    
418 grchrist 1.23 try:
419     refrunfile="RefRuns/%s/Rates_HLT_10LS_JPAP.pkl" % (thisyear)
420     pkl_file = open(refrunfile, 'rb')
421     RefRatesInput = pickle.load(pkl_file)
422     pkl_file.close()
423     except:
424 grchrist 1.24 RefRatesInput={}
425 grchrist 1.34 print "Didn't open ref file"
426 amott 1.1
427 grchrist 1.30
428     trig_list=Config.MonitorList
429    
430     if Config.NoVersion:
431     trig_list=[]
432    
433     for trigger in Config.MonitorList:
434     trig_list.append(StripVersion(trigger))
435     for trigger in FitInput.iterkeys():
436 awoodard 1.51 FitInput[StripVersion(trigger)]=FitInput.pop(trigger)
437     for trigger in HeadUnprescaledRates:
438     HeadUnprescaledRates[StripVersion(trigger)]=HeadUnprescaledRates.pop(trigger)
439 awoodard 1.49 else:
440     trig_list=Config.MonitorList
441    
442 abrinke1 1.3 for HeadName in HeadUnprescaledRates:
443 awoodard 1.49 if RefParser.RunNumber == 0: ## If not ref run then just use trigger list
444 awoodard 1.51 if HeadName not in trig_list and not ListIgnoredPaths and not ShowAllBadRates:
445     continue
446     if HeadName not in FitInput.keys() and not ListIgnoredPaths and not ShowAllBadRates:
447     continue
448 awoodard 1.49
449 grchrist 1.28 masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_Zero"]
450 abrinke1 1.9 masked_trig = False
451     for mask in masked_triggers:
452     if str(mask) in HeadName:
453     masked_trig = True
454     if masked_trig:
455     continue
456    
457 amott 1.1 skipTrig=False
458 abrinke1 1.3 TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)
459 awoodard 1.49
460 amott 1.1 if RefParser.RunNumber == 0: ## Use rate prediction functions
461 awoodard 1.44
462 grchrist 1.30 PSCorrectedExpectedRate = Config.GetExpectedRate(HeadName,FitInput,RefRatesInput,HeadAvLiveLumi,HeadAvDeliveredLumi,deadtimebeamactive)
463 awoodard 1.51 VC = ""
464 abrinke1 1.9
465 grchrist 1.32 try:
466     ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
467 awoodard 1.52 sigma = PSCorrectedExpectedRate[1]/(sqrt(len(HeadLumiRange))* HeadUnprescaledRates[HeadName][1])
468 awoodard 1.44
469 grchrist 1.32 except:
470 awoodard 1.52 ExpectedRate = 0.0 ##This means we don't have a prediction for this trigger
471     PerDiff = 0.0
472     SigmaDiff = 0.0
473 awoodard 1.53 if HeadUnprescaledRates[HeadName][1] != 0:
474     VC="No prediction"
475 grchrist 1.30
476 awoodard 1.51 if ExpectedRate > 0:
477 amott 1.1 PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
478 grchrist 1.37 else:
479 awoodard 1.52 PerDiff = 0.0
480 awoodard 1.51
481 awoodard 1.52 if sigma > 0:
482 awoodard 1.51 SigmaDiff = round( (TriggerRate - ExpectedRate)/sigma, 2)
483     else:
484 awoodard 1.52 SigmaDiff =-999 #Zero sigma means that when there were no rates for this trigger when the fit was made
485 amott 1.1
486 amott 1.33 if TriggerRate < IgnoreThreshold and (ExpectedRate < IgnoreThreshold and ExpectedRate!=0):
487 amott 1.1 continue
488    
489 awoodard 1.44 Data.append([HeadName, TriggerRate, ExpectedRate, PerDiff, SigmaDiff, round(HeadUnprescaledRates[HeadName][1],1),VC])
490 amott 1.1
491     else: ## Use a reference run
492     ## cheap trick to only get triggers in list when in shifter mode
493     #print "shifter mode=",int(Config.ShifterMode)
494 grchrist 1.34 ## continue
495 amott 1.1
496     RefInstLumi = 0
497     RefIterator = 0
498     RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
499     RefLen = -10
500 grchrist 1.34
501    
502 abrinke1 1.3 RefUnprescaledRates = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
503 amott 1.1 [RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefParser.GetLSRange(RefStartIndex,RefLen))
504 grchrist 1.34 deadtimebeamactive=RefParser.GetDeadTimeBeamActive(RefParser.GetLSRange(RefStartIndex,RefLen))
505    
506 amott 1.1 RefRate = -1
507     for k,v in RefUnprescaledRates.iteritems():
508 grchrist 1.34 if HeadName==k:
509     RefRate = RefUnprescaledRates[k][2]
510    
511     try:
512     ScaledRefRate = round( (RefRate*HeadAvLiveLumi/RefAvLiveLumi*(1-deadtimebeamactive)), 2 )
513    
514     except ZeroDivisionError:
515     ScaledRefRate=0
516    
517 awoodard 1.44 SigmaDiff = 0
518 amott 1.1 if ScaledRefRate == 0:
519 awoodard 1.44 PerDiff = -999
520 amott 1.1 else:
521     PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
522 awoodard 1.44
523 amott 1.1 if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
524     continue
525    
526     VC = ""
527 awoodard 1.44 Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,SigmaDiff,round((HeadUnprescaledRates[HeadName][1]),1),VC])
528 amott 1.1
529 amott 1.33 SortedData = []
530     if SortBy == "":
531     SortedData = Data # don't do any sorting
532 grchrist 1.35 if RefParser.RunNumber>0:
533 grchrist 1.34 SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
534 amott 1.33 elif SortBy == "name":
535     SortedData=sorted(Data, key=lambda entry: entry[0])
536     elif SortBy == "rate":
537     SortedData=sorted(Data, key=lambda entry: entry[1],reverse=True)
538 awoodard 1.44 elif SortBy == "ratePercDiff":
539 amott 1.33 SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
540 awoodard 1.44 elif SortBy == "rateSigmaDiff":
541     SortedData=sorted(Data, key=lambda entry: abs(entry[4]),reverse=True)
542 amott 1.33 else:
543     print "Invalid sorting option %s\n"%SortBy
544     SortedData = Data
545    
546     #check for triggers above the warning threshold
547     Warn=[]
548 awoodard 1.51 core_data=[]
549 amott 1.33 for entry in SortedData:
550 awoodard 1.51 bad_rate = (abs(entry[4]) > AllowedRateSigmaDiff and WarnOnSigmaDiff) or (abs(entry[3]) > AllowedRatePercDiff and not WarnOnSigmaDiff)
551     if entry[0] in trig_list or ListIgnoredPaths:
552     core_data.append(entry)
553     if bad_rate:
554     Warn.append(True)
555     else:
556     Warn.append(False)
557 amott 1.33 else:
558 awoodard 1.51 if bad_rate and entry[6] != "No prediction" and ShowAllBadRates:
559     core_data.append(entry)
560     Warn.append(True)
561 amott 1.33
562 awoodard 1.47 if ShowSigmaAndPercDiff == 1:
563     Header = ["Trigger Name", "Actual", "Expected","% Diff","Deviation", "Cur PS", "Comments"]
564 awoodard 1.51 table_data=core_data
565 awoodard 1.47 PrettyPrintTable(Header,table_data,[80,10,10,10,10,10,20],Warn)
566 awoodard 1.49 print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
567 awoodard 1.47 elif WarnOnSigmaDiff == 1:
568 awoodard 1.45 Header = ["Trigger Name", "Actual", "Expected","Deviation", "Cur PS", "Comments"]
569 awoodard 1.51 table_data = [[col[0], col[1], col[2], col[4], col[5], col[6]] for col in core_data]
570 awoodard 1.47 PrettyPrintTable(Header,table_data,[80,10,10,10,10,10,20],Warn)
571 awoodard 1.49 print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
572 awoodard 1.44 else:
573 awoodard 1.47 Header = ["Trigger Name", "Actual", "Expected", "% Diff", "Cur PS", "Comments"]
574 awoodard 1.51 table_data = [[col[0], col[1], col[2], col[3], col[5], col[6]] for col in core_data]
575 awoodard 1.49 PrettyPrintTable(Header,table_data,[80,10,10,10,10,20],Warn)
576 awoodard 1.44
577 muell149 1.50 if writeb:
578 awoodard 1.51 prettyCSVwriter("rateMon_newmenu.csv",[80,10,10,10,10,20,20],Header,core_data,Warn)
579 awoodard 1.45
580 awoodard 1.49 MoreTableInfo(HeadParser,HeadLumiRange,Config,True)
581 amott 1.1
582 grchrist 1.39 for warning in Warn:
583     if warning==True:
584     write(bcolors.WARNING)
585     print "If any trigger remains red for 3 minutes, CALL HLT DOC"
586     print "More instructions at https://twiki.cern.ch/twiki/bin/view/CMS/TriggerShiftHLTGuide"
587     write(bcolors.ENDC+"\n")
588     break
589 awoodard 1.51
590    
591 awoodard 1.44 def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedPercRateDiff,IgnoreThreshold,Config):
592 amott 1.1 L1Zeros=[]
593     IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
594     for key in HeadParser.TriggerRates:
595     ## Skip events in the skip list
596     skipTrig=False
597     ##for trig in Config.ExcludeList:
598     ##if not trigN.find(trig) == -1:
599     ##skipTrig=True
600     ##break
601     if skipTrig:
602     continue
603     ## if no events pass the L1, add it to the L1Zeros list if not already there
604     if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
605     if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
606    
607     L1Zeros.append(HeadParser.TriggerRates[key][4])
608     print "L1Zeros=", L1Zeros
609    
610     if len(L1Zeros) == 0:
611     #print "It looks like no masked L1 bits seed trigger paths"
612     pass
613     else:
614     print "The following seeds are used to seed HLT bits but accept 0 events:"
615     #print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
616     for Seed in L1Zeros:
617     print Seed
618    
619     if __name__=='__main__':
620 grchrist 1.20 global thisyear
621 amott 1.1 main()