ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/RateMonHeavyIon/RateMonitorShifter.py
Revision: 1.4
Committed: Tue Nov 8 13:59:09 2011 UTC (13 years, 5 months ago) by grchrist
Content type: text/x-python
Branch: MAIN
Changes since 1.3: +12 -6 lines
Log Message:
Now if problem parsing lumi page returns default "junk" values rather than terminating script. Also in the expected rate calculation if div /0 error no longer terminates script but prints an error. Updated the heavy ion triggers in the monitor list and returned to removing the version label when comparing names of triggers as they are in the 2011 HI menu.

File Contents

# User Rev Content
1 grchrist 1.1 #!/usr/bin/env python
2    
3     from Page1Parser import Page1Parser
4     from GetRun import GetRun
5     import sys
6     import os
7     import cPickle as pickle
8     import getopt
9     import time
10     from ReadConfig import RateMonConfig
11     from colors import *
12    
13     WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
14     WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
15    
16     RefRunNameTemplate = "RefRuns/Run_%s.pk"
17    
18     # define a function that clears the terminal screen
19     def clear():
20     print("\x1B[2J")
21    
22     def usage():
23     print sys.argv[0]+" [Options]"
24     print "This script monitors some HLT paths for bizarre rates"
25     print "Options: "
26     print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
27     print "--AllowedDiff=<diff> Report only if difference in trigger rate is greater than <diff>%"
28     print "--ZerosOnly Only report if trigger rate is 0"
29     print "--FindL1Zeros Look for physics paths with 0 L1 rate"
30     print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
31     print "--ConfigFile=<file> Specify a configuration file (Default = defaults.cfg)"
32     print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
33     print "--LastLS=<ls> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
34     print " or set LSSlidingWindow = -1"
35     print "--help Print this help"
36    
37     def main():
38     try:
39     opt, args = getopt.getopt(sys.argv[1:],"",["IgnoreVersion","ZerosOnly","CompareRun=","AllowedDiff=",\
40     "IgnoreLowRate=","FindL1Zeros","ListIgnoredPaths","ConfigFile=",\
41     "FirstLS=","LastLS=","help"])
42     except getopt.GetoptError, err:
43     print str(err)
44     usage()
45     sys.exit(2)
46    
47     Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
48     for o,a in opt:
49     if o=="--ConfigFile":
50     Config.CFGfile=a
51     Config.ReadCFG()
52    
53     IgnoreVersion = False
54     ZerosOnly = False
55     ListIgnoredPaths = False
56     AllowedRateDiff = Config.DefAllowRateDiff
57     IgnoreThreshold = Config.DefAllowIgnoreThresh
58     CompareRunNum = ""
59     FindL1Zeros = Config.FindL1Zeros
60     FirstLS = 999999
61     EndEndLS = 111111
62     LastLS = FirstLS+10
63     RefRunNum = int(Config.ReferenceRun)
64     NWarnPSZero = 1
65 grchrist 1.2
66    
67    
68 grchrist 1.1
69     if Config.LSWindow > 0:
70     FirstLS = -1*Config.LSWindow
71    
72    
73    
74     for o,a in opt: # get options passed on the command line
75     if o=="--IgnoreVersion":
76     IgnoreVersion=True
77     elif o=="--ZerosOnly":
78     ZerosOnly=True
79     elif o=="--CompareRun":
80     CompareRunNum=a
81     elif o=="--AllowedDiff":
82     AllowedRateDiff = float(a)
83     elif o=="--IgnoreLowRate":
84     IgnoreThreshold = float(a)
85     elif o=="--FindL1Zeros":
86     FindL1Zeros=True
87     elif o=="--ListIgnoredPaths":
88     ListIgnoredPaths=True
89     elif o=="--FirstLS":
90     FirstLS = int(a)
91     elif o=="--LastLS":
92     EndEndLS = int(a)
93     elif o=="--ConfigFile":
94     print "Using your custom config file ... %s" % (a)
95     elif o=="--help":
96     usage()
97     sys.exit(0)
98     else:
99     print "Invalid Option "+a
100     sys.exit(1)
101     # end else
102     #end for
103    
104     try:
105     NPSZeros = 0
106     while True:
107     ### Get the most recent run
108     SaveRun=False
109     if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
110     RunListParser = Page1Parser()
111    
112     RunListParser._Parse(WBMRunInfoPage) # this is the page that lists all the runs in the last 24 hours with at least 1 trigger
113     RunListPage = RunListParser.ParsePage1()
114    
115     if RunListPage == '': # this will be '' if the mode of the most recent run is not l1_hlt_collisions/v*
116     print "WBM info page is "+WBMRunInfoPage
117     print "Most Recent run"+" is NOT collisions"
118     sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
119     CompareRunNum = RunListParser.RunNumber
120     print "Most Recent run is "+CompareRunNum
121     else:
122     SaveRun=False
123     HeadRunFile = RefRunNameTemplate % CompareRunNum
124     RefRunFile = RefRunNameTemplate % str(RefRunNum)
125 grchrist 1.2 print RefRunFile
126 grchrist 1.1
127    
128     if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
129     print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
130     print RefRunFile[:RefRunFile.rfind('/')]
131     sys.exit(0)
132    
133 grchrist 1.2
134    
135     ###if not os.path.exists(RefRunFile) and RefRunNum != 0: # if the reference run is not saved, get it from wbm
136     print "Updated reference run file"
137     RefParser = GetRun(RefRunNum, RefRunFile, True)
138 grchrist 1.4
139 grchrist 1.2 #else: # otherwise load it from the file
140     # if RefRunNum != 0:
141     # RefParser = pickle.load( open( RefRunFile ) )
142     # print "going to ref run file"
143 grchrist 1.1
144     if os.path.exists(HeadRunFile): # check if a run file for the run we want to compare already exists, it probably won't but just in case we don't have to interrogate WBM
145    
146     HeadParser = pickle.load( open( HeadRunFile ) )
147     else:
148 grchrist 1.3 print "loading for HeadParser WBM"
149 grchrist 1.1 HeadParser = GetRun(CompareRunNum,HeadRunFile,SaveRun,FirstLS,EndEndLS)
150    
151     if HeadParser.FirstLS==-1:
152     print bcolors.OKBLUE+">> Stable Beams NOT yet declared, be patient..."+bcolors.ENDC
153     sys.exit(0)
154    
155     HeadRates = HeadParser.TriggerRates
156    
157     write=sys.stdout.write
158    
159     write('#'*50+'\n')
160     write('Please include all info below in any elog posts\nPost to elog HLT on call\nIt may be helpful to experts\n')
161     write('#'*50+'\n')
162    
163     write('Script called with following command line:\n\n')
164    
165     for thing in sys.argv:
166     write (thing)
167     write ("\n")
168    
169     write('\nUsing the following parameters:\n')
170     write('IgnoreVersion = %s\n' % (IgnoreVersion) )
171     write('ZerosOnly = %s\n' % (ZerosOnly) )
172     write('CompareRun = %s\n' % (CompareRunNum) )
173     write('AllowedDiff = %s\n' % (AllowedRateDiff) )
174     write('IgnoreLowRate = %s\n' % (IgnoreThreshold) )
175     write('FindL1Zeros = %s\n' % (FindL1Zeros) )
176     write('ListIgnoredPaths = %s\n' %(ListIgnoredPaths))
177     write('FirstLS = %s\n' % (FirstLS))
178     write('LastLS = %s\n' % (EndEndLS))
179     write('ConfigFile = %s\n\n' % (Config.CFGfile))
180     ##write('RefRunFile = %s\n' % (RefRunFile))
181    
182    
183     nameBufLen=60
184     RateBuffLen=10
185     write('*'*(nameBufLen+3*RateBuffLen+10))
186     write ('\nCalculation using FirstLS = %s to LastLS = %s of run %s \n' % (HeadParser.FirstLS, HeadParser.LastLS, CompareRunNum))
187    
188     write("The average delivered lumi of these lumi sections is: ")
189     write(str(round(HeadParser.AvDeliveredLumi,1))+"e30"+"\n")
190     write("The average live (recorded) lumi of these lumi sections is: ")
191     if HeadParser.AvLiveLumi==0:
192     write(bcolors.FAIL)
193     elif HeadParser.AvLiveLumi<100:
194     write(bcolors.WARNING)
195    
196     write(str(round(HeadParser.AvLiveLumi,1))+"e30")
197     write(bcolors.ENDC+"\n")
198     write("The average deadtime of these lumi sections is: ")
199     if HeadParser.AvDeadtime > 5:
200     write(bcolors.FAIL)
201     elif HeadParser.AvDeadtime > 10:
202     write(bcolors.WARNING)
203     else:
204     write(bcolors.OKBLUE)
205     write(str(round(HeadParser.AvDeadtime,1))+"%")
206     write(bcolors.ENDC+"\n")
207    
208     print "Using prescale column "+str(HeadParser.PrescaleColumnString)
209    
210     if HeadParser.PrescaleColumnString=="0":
211     if NPSZeros >= NWarnPSZero:
212     write(bcolors.FAIL)
213     write("WARNING: You are using prescale column 0 in Lumi Section %s! This is the emergency column and should only be used if there is no other way to take data\n" % (HeadParser.LastLS) )
214     write("If this is a mistake FIX IT NOW \nif not, the TFM and HLT DOC must be informed\n\n")
215     write(bcolors.ENDC)
216     raw_input("Press ENTER to continue...")
217     write("\n\n")
218     else:
219     NPSZeros+=1
220     else:
221     NPSZeros=0
222    
223     IgnoredRates=[]
224     LargeRateDifference=False
225 grchrist 1.2 if not Config.CompareReference:
226     write('Using quadratic fit comparisions\n')
227     write('*'*(nameBufLen+3*RateBuffLen+10)+'\n')
228     write('* Trigger Name'+' '*(nameBufLen-17)+'* HLT PS * Actual * Expected * % Diff *\n')
229     write('*'*(nameBufLen+3*RateBuffLen+10)+'\n')
230    
231    
232 grchrist 1.3
233 grchrist 1.2 for headTrigN,headTrigRate,headTrigPS,headL1 in HeadRates:
234 grchrist 1.4 headTrigNNoVersion = headTrigN[:headTrigN.rfind('_')]
235     ##headTrigNNoVersion=headTrigN
236 grchrist 1.2 if not Config.AnalyzeTrigger(headTrigNNoVersion): ## SKIP triggers in the skip list
237     continue
238 grchrist 1.4
239 grchrist 1.2 ExpectedRate = Config.GetExpectedRate(headTrigNNoVersion,HeadParser.AvLiveLumi)
240     ExpectedRate = round((ExpectedRate / headTrigPS),2)
241     Prescale = round(headTrigPS,1)
242     PerDiff=0
243     if ExpectedRate>0:
244     PerDiff = int(round( (headTrigRate-ExpectedRate)/ExpectedRate,2 )*100)
245 grchrist 1.1 ##Write Line ##
246 grchrist 1.2 if headTrigRate==0:
247     write(bcolors.FAIL)
248     elif abs(PerDiff) > AllowedRateDiff:
249     write(bcolors.FAIL)
250     else:
251     write(bcolors.OKGREEN)
252     write('* '+headTrigN+' '*(nameBufLen-len(headTrigN)-5)+'*')
253     write(' '*(RateBuffLen-4-len(str(Prescale))-1)+str(Prescale)+' *')
254     write(' '*(RateBuffLen-len(str(headTrigRate))-1)+str(headTrigRate)+' *')
255     write(' '*(RateBuffLen-len(str(ExpectedRate))-1)+str(ExpectedRate)+' *')
256     if ExpectedRate>0:
257     if abs(PerDiff) > AllowedRateDiff/2:
258 grchrist 1.1 ###write(' '+' '*(RateBuffLen-len(str(PerDiff))-2)+str(PerDiff)+'%')
259 grchrist 1.2 write(' '+' '*(RateBuffLen-len(str(PerDiff))-4)+str(PerDiff)+'%')
260     else:
261     ###write(' good ')
262     write(' good'+' '*(RateBuffLen-len(str(PerDiff))-6)+str(PerDiff)+'%')
263 grchrist 1.1 else:
264 grchrist 1.2 write(' ')
265     write(' *')
266     if headTrigRate==0:
267     write(" << TRIGGER RATE IS ZERO! INFORM SHIFT LEADER & CALL HLT DOC")
268     elif abs(PerDiff) > AllowedRateDiff:
269     write(" << LARGE RATE DIFFERENCE: POST IN HLT on call ELOG")
270     LargeRateDifference=True # this means we automatically check the reference run
271     write(bcolors.ENDC+'\n')
272 grchrist 1.1
273     CallDOC=False
274 grchrist 1.2
275     LargeRateDifference=False
276    
277     if Config.CompareReference and RefRunNum != 0:
278 grchrist 1.1 if LargeRateDifference:
279     write(bcolors.WARNING)
280     print """
281     \n\n
282     ********************************************************************
283     A trigger in this run has a substantial difference from expectations.\n
284     Comparing the current run to a reference run
285     ********************************************************************
286     """
287     write(bcolors.ENDC)
288     else:
289 grchrist 1.2 write('Comparing to ref run: '+str(RefRunNum)+'\n')
290    
291    
292     write('*'*(nameBufLen+3*RateBuffLen+9)+'\n')
293     write('* Trigger Name'+' '*(nameBufLen-17)+'* HLT PS * Actual * Expected * % Diff *\n')
294     write('*'*(nameBufLen+3*RateBuffLen+9)+'\n')
295 grchrist 1.1
296     NotFound=[]
297     for headTrigN,headTrigRate,headTrigPS,headL1 in HeadRates:
298 grchrist 1.2
299 grchrist 1.4 headTrigNNoVersion = headTrigN[:headTrigN.rfind('_')]
300     ##headTrigNNoVersion = headTrigN###no versions in heavy ions 2010
301 grchrist 1.2
302 grchrist 1.1 if not Config.AnalyzeTrigger(headTrigNNoVersion): ## SKIP triggers in the skip list
303     continue
304 grchrist 1.4
305 grchrist 1.1 ExpectedRate=-1
306 grchrist 1.2 Prescale = round(headTrigPS,1)
307 grchrist 1.1 for refTrigN,refTrigRate,refTrigPS,refLa in RefParser.TriggerRates:
308     refTrigNNoVersion = refTrigN[:refTrigN.rfind('_')]
309     if refTrigNNoVersion == headTrigNNoVersion:
310 grchrist 1.4 try:
311     ExpectedRate = round(refTrigRate * HeadParser.AvLiveLumi/RefParser.AvLiveLumi/headTrigPS,2)
312    
313     except:
314     print "divide by 0 error in expected rate calculation"
315 grchrist 1.3
316 grchrist 1.1 break
317     if ExpectedRate==-1:
318     NotFound.append(headTrigNNoVersion)
319     continue
320     PerDiff=0
321     if ExpectedRate>0:
322     PerDiff = int(round( (headTrigRate-ExpectedRate)/ExpectedRate,2 )*100)
323     ##Write Line ##
324     if headTrigRate==0:
325     write(bcolors.FAIL)
326     elif abs(PerDiff) >AllowedRateDiff:
327     write(bcolors.FAIL)
328     else:
329     write(bcolors.OKGREEN)
330     write('* '+headTrigN+' '*(nameBufLen-len(headTrigN)-5)+'*')
331 grchrist 1.2 write(' '*(RateBuffLen-4-len(str(Prescale))-1)+str(Prescale)+' *')
332 grchrist 1.1 write(' '*(RateBuffLen-len(str(headTrigRate))-1)+str(headTrigRate)+' *')
333     write(' '*(RateBuffLen-len(str(ExpectedRate))-1)+str(ExpectedRate)+' *')
334     if ExpectedRate>0:
335     if abs(PerDiff) > AllowedRateDiff/2:
336     write(' '+' '*(RateBuffLen-len(str(PerDiff))-2)+str(PerDiff)+'%')
337     else:
338 grchrist 1.2 write(' good '+' '*(RateBuffLen-len(str(PerDiff))-7)+str(PerDiff)+'%')
339 grchrist 1.1 else:
340     write(' ')
341     write(' *')
342 grchrist 1.3
343 grchrist 1.1 if headTrigRate==0:
344     write(" << TRIGGER RATE IS ZERO! CALL HLT DOC")
345     CallDOC=True
346     elif abs(PerDiff) > AllowedRateDiff:
347     write(" << LARGE RATE DIFFERENCE WITH REFERENCE RUN")
348     CallDOC=True
349     write(bcolors.ENDC+'\n')
350 grchrist 1.3
351 grchrist 1.1 if CallDOC:
352     write(bcolors.FAIL)
353     print "\nSomething looks very wrong in this run"
354     print "If there is no obvious reason for this (subdetector out, etc.): **inform the SHIFT LEADER and call the HLT DOC!**"
355     raw_input("Press Enter to continue ... ")
356     write(bcolors.ENDC)
357    
358     if FindL1Zeros:
359     L1Zeros=[]
360     IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
361     for trigN,L1Pass,PSPass,PAccept,SeedName in HeadParser.Nevts:
362     ## Skip events in the skip list
363     trigNNoVersion = trigN[:trigN.rfind('_')]
364     if Config.AnalyzeTrigger(trigNNoVersion):
365     continue
366     ## if no events pass the L1, add it to the L1Zeros list if not already there
367     if SeedName in IgnoreBits:
368     continue
369     if L1Pass==0 and not SeedName in L1Zeros and SeedName.find("BeamGas")==-1 and SeedName.find('L1_SingleMuOpen')==-1 and SeedName.find('L1_BeamHalo')==-1 and SeedName.find('L1_InterBunch_Bsc')==-1 and SeedName.find('L1_PreCollisions')==-1:
370     L1Zeros.append(SeedName)
371     if len(L1Zeros) == 0:
372     pass
373     #print bcolors.OKGREEN+">>> L1 Seeds are fine"+bcolors.ENDC
374     else:
375     print "\n\n\n"
376     print ">>> The following seeds are used to seed HLT bits but accept 0 events:"
377     if len(L1Zeros)<10:
378     print bcolors.WARNING
379     for Seed in L1Zeros:
380     print Seed
381     print bcolors.ENDC
382     else:
383     print bcolors.FAIL
384     print "\n************************"
385     print "**MANY L1 seeds are 0!**"
386     print "** If in doubt **"
387     print "** Call the TFM/HLT! **"
388     print "************************"
389     print bcolors.ENDC
390     print '\n\n'
391     # end if find l1 zeros
392     print "Sleeping for 1 minute before repeating "
393     for iSleep in range(6):
394     for iDot in range(iSleep+1):
395     print ".",
396     print "."
397     time.sleep(10)
398     clear()
399     # End of while True
400     #end of try
401     except KeyboardInterrupt:
402     print "Quitting the program"
403    
404    
405     if __name__=='__main__':
406     main()