| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
from Page1Parser import Page1Parser
|
| 4 |
from GetRun import GetRun
|
| 5 |
import sys
|
| 6 |
import os
|
| 7 |
import cPickle as pickle
|
| 8 |
import getopt
|
| 9 |
import time
|
| 10 |
from ReadConfig import RateMonConfig
|
| 11 |
from colors import *
|
| 12 |
|
| 13 |
WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
|
| 14 |
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
|
| 15 |
|
| 16 |
RefRunNameTemplate = "RefRuns/Run_%s.pk"
|
| 17 |
|
| 18 |
# define a function that clears the terminal screen
|
| 19 |
def clear():
|
| 20 |
print("\x1B[2J")
|
| 21 |
|
| 22 |
def usage():
|
| 23 |
print sys.argv[0]+" [Options]"
|
| 24 |
print "This script monitors some HLT paths for bizarre rates"
|
| 25 |
print "Options: "
|
| 26 |
print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
|
| 27 |
print "--AllowedDiff=<diff> Report only if difference in trigger rate is greater than <diff>%"
|
| 28 |
print "--ZerosOnly Only report if trigger rate is 0"
|
| 29 |
print "--FindL1Zeros Look for physics paths with 0 L1 rate"
|
| 30 |
print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
|
| 31 |
print "--ConfigFile=<file> Specify a configuration file (Default = defaults.cfg)"
|
| 32 |
print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
|
| 33 |
print "--LastLS=<ls> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
|
| 34 |
print " or set LSSlidingWindow = -1"
|
| 35 |
print "--help Print this help"
|
| 36 |
|
| 37 |
def main():
|
| 38 |
try:
|
| 39 |
opt, args = getopt.getopt(sys.argv[1:],"",["IgnoreVersion","ZerosOnly","CompareRun=","AllowedDiff=",\
|
| 40 |
"IgnoreLowRate=","FindL1Zeros","ListIgnoredPaths","ConfigFile=",\
|
| 41 |
"FirstLS=","LastLS=","help"])
|
| 42 |
except getopt.GetoptError, err:
|
| 43 |
print str(err)
|
| 44 |
usage()
|
| 45 |
sys.exit(2)
|
| 46 |
|
| 47 |
Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
|
| 48 |
for o,a in opt:
|
| 49 |
if o=="--ConfigFile":
|
| 50 |
Config.CFGfile=a
|
| 51 |
Config.ReadCFG()
|
| 52 |
|
| 53 |
IgnoreVersion = False
|
| 54 |
ZerosOnly = False
|
| 55 |
ListIgnoredPaths = False
|
| 56 |
AllowedRateDiff = Config.DefAllowRateDiff
|
| 57 |
IgnoreThreshold = Config.DefAllowIgnoreThresh
|
| 58 |
CompareRunNum = ""
|
| 59 |
FindL1Zeros = Config.FindL1Zeros
|
| 60 |
FirstLS = 999999
|
| 61 |
EndEndLS = 111111
|
| 62 |
LastLS = FirstLS+10
|
| 63 |
RefRunNum = int(Config.ReferenceRun)
|
| 64 |
NWarnPSZero = 1
|
| 65 |
|
| 66 |
if Config.LSWindow > 0:
|
| 67 |
FirstLS = -1*Config.LSWindow
|
| 68 |
|
| 69 |
|
| 70 |
|
| 71 |
for o,a in opt: # get options passed on the command line
|
| 72 |
if o=="--IgnoreVersion":
|
| 73 |
IgnoreVersion=True
|
| 74 |
elif o=="--ZerosOnly":
|
| 75 |
ZerosOnly=True
|
| 76 |
elif o=="--CompareRun":
|
| 77 |
CompareRunNum=a
|
| 78 |
elif o=="--AllowedDiff":
|
| 79 |
AllowedRateDiff = float(a)
|
| 80 |
elif o=="--IgnoreLowRate":
|
| 81 |
IgnoreThreshold = float(a)
|
| 82 |
elif o=="--FindL1Zeros":
|
| 83 |
FindL1Zeros=True
|
| 84 |
elif o=="--ListIgnoredPaths":
|
| 85 |
ListIgnoredPaths=True
|
| 86 |
elif o=="--FirstLS":
|
| 87 |
FirstLS = int(a)
|
| 88 |
elif o=="--LastLS":
|
| 89 |
EndEndLS = int(a)
|
| 90 |
elif o=="--ConfigFile":
|
| 91 |
print "Using your custom config file ... %s" % (a)
|
| 92 |
elif o=="--help":
|
| 93 |
usage()
|
| 94 |
sys.exit(0)
|
| 95 |
else:
|
| 96 |
print "Invalid Option "+a
|
| 97 |
sys.exit(1)
|
| 98 |
# end else
|
| 99 |
#end for
|
| 100 |
|
| 101 |
try:
|
| 102 |
NPSZeros = 0
|
| 103 |
while True:
|
| 104 |
### Get the most recent run
|
| 105 |
SaveRun=False
|
| 106 |
if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
|
| 107 |
RunListParser = Page1Parser()
|
| 108 |
|
| 109 |
RunListParser._Parse(WBMRunInfoPage) # this is the page that lists all the runs in the last 24 hours with at least 1 trigger
|
| 110 |
RunListPage = RunListParser.ParsePage1()
|
| 111 |
|
| 112 |
if RunListPage == '': # this will be '' if the mode of the most recent run is not l1_hlt_collisions/v*
|
| 113 |
print "WBM info page is "+WBMRunInfoPage
|
| 114 |
print "Most Recent run"+" is NOT collisions"
|
| 115 |
sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
|
| 116 |
CompareRunNum = RunListParser.RunNumber
|
| 117 |
print "Most Recent run is "+CompareRunNum
|
| 118 |
else:
|
| 119 |
SaveRun=False
|
| 120 |
HeadRunFile = RefRunNameTemplate % CompareRunNum
|
| 121 |
RefRunFile = RefRunNameTemplate % str(RefRunNum)
|
| 122 |
|
| 123 |
|
| 124 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 125 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 126 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 127 |
sys.exit(0)
|
| 128 |
|
| 129 |
if not os.path.exists(RefRunFile) and RefRunNum != 0: # if the reference run is not saved, get it from wbm
|
| 130 |
print "Updated reference run file"
|
| 131 |
RefParser = GetRun(RefRunNum, RefRunFile, True)
|
| 132 |
else: # otherwise load it from the file
|
| 133 |
if RefRunNum != 0:
|
| 134 |
RefParser = pickle.load( open( RefRunFile ) )
|
| 135 |
print "going to ref run file"
|
| 136 |
|
| 137 |
if os.path.exists(HeadRunFile): # check if a run file for the run we want to compare already exists, it probably won't but just in case we don't have to interrogate WBM
|
| 138 |
|
| 139 |
HeadParser = pickle.load( open( HeadRunFile ) )
|
| 140 |
else:
|
| 141 |
HeadParser = GetRun(CompareRunNum,HeadRunFile,SaveRun,FirstLS,EndEndLS)
|
| 142 |
|
| 143 |
if HeadParser.FirstLS==-1:
|
| 144 |
print bcolors.OKBLUE+">> Stable Beams NOT yet declared, be patient..."+bcolors.ENDC
|
| 145 |
sys.exit(0)
|
| 146 |
|
| 147 |
HeadRates = HeadParser.TriggerRates
|
| 148 |
|
| 149 |
write=sys.stdout.write
|
| 150 |
|
| 151 |
write('#'*50+'\n')
|
| 152 |
write('Please include all info below in any elog posts\nPost to elog HLT on call\nIt may be helpful to experts\n')
|
| 153 |
write('#'*50+'\n')
|
| 154 |
|
| 155 |
write('Script called with following command line:\n\n')
|
| 156 |
|
| 157 |
for thing in sys.argv:
|
| 158 |
write (thing)
|
| 159 |
write ("\n")
|
| 160 |
|
| 161 |
write('\nUsing the following parameters:\n')
|
| 162 |
write('IgnoreVersion = %s\n' % (IgnoreVersion) )
|
| 163 |
write('ZerosOnly = %s\n' % (ZerosOnly) )
|
| 164 |
write('CompareRun = %s\n' % (CompareRunNum) )
|
| 165 |
write('AllowedDiff = %s\n' % (AllowedRateDiff) )
|
| 166 |
write('IgnoreLowRate = %s\n' % (IgnoreThreshold) )
|
| 167 |
write('FindL1Zeros = %s\n' % (FindL1Zeros) )
|
| 168 |
write('ListIgnoredPaths = %s\n' %(ListIgnoredPaths))
|
| 169 |
write('FirstLS = %s\n' % (FirstLS))
|
| 170 |
write('LastLS = %s\n' % (EndEndLS))
|
| 171 |
write('ConfigFile = %s\n\n' % (Config.CFGfile))
|
| 172 |
##write('RefRunFile = %s\n' % (RefRunFile))
|
| 173 |
|
| 174 |
|
| 175 |
nameBufLen=60
|
| 176 |
RateBuffLen=10
|
| 177 |
write('*'*(nameBufLen+3*RateBuffLen+10))
|
| 178 |
write ('\nCalculation using FirstLS = %s to LastLS = %s of run %s \n' % (HeadParser.FirstLS, HeadParser.LastLS, CompareRunNum))
|
| 179 |
|
| 180 |
write("The average delivered lumi of these lumi sections is: ")
|
| 181 |
write(str(round(HeadParser.AvDeliveredLumi,1))+"e30"+"\n")
|
| 182 |
write("The average live (recorded) lumi of these lumi sections is: ")
|
| 183 |
if HeadParser.AvLiveLumi==0:
|
| 184 |
write(bcolors.FAIL)
|
| 185 |
elif HeadParser.AvLiveLumi<100:
|
| 186 |
write(bcolors.WARNING)
|
| 187 |
|
| 188 |
write(str(round(HeadParser.AvLiveLumi,1))+"e30")
|
| 189 |
write(bcolors.ENDC+"\n")
|
| 190 |
|
| 191 |
###DT from calculation
|
| 192 |
write("The average deadtime of these lumi sections is: ")
|
| 193 |
if HeadParser.AvDeadtime > 5:
|
| 194 |
write(bcolors.FAIL)
|
| 195 |
elif HeadParser.AvDeadtime > 10:
|
| 196 |
write(bcolors.WARNING)
|
| 197 |
else:
|
| 198 |
write(bcolors.OKBLUE)
|
| 199 |
write(str(round(HeadParser.AvDeadtime,1))+"%")
|
| 200 |
write(bcolors.ENDC+"\n")
|
| 201 |
|
| 202 |
|
| 203 |
### DT from L1 page
|
| 204 |
write("The DeadTimeBeamActive of these lumi sections is: ")
|
| 205 |
if float(HeadParser.DeadTime[1]) > 5:
|
| 206 |
write(bcolors.FAIL)
|
| 207 |
elif float(HeadParser.DeadTime[1]) > 10:
|
| 208 |
write(bcolors.WARNING)
|
| 209 |
else:
|
| 210 |
write(bcolors.OKBLUE)
|
| 211 |
write(str(round(float(HeadParser.DeadTime[1]),1))+"%")
|
| 212 |
write(bcolors.ENDC+"\n")
|
| 213 |
write("The DeadTime of these lumi sections is: ")
|
| 214 |
write(str(round(float(HeadParser.DeadTime[0]),1))+"%\n")
|
| 215 |
|
| 216 |
print "Using prescale column "+str(HeadParser.PrescaleColumnString)
|
| 217 |
|
| 218 |
if HeadParser.PrescaleColumnString=="0":
|
| 219 |
if NPSZeros >= NWarnPSZero:
|
| 220 |
write(bcolors.FAIL)
|
| 221 |
write("WARNING: You are using prescale column 0 in Lumi Section %s! This is the emergency column and should only be used if there is no other way to take data\n" % (HeadParser.LastLS) )
|
| 222 |
write("If this is a mistake FIX IT NOW \nif not, the TFM and HLT DOC must be informed\n\n")
|
| 223 |
write(bcolors.ENDC)
|
| 224 |
raw_input("Press ENTER to continue...")
|
| 225 |
write("\n\n")
|
| 226 |
else:
|
| 227 |
NPSZeros+=1
|
| 228 |
else:
|
| 229 |
NPSZeros=0
|
| 230 |
|
| 231 |
|
| 232 |
write('*'*(nameBufLen+3*RateBuffLen+10)+'\n')
|
| 233 |
write('* Trigger Name'+' '*(nameBufLen-17)+'* PS * Actual * Expected * % Diff *\n')
|
| 234 |
write('*'*(nameBufLen+3*RateBuffLen+10)+'\n')
|
| 235 |
|
| 236 |
IgnoredRates=[]
|
| 237 |
LargeRateDifference=False
|
| 238 |
for headTrigN,headTrigRate,headTrigPS,headL1 in HeadRates:
|
| 239 |
headTrigNNoVersion = headTrigN[:headTrigN.rfind('_')]
|
| 240 |
if not Config.AnalyzeTrigger(headTrigNNoVersion): ## SKIP triggers in the skip list
|
| 241 |
continue
|
| 242 |
ExpectedRate = Config.GetExpectedRate(headTrigNNoVersion,HeadParser.AvLiveLumi)
|
| 243 |
ExpectedRate = round((ExpectedRate / headTrigPS),2)
|
| 244 |
Prescale = round(headTrigPS,1)
|
| 245 |
PerDiff=0
|
| 246 |
if ExpectedRate>0:
|
| 247 |
PerDiff = int(round( (headTrigRate-ExpectedRate)/ExpectedRate,2 )*100)
|
| 248 |
##Write Line ##
|
| 249 |
if headTrigRate==0:
|
| 250 |
write(bcolors.FAIL)
|
| 251 |
elif abs(PerDiff) > AllowedRateDiff:
|
| 252 |
write(bcolors.FAIL)
|
| 253 |
else:
|
| 254 |
write(bcolors.OKGREEN)
|
| 255 |
write('* '+headTrigN+' '*(nameBufLen-len(headTrigN)-5)+'*')
|
| 256 |
write(' '*(RateBuffLen-4-len(str(Prescale))-1)+str(Prescale)+' *')
|
| 257 |
write(' '*(RateBuffLen-len(str(headTrigRate))-1)+str(headTrigRate)+' *')
|
| 258 |
write(' '*(RateBuffLen-len(str(ExpectedRate))-1)+str(ExpectedRate)+' *')
|
| 259 |
if ExpectedRate>0:
|
| 260 |
if abs(PerDiff) > AllowedRateDiff/2:
|
| 261 |
###write(' '+' '*(RateBuffLen-len(str(PerDiff))-2)+str(PerDiff)+'%')
|
| 262 |
write(' '+' '*(RateBuffLen-len(str(PerDiff))-4)+str(PerDiff)+'%')
|
| 263 |
else:
|
| 264 |
###write(' good ')
|
| 265 |
write(' good'+' '*(RateBuffLen-len(str(PerDiff))-6)+str(PerDiff)+'%')
|
| 266 |
else:
|
| 267 |
write(' ')
|
| 268 |
write(' *')
|
| 269 |
if headTrigRate==0:
|
| 270 |
write(" << TRIGGER RATE IS ZERO! INFORM SHIFT LEADER & CALL HLT DOC")
|
| 271 |
elif abs(PerDiff) > AllowedRateDiff:
|
| 272 |
write(" << LARGE RATE DIFFERENCE: POST IN HLT on call ELOG")
|
| 273 |
LargeRateDifference=True # this means we automatically check the reference run
|
| 274 |
write(bcolors.ENDC+'\n')
|
| 275 |
|
| 276 |
CallDOC=False
|
| 277 |
if ( LargeRateDifference or Config.CompareReference ) and RefRunNum != 0:
|
| 278 |
if LargeRateDifference:
|
| 279 |
write(bcolors.WARNING)
|
| 280 |
print """
|
| 281 |
\n\n
|
| 282 |
********************************************************************
|
| 283 |
A trigger in this run has a substantial difference from expectations.\n
|
| 284 |
Comparing the current run to a reference run
|
| 285 |
********************************************************************
|
| 286 |
"""
|
| 287 |
write(bcolors.ENDC)
|
| 288 |
else:
|
| 289 |
print "\n\n Comparing to reference Run:\n\n"
|
| 290 |
|
| 291 |
write('*'*(nameBufLen+3*RateBuffLen+2)+'\n')
|
| 292 |
write('* Trigger Name'+' '*(nameBufLen-17)+'* Actual * Expected * % Diff *\n')
|
| 293 |
write('*'*(nameBufLen+3*RateBuffLen+2)+'\n')
|
| 294 |
|
| 295 |
NotFound=[]
|
| 296 |
for headTrigN,headTrigRate,headTrigPS,headL1 in HeadRates:
|
| 297 |
headTrigNNoVersion = headTrigN[:headTrigN.rfind('_')]
|
| 298 |
if not Config.AnalyzeTrigger(headTrigNNoVersion): ## SKIP triggers in the skip list
|
| 299 |
continue
|
| 300 |
ExpectedRate=-1
|
| 301 |
for refTrigN,refTrigRate,refTrigPS,refLa in RefParser.TriggerRates:
|
| 302 |
refTrigNNoVersion = refTrigN[:refTrigN.rfind('_')]
|
| 303 |
if refTrigNNoVersion == headTrigNNoVersion:
|
| 304 |
ExpectedRate = round(refTrigRate * HeadParser.AvLiveLumi/RefParser.AvLiveLumi,2)
|
| 305 |
break
|
| 306 |
if ExpectedRate==-1:
|
| 307 |
NotFound.append(headTrigNNoVersion)
|
| 308 |
continue
|
| 309 |
PerDiff=0
|
| 310 |
if ExpectedRate>0:
|
| 311 |
PerDiff = int(round( (headTrigRate-ExpectedRate)/ExpectedRate,2 )*100)
|
| 312 |
##Write Line ##
|
| 313 |
if headTrigRate==0:
|
| 314 |
write(bcolors.FAIL)
|
| 315 |
elif abs(PerDiff) >AllowedRateDiff:
|
| 316 |
write(bcolors.FAIL)
|
| 317 |
else:
|
| 318 |
write(bcolors.OKGREEN)
|
| 319 |
write('* '+headTrigN+' '*(nameBufLen-len(headTrigN)-5)+'*')
|
| 320 |
write(' '*(RateBuffLen-len(str(headTrigRate))-1)+str(headTrigRate)+' *')
|
| 321 |
write(' '*(RateBuffLen-len(str(ExpectedRate))-1)+str(ExpectedRate)+' *')
|
| 322 |
if ExpectedRate>0:
|
| 323 |
if abs(PerDiff) > AllowedRateDiff/2:
|
| 324 |
write(' '+' '*(RateBuffLen-len(str(PerDiff))-2)+str(PerDiff)+'%')
|
| 325 |
else:
|
| 326 |
write(' good ')
|
| 327 |
else:
|
| 328 |
write(' ')
|
| 329 |
write(' *')
|
| 330 |
if headTrigRate==0:
|
| 331 |
write(" << TRIGGER RATE IS ZERO! CALL HLT DOC")
|
| 332 |
CallDOC=True
|
| 333 |
elif abs(PerDiff) > AllowedRateDiff:
|
| 334 |
write(" << LARGE RATE DIFFERENCE WITH REFERENCE RUN")
|
| 335 |
CallDOC=True
|
| 336 |
write(bcolors.ENDC+'\n')
|
| 337 |
|
| 338 |
if CallDOC:
|
| 339 |
write(bcolors.FAIL)
|
| 340 |
print "\nSomething looks very wrong in this run"
|
| 341 |
print "If there is no obvious reason for this (subdetector out, etc.): **inform the SHIFT LEADER and call the HLT DOC!**"
|
| 342 |
raw_input("Press Enter to continue ... ")
|
| 343 |
write(bcolors.ENDC)
|
| 344 |
|
| 345 |
if FindL1Zeros:
|
| 346 |
L1Zeros=[]
|
| 347 |
IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
|
| 348 |
for trigN,L1Pass,PSPass,PAccept,SeedName in HeadParser.Nevts:
|
| 349 |
## Skip events in the skip list
|
| 350 |
trigNNoVersion = trigN[:trigN.rfind('_')]
|
| 351 |
if Config.AnalyzeTrigger(trigNNoVersion):
|
| 352 |
continue
|
| 353 |
## if no events pass the L1, add it to the L1Zeros list if not already there
|
| 354 |
if SeedName in IgnoreBits:
|
| 355 |
continue
|
| 356 |
if L1Pass==0 and not SeedName in L1Zeros and SeedName.find("BeamGas")==-1 and SeedName.find('L1_SingleMuOpen')==-1 and SeedName.find('L1_BeamHalo')==-1 and SeedName.find('L1_InterBunch_Bsc')==-1 and SeedName.find('L1_PreCollisions')==-1:
|
| 357 |
L1Zeros.append(SeedName)
|
| 358 |
if len(L1Zeros) == 0:
|
| 359 |
pass
|
| 360 |
#print bcolors.OKGREEN+">>> L1 Seeds are fine"+bcolors.ENDC
|
| 361 |
else:
|
| 362 |
print "\n\n\n"
|
| 363 |
print ">>> The following seeds are used to seed HLT bits but accept 0 events:"
|
| 364 |
if len(L1Zeros)<10:
|
| 365 |
print bcolors.WARNING
|
| 366 |
for Seed in L1Zeros:
|
| 367 |
print Seed
|
| 368 |
print bcolors.ENDC
|
| 369 |
else:
|
| 370 |
print bcolors.FAIL
|
| 371 |
print "\n************************"
|
| 372 |
print "**MANY L1 seeds are 0!**"
|
| 373 |
print "** If in doubt **"
|
| 374 |
print "** Call the TFM/HLT! **"
|
| 375 |
print "************************"
|
| 376 |
print bcolors.ENDC
|
| 377 |
print '\n\n'
|
| 378 |
# end if find l1 zeros
|
| 379 |
print "Sleeping for 1 minute before repeating "
|
| 380 |
for iSleep in range(6):
|
| 381 |
for iDot in range(iSleep+1):
|
| 382 |
print ".",
|
| 383 |
print "."
|
| 384 |
time.sleep(10)
|
| 385 |
clear()
|
| 386 |
# End of while True
|
| 387 |
#end of try
|
| 388 |
except KeyboardInterrupt:
|
| 389 |
print "Quitting the program"
|
| 390 |
|
| 391 |
|
| 392 |
if __name__=='__main__':
|
| 393 |
main()
|
