| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
#from AndrewGetRun import GetRun
|
| 4 |
from DatabaseParser import *
|
| 5 |
from ReadConfig import RateMonConfig
|
| 6 |
import sys
|
| 7 |
import os
|
| 8 |
import cPickle as pickle
|
| 9 |
import getopt
|
| 10 |
import time
|
| 11 |
from colors import *
|
| 12 |
from TablePrint import *
|
| 13 |
from AddTableInfo_db import MoreTableInfo
|
| 14 |
from math import *
|
| 15 |
|
| 16 |
WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
|
| 17 |
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
|
| 18 |
|
| 19 |
RefRunNameTemplate = "RefRuns/Run_%s.pk"
|
| 20 |
|
| 21 |
# define a function that clears the terminal screen
|
| 22 |
def clear():
|
| 23 |
print("\x1B[2J")
|
| 24 |
|
| 25 |
|
| 26 |
def usage():
|
| 27 |
print sys.argv[0]+" [Options]"
|
| 28 |
print "This script gets the current HLT trigger rates and compares them to a reference run"
|
| 29 |
print "Options: "
|
| 30 |
print "--AllowedDiff=<diff> Report only if difference in trigger rate is greater than <diff>%"
|
| 31 |
print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
|
| 32 |
print "--FindL1Zeros Look for physics paths with 0 L1 rate"
|
| 33 |
print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
|
| 34 |
print "--NumberLS=<#> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
|
| 35 |
print " or set LSSlidingWindow = -1"
|
| 36 |
print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate>"
|
| 37 |
print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
|
| 38 |
print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
|
| 39 |
print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
|
| 40 |
print "--ShowPSTriggers Show prescaled triggers in rate comparison"
|
| 41 |
print "--force Override the check for collisions run"
|
| 42 |
print "--help Print this help"
|
| 43 |
|
| 44 |
def main():
|
| 45 |
try:
|
| 46 |
opt, args = getopt.getopt(sys.argv[1:],"",["AllowedDiff=","CompareRun=","FindL1Zeros",\
|
| 47 |
"FirstLS=","NumberLS=","IgnoreLowRate=","ListIgnoredPaths",\
|
| 48 |
"PrintLumi","RefRun=","ShowPSTriggers","force","help"])
|
| 49 |
except getopt.GetoptError, err:
|
| 50 |
print str(err)
|
| 51 |
usage()
|
| 52 |
sys.exit(2)
|
| 53 |
|
| 54 |
Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
|
| 55 |
for o,a in opt:
|
| 56 |
if o=="--ConfigFile":
|
| 57 |
Config.CFGfile=a
|
| 58 |
Config.ReadCFG()
|
| 59 |
|
| 60 |
AllowedRateDiff = Config.DefAllowRateDiff
|
| 61 |
CompareRunNum = ""
|
| 62 |
FindL1Zeros = False
|
| 63 |
FirstLS = 9999
|
| 64 |
NumLS = -10
|
| 65 |
IgnoreThreshold = Config.DefAllowIgnoreThresh
|
| 66 |
ListIgnoredPaths = False
|
| 67 |
PrintLumi = False
|
| 68 |
RefRunNum = int(Config.ReferenceRun)
|
| 69 |
ShowPSTriggers = True
|
| 70 |
Force = False
|
| 71 |
|
| 72 |
if int(Config.ShifterMode):
|
| 73 |
print "ShifterMode!!"
|
| 74 |
else:
|
| 75 |
print "ExpertMode"
|
| 76 |
|
| 77 |
if Config.LSWindow > 0:
|
| 78 |
NumLS = -1*Config.LSWindow
|
| 79 |
|
| 80 |
for o,a in opt: # get options passed on the command line
|
| 81 |
if o=="--AllowedDiff":
|
| 82 |
AllowedRateDiff = float(a)/100.0
|
| 83 |
elif o=="--CompareRun":
|
| 84 |
CompareRunNum=int(a)
|
| 85 |
elif o=="--FindL1Zeros":
|
| 86 |
FindL1Zeros = True
|
| 87 |
elif o=="--FirstLS":
|
| 88 |
FirstLS = int(a)
|
| 89 |
elif o=="--NumberLS":
|
| 90 |
NumLS = int(a)
|
| 91 |
elif o=="--IgnoreLowRate":
|
| 92 |
IgnoreThreshold = float(a)
|
| 93 |
elif o=="--ListIgnoredPaths":
|
| 94 |
ListIgnoredPaths=True
|
| 95 |
elif o=="--PrintLumi":
|
| 96 |
PrintLumi = True
|
| 97 |
elif o=="--RefRun":
|
| 98 |
RefRunNum=int(a)
|
| 99 |
elif o=="--ShowPSTriggers":
|
| 100 |
ShowPSTriggers=True
|
| 101 |
elif o=="--force":
|
| 102 |
Force = True
|
| 103 |
elif o=="--help":
|
| 104 |
usage()
|
| 105 |
sys.exit(0)
|
| 106 |
else:
|
| 107 |
print "Invalid Option "+a
|
| 108 |
sys.exit(1)
|
| 109 |
|
| 110 |
|
| 111 |
RefLumisExists = False
|
| 112 |
|
| 113 |
"""
|
| 114 |
if RefRunNum > 0:
|
| 115 |
RefRates = {}
|
| 116 |
for Iterator in range(1,100):
|
| 117 |
if RefLumisExists: ## Quits at the end of a run
|
| 118 |
if max(RefLumis[0]) <= (Iterator+1)*10:
|
| 119 |
break
|
| 120 |
|
| 121 |
RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
|
| 122 |
print "RefRunFile=",RefRunFile
|
| 123 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 124 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 125 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 126 |
sys.exit(0)
|
| 127 |
|
| 128 |
if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
|
| 129 |
print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
|
| 130 |
print "Creating ..."
|
| 131 |
try:
|
| 132 |
RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
|
| 133 |
print "parsing"
|
| 134 |
except:
|
| 135 |
print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 136 |
continue
|
| 137 |
|
| 138 |
else: # otherwise load it from the file
|
| 139 |
RefParser = pickle.load( open( RefRunFile ) )
|
| 140 |
print "loading"
|
| 141 |
if not RefLumisExists:
|
| 142 |
RefLumis = RefParser.LumiInfo
|
| 143 |
RefLumisExists = True
|
| 144 |
|
| 145 |
try:
|
| 146 |
RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
|
| 147 |
LastSuccessfulIterator = Iterator
|
| 148 |
except:
|
| 149 |
print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 150 |
"""
|
| 151 |
|
| 152 |
RefRunFile = RefRunNameTemplate % RefRunNum
|
| 153 |
RefParser = DatabaseParser()
|
| 154 |
print "Reference Run: "+str(RefRunNum)
|
| 155 |
if RefRunNum > 0:
|
| 156 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 157 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 158 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 159 |
sys.exit(0)
|
| 160 |
return
|
| 161 |
if not os.path.exists(RefRunFile):
|
| 162 |
# create the reference run file
|
| 163 |
try:
|
| 164 |
RefParser.RunNumber = RefRunNum
|
| 165 |
RefParser.ParseRunSetup()
|
| 166 |
#RefParser.GetAllTriggerRatesByLS()
|
| 167 |
#RefParser.Save( RefRunFile )
|
| 168 |
except e:
|
| 169 |
print "PROBLEM GETTING REFERNCE RUN"
|
| 170 |
raise
|
| 171 |
else:
|
| 172 |
RefParser = pickle.load( open( RefRunFile ) )
|
| 173 |
|
| 174 |
# OK, Got the Reference Run
|
| 175 |
# Now get the most recent run
|
| 176 |
|
| 177 |
SaveRun = False
|
| 178 |
if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
|
| 179 |
CompareRunNum,isCol = GetLatestRunNumber()
|
| 180 |
|
| 181 |
if not isCol:
|
| 182 |
print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
|
| 183 |
print "Monitoring only stream A and Express"
|
| 184 |
#if not Force:
|
| 185 |
# sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
|
| 186 |
print "Most Recent run is "+str(CompareRunNum)
|
| 187 |
else:
|
| 188 |
CompareRunNum,isCol = GetLatestRunNumber(CompareRunNum)
|
| 189 |
|
| 190 |
HeadRunFile = RefRunNameTemplate % CompareRunNum
|
| 191 |
if os.path.exists(HeadRunFile): #check if a run file for the run we want to compare already exists
|
| 192 |
HeadParser = pickle.load( open( HeadRunFile ) )
|
| 193 |
else:
|
| 194 |
HeadParser = DatabaseParser()
|
| 195 |
HeadParser.RunNumber = CompareRunNum
|
| 196 |
HeadParser.ParseRunSetup()
|
| 197 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
| 198 |
if PrintLumi:
|
| 199 |
for LS in HeadParser.LumiInfo[0]:
|
| 200 |
try:
|
| 201 |
if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
|
| 202 |
continue
|
| 203 |
print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
|
| 204 |
except:
|
| 205 |
print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
|
| 206 |
|
| 207 |
sys.exit(0)
|
| 208 |
|
| 209 |
if RefRunNum == 0:
|
| 210 |
RefRates = 0
|
| 211 |
RefLumis = 0
|
| 212 |
LastSuccessfulIterator = 0
|
| 213 |
|
| 214 |
### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
|
| 215 |
#CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
|
| 216 |
|
| 217 |
try:
|
| 218 |
while True:
|
| 219 |
if not isCol:
|
| 220 |
clear()
|
| 221 |
MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
|
| 222 |
else:
|
| 223 |
RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths)
|
| 224 |
|
| 225 |
if FindL1Zeros:
|
| 226 |
CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
|
| 227 |
if int(Config.ShifterMode):
|
| 228 |
print "Shifter Mode. Continuing"
|
| 229 |
else:
|
| 230 |
print "Expert Mode. Quitting."
|
| 231 |
sys.exit(0)
|
| 232 |
|
| 233 |
|
| 234 |
print "Sleeping for 1 minute before repeating "
|
| 235 |
for iSleep in range(20):
|
| 236 |
write(".")
|
| 237 |
sys.stdout.flush()
|
| 238 |
time.sleep(3)
|
| 239 |
write(" Updating")
|
| 240 |
sys.stdout.flush()
|
| 241 |
#end while True
|
| 242 |
#end try
|
| 243 |
except KeyboardInterrupt:
|
| 244 |
print "Quitting. Peace Out."
|
| 245 |
|
| 246 |
|
| 247 |
def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths):
|
| 248 |
|
| 249 |
Header = ["Trigger Name","Actual","Expected","% Inc","Cur PS","Comments"]
|
| 250 |
Data = []
|
| 251 |
Warn = []
|
| 252 |
IgnoredRates=[]
|
| 253 |
|
| 254 |
[HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
|
| 255 |
##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
|
| 256 |
HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
|
| 257 |
[PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo
|
| 258 |
|
| 259 |
pkl_file = open("Fits/2011/Fit_HLT_10LS_Run176023to180252.pkl", 'rb')
|
| 260 |
FitInput = pickle.load(pkl_file)
|
| 261 |
pkl_file.close()
|
| 262 |
|
| 263 |
pkl_file = open("RefRuns/2011/Rates_HLT_10LS_JPAP.pkl", 'rb')
|
| 264 |
RefRatesInput = pickle.load(pkl_file)
|
| 265 |
pkl_file.close()
|
| 266 |
|
| 267 |
for HeadName in HeadUnprescaledRates:
|
| 268 |
## SKIP triggers in the skip list
|
| 269 |
## if not HeadTotalPrescales.has_key(HeadName): ## for whatever reason we have no prescale here, so skip (calibration paths)
|
| 270 |
## continue
|
| 271 |
## if not HeadTotalPrescales[HeadName]: ## prescale is thought to be 0
|
| 272 |
## continue
|
| 273 |
|
| 274 |
## unless we are Listing Ignored paths only look at triggers in the .list file specifed in defaults.cfg
|
| 275 |
if StripVersion(HeadName) not in Config.MonitorList and not ListIgnoredPaths:
|
| 276 |
continue
|
| 277 |
|
| 278 |
|
| 279 |
|
| 280 |
masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_L2", "HLT_Zero"]
|
| 281 |
masked_trig = False
|
| 282 |
for mask in masked_triggers:
|
| 283 |
if str(mask) in HeadName:
|
| 284 |
masked_trig = True
|
| 285 |
if masked_trig:
|
| 286 |
continue
|
| 287 |
|
| 288 |
skipTrig=False
|
| 289 |
TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)
|
| 290 |
|
| 291 |
if RefParser.RunNumber == 0: ## Use rate prediction functions
|
| 292 |
|
| 293 |
##PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),HeadAvInstLumi)
|
| 294 |
PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),FitInput,RefRatesInput,HeadAvLiveLumi,HeadAvDeliveredLumi)
|
| 295 |
|
| 296 |
if PSCorrectedExpectedRate[0] < 0: ##This means we don't have a prediction for this trigger
|
| 297 |
continue
|
| 298 |
## if not HeadTotalPrescales[HeadName]:
|
| 299 |
## print HeadName+ " has total prescale 0"
|
| 300 |
## continue
|
| 301 |
ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
|
| 302 |
PerDiff=0
|
| 303 |
if ExpectedRate>0:
|
| 304 |
PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
|
| 305 |
if abs(PerDiff) > max(AllowedRateDiff/max(sqrt(TriggerRate),sqrt(ExpectedRate)),AllowedRateDiff/2.0):
|
| 306 |
Warn.append(True)
|
| 307 |
else:
|
| 308 |
Warn.append(False)
|
| 309 |
else:
|
| 310 |
Warn.append(False)
|
| 311 |
|
| 312 |
if TriggerRate < IgnoreThreshold and ExpectedRate < IgnoreThreshold:
|
| 313 |
continue
|
| 314 |
|
| 315 |
VC = ""
|
| 316 |
|
| 317 |
Data.append([HeadName,TriggerRate,ExpectedRate,PerDiff,round(HeadUnprescaledRates[HeadName][1],1),VC])
|
| 318 |
|
| 319 |
else: ## Use a reference run
|
| 320 |
## cheap trick to only get triggers in list when in shifter mode
|
| 321 |
#print "shifter mode=",int(Config.ShifterMode)
|
| 322 |
if int(Config.ShifterMode)==1:
|
| 323 |
if not HeadParser.AvgL1Prescales[HeadParser.HLTSeed[HeadName]]==1:
|
| 324 |
continue
|
| 325 |
|
| 326 |
RefInstLumi = 0
|
| 327 |
RefIterator = 0
|
| 328 |
|
| 329 |
RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
|
| 330 |
RefLen = -10
|
| 331 |
|
| 332 |
##[RefUnprescaledRates, RefTotalPrescales, RefL1Prescales, RefTriggerRates] = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
|
| 333 |
RefUnprescaledRates = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
|
| 334 |
[RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefParser.GetLSRange(RefStartIndex,RefLen))
|
| 335 |
RefRate = -1
|
| 336 |
for k,v in RefUnprescaledRates.iteritems():
|
| 337 |
if StripVersion(HeadName) == StripVersion(k): # versions may not match
|
| 338 |
RefRate = v
|
| 339 |
|
| 340 |
ScaledRefRate = round( RefRate*HeadAvLiveLumi/RefAvLiveLumi/(HeadUnprescaledRates[HeadName][1]), 2 )
|
| 341 |
|
| 342 |
if ScaledRefRate == 0:
|
| 343 |
PerDiff = 100
|
| 344 |
else:
|
| 345 |
PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
|
| 346 |
|
| 347 |
if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
|
| 348 |
continue
|
| 349 |
|
| 350 |
if abs(PerDiff) > AllowedRateDiff:
|
| 351 |
Warn.append(True)
|
| 352 |
else:
|
| 353 |
Warn.append(False)
|
| 354 |
VC = ""
|
| 355 |
Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,round((HeadUnprescaledRates[HeadName][1]),1),VC])
|
| 356 |
|
| 357 |
clear()
|
| 358 |
PrettyPrintTable(Header,Data,[80,10,10,10,10,20],Warn)
|
| 359 |
|
| 360 |
MoreTableInfo(HeadParser,HeadLumiRange,Config)
|
| 361 |
|
| 362 |
def CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
|
| 363 |
print "checking trigger list"
|
| 364 |
|
| 365 |
def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
|
| 366 |
L1Zeros=[]
|
| 367 |
IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
|
| 368 |
for key in HeadParser.TriggerRates:
|
| 369 |
## Skip events in the skip list
|
| 370 |
skipTrig=False
|
| 371 |
##for trig in Config.ExcludeList:
|
| 372 |
##if not trigN.find(trig) == -1:
|
| 373 |
##skipTrig=True
|
| 374 |
##break
|
| 375 |
if skipTrig:
|
| 376 |
continue
|
| 377 |
## if no events pass the L1, add it to the L1Zeros list if not already there
|
| 378 |
if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
|
| 379 |
if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
|
| 380 |
|
| 381 |
L1Zeros.append(HeadParser.TriggerRates[key][4])
|
| 382 |
print "L1Zeros=", L1Zeros
|
| 383 |
|
| 384 |
if len(L1Zeros) == 0:
|
| 385 |
#print "It looks like no masked L1 bits seed trigger paths"
|
| 386 |
pass
|
| 387 |
else:
|
| 388 |
print "The following seeds are used to seed HLT bits but accept 0 events:"
|
| 389 |
#print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
|
| 390 |
for Seed in L1Zeros:
|
| 391 |
print Seed
|
| 392 |
|
| 393 |
if __name__=='__main__':
|
| 394 |
main()
|
