| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
from AndrewGetRun import GetRun
|
| 4 |
from AndrewWBMParser import AndrewWBMParser
|
| 5 |
from ReadConfig import RateMonConfig
|
| 6 |
import sys
|
| 7 |
import os
|
| 8 |
import cPickle as pickle
|
| 9 |
import getopt
|
| 10 |
import time
|
| 11 |
from colors import *
|
| 12 |
from TablePrint import *
|
| 13 |
from AddTableInfo import MoreTableInfo
|
| 14 |
from math import *
|
| 15 |
|
| 16 |
WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
|
| 17 |
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
|
| 18 |
|
| 19 |
RefRunNameTemplate = "RefRuns/Run_%s.pk"
|
| 20 |
|
| 21 |
# define a function that clears the terminal screen
|
| 22 |
def clear():
|
| 23 |
print("\x1B[2J")
|
| 24 |
|
| 25 |
|
| 26 |
def usage():
|
| 27 |
print sys.argv[0]+" [Options]"
|
| 28 |
print "This script gets the current HLT trigger rates and compares them to a reference run"
|
| 29 |
print "Options: "
|
| 30 |
print "--AllowedDiff=<diff> Report only if difference in trigger rate is greater than <diff>% (Default = "+str(DefaultAllowedRateDiff*100)+"% )"
|
| 31 |
print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
|
| 32 |
print "--FindL1Zeros Look for physics paths with 0 L1 rate"
|
| 33 |
print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
|
| 34 |
print "--LastLS=<ls> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
|
| 35 |
print " or set LSSlidingWindow = -1"
|
| 36 |
print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate> (Default = "+str(DefaultIgnoreThreshold)+")"
|
| 37 |
print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
|
| 38 |
print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
|
| 39 |
print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
|
| 40 |
print "--ShowPSTriggers Show prescaled triggers in rate comparison"
|
| 41 |
print "--help Print this help"
|
| 42 |
|
| 43 |
def main():
|
| 44 |
try:
|
| 45 |
opt, args = getopt.getopt(sys.argv[1:],"",["AllowedDiff=","CompareRun=","FindL1Zeros",\
|
| 46 |
"FirstLS=","LastLS=","IgnoreLowRate=","ListIgnoredPaths",\
|
| 47 |
"PrintLumi","RefRun=","ShowPSTriggers","help"])
|
| 48 |
except getopt.GetoptError, err:
|
| 49 |
print str(err)
|
| 50 |
usage()
|
| 51 |
sys.exit(2)
|
| 52 |
|
| 53 |
Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
|
| 54 |
for o,a in opt:
|
| 55 |
if o=="--ConfigFile":
|
| 56 |
Config.CFGfile=a
|
| 57 |
Config.ReadCFG()
|
| 58 |
|
| 59 |
AllowedRateDiff = Config.DefAllowRateDiff
|
| 60 |
CompareRunNum = ""
|
| 61 |
FindL1Zeros = True
|
| 62 |
FirstLS = 999999
|
| 63 |
LastLS = 111111
|
| 64 |
IgnoreThreshold = Config.DefAllowIgnoreThresh
|
| 65 |
ListIgnoredPaths = False
|
| 66 |
PrintLumi = False
|
| 67 |
RefRunNum = int(Config.ReferenceRun)
|
| 68 |
ShowPSTriggers = True
|
| 69 |
|
| 70 |
|
| 71 |
if Config.ShifterMode:
|
| 72 |
print "ShifterMode!!"
|
| 73 |
else:
|
| 74 |
print "ExpertMode"
|
| 75 |
|
| 76 |
if Config.LSWindow > 0:
|
| 77 |
FirstLS = -1*Config.LSWindow
|
| 78 |
|
| 79 |
for o,a in opt: # get options passed on the command line
|
| 80 |
if o=="--AllowedDiff":
|
| 81 |
AllowedRateDiff = float(a)/100.0
|
| 82 |
elif o=="--CompareRun":
|
| 83 |
CompareRunNum=int(a)
|
| 84 |
elif o=="--FindL1Zeros":
|
| 85 |
FindL1Zeros = True
|
| 86 |
elif o=="--FirstLS":
|
| 87 |
FirstLS = int(a)
|
| 88 |
elif o=="--LastLS":
|
| 89 |
LastLS = int(a)
|
| 90 |
elif o=="--IgnoreLowRate":
|
| 91 |
IgnoreThreshold = float(a)
|
| 92 |
elif o=="--ListIgnoredPaths":
|
| 93 |
ListIgnoredPaths=True
|
| 94 |
elif o=="--PrintLumi":
|
| 95 |
PrintLumi = True
|
| 96 |
elif o=="--RefRun":
|
| 97 |
RefRunNum=int(a)
|
| 98 |
elif o=="--ShowPSTriggers":
|
| 99 |
ShowPSTriggers=True
|
| 100 |
elif o=="--help":
|
| 101 |
usage()
|
| 102 |
sys.exit(0)
|
| 103 |
else:
|
| 104 |
print "Invalid Option "+a
|
| 105 |
sys.exit(1)
|
| 106 |
|
| 107 |
|
| 108 |
RefLumisExists = False
|
| 109 |
if RefRunNum > 0:
|
| 110 |
RefRates = {}
|
| 111 |
for Iterator in range(1,100):
|
| 112 |
if RefLumisExists: ## Quits at the end of a run
|
| 113 |
if max(RefLumis[0]) <= (Iterator+1)*10:
|
| 114 |
break
|
| 115 |
|
| 116 |
RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
|
| 117 |
print "RefRunFile=",RefRunFile
|
| 118 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 119 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 120 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 121 |
sys.exit(0)
|
| 122 |
|
| 123 |
if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
|
| 124 |
print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
|
| 125 |
print "Creating ..."
|
| 126 |
try:
|
| 127 |
RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
|
| 128 |
print "parsing"
|
| 129 |
except:
|
| 130 |
print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 131 |
continue
|
| 132 |
|
| 133 |
else: # otherwise load it from the file
|
| 134 |
RefParser = pickle.load( open( RefRunFile ) )
|
| 135 |
print "loading"
|
| 136 |
if not RefLumisExists:
|
| 137 |
RefLumis = RefParser.LumiInfo
|
| 138 |
RefLumisExists = True
|
| 139 |
|
| 140 |
try:
|
| 141 |
RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
|
| 142 |
LastSuccessfulIterator = Iterator
|
| 143 |
except:
|
| 144 |
print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 145 |
|
| 146 |
|
| 147 |
# OK, Got the Reference Run
|
| 148 |
# Now get the most recent run
|
| 149 |
|
| 150 |
SaveRun = False
|
| 151 |
if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
|
| 152 |
RunListParser = AndrewWBMParser()
|
| 153 |
|
| 154 |
RunListParser._Parse(WBMRunInfoPage) # this is the page that lists all the runs in the last 24 hours with at least 1 trigger
|
| 155 |
RunListPage = RunListParser.ParsePage1()
|
| 156 |
if RunListPage == '': #this will be '' if the mode of the most recent run is not l1_hlt_collisions/v*
|
| 157 |
print "Most Recent run, "+str(RunListParser.RunNumber)+", is NOT collisions"
|
| 158 |
sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
|
| 159 |
CompareRunNum = RunListParser.RunNumber
|
| 160 |
print "Most Recent run is "+CompareRunNum
|
| 161 |
|
| 162 |
HeadRunFile = RefRunNameTemplate % CompareRunNum
|
| 163 |
|
| 164 |
|
| 165 |
if os.path.exists(HeadRunFile): #check if a run file for the run we want to compare already exists, it probably won't but just in case we don't have to interrogate WBM
|
| 166 |
HeadParser = pickle.load( open( HeadRunFile ) )
|
| 167 |
else:
|
| 168 |
HeadParser = GetRun(CompareRunNum,HeadRunFile,SaveRun,FirstLS,LastLS)
|
| 169 |
if PrintLumi:
|
| 170 |
for LS in HeadParser.LumiInfo[0]:
|
| 171 |
try:
|
| 172 |
if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
|
| 173 |
continue
|
| 174 |
print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
|
| 175 |
except:
|
| 176 |
print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
|
| 177 |
|
| 178 |
sys.exit(0)
|
| 179 |
|
| 180 |
if RefRunNum == 0:
|
| 181 |
RefRates = 0
|
| 182 |
RefLumis = 0
|
| 183 |
LastSuccessfulIterator = 0
|
| 184 |
|
| 185 |
### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
|
| 186 |
CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
|
| 187 |
|
| 188 |
try:
|
| 189 |
while True:
|
| 190 |
RunComparison(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
|
| 191 |
|
| 192 |
if FindL1Zeros:
|
| 193 |
CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
|
| 194 |
if int(Config.ShifterMode)==0:
|
| 195 |
print "Expert Mode. Quitting."
|
| 196 |
sys.exit(0)
|
| 197 |
else:
|
| 198 |
print "Shifter Mode. Continuing"
|
| 199 |
|
| 200 |
print "Sleeping for 1 minute before repeating "
|
| 201 |
for iSleep in range(6):
|
| 202 |
for iDot in range(iSleep+1):
|
| 203 |
print ".",
|
| 204 |
print "."
|
| 205 |
time.sleep(10)
|
| 206 |
clear()
|
| 207 |
#end while True
|
| 208 |
#end try
|
| 209 |
except KeyboardInterrupt:
|
| 210 |
print "Quitting. Peace Out."
|
| 211 |
|
| 212 |
|
| 213 |
def RunComparison(HeadParser,RefRunNum,RefRates,RefLumis,Iterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
|
| 214 |
|
| 215 |
Header = ["Trigger Name","Actual","Expected","% Inc","Cur PS","Comments"]
|
| 216 |
Data = []
|
| 217 |
Warn = []
|
| 218 |
IgnoredRates=[]
|
| 219 |
|
| 220 |
LumiInfo = HeadParser.LumiInfo
|
| 221 |
TriggerRates = HeadParser.TriggerRates
|
| 222 |
TriggerInfo = HeadParser.TriggerInfo
|
| 223 |
PrescaleValues = HeadParser.PrescaleValues
|
| 224 |
TotalPSInfo = HeadParser.TotalPSInfo
|
| 225 |
CorrectedPSInfo = HeadParser.CorrectedPSInfo
|
| 226 |
|
| 227 |
[LumiSection,PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,AvInstLumi,AvDeliveredLumi,AvLiveLumi] = LumiInfo
|
| 228 |
[L1TriggerMode, HLTTriggerMode, HLTSeed] = TriggerInfo
|
| 229 |
[L1Prescale,HLTPrescale,MissingPrescale] = PrescaleValues
|
| 230 |
[L1_zero,HLT_zero,IdealPrescale,IdealHLTPrescale,n1,n2,L1,L2,H1,H2] = TotalPSInfo
|
| 231 |
[RealPrescale,xLS,L1,L2,H1,H2] = CorrectedPSInfo
|
| 232 |
|
| 233 |
|
| 234 |
for key in TriggerRates:
|
| 235 |
## SKIP triggers in the skip list
|
| 236 |
skipTrig=False
|
| 237 |
##for trig in ExcludeList:
|
| 238 |
##if not headTrigN.find(trig) == -1:
|
| 239 |
##skipTrig=True
|
| 240 |
##break
|
| 241 |
##if skipTrig:
|
| 242 |
##IgnoredRates.append([headTrigN,headTrigRate]) # get the current rates of the ignored paths
|
| 243 |
##continue
|
| 244 |
|
| 245 |
if L1_zero[key] == True or HLT_zero[key] == True:
|
| 246 |
continue
|
| 247 |
|
| 248 |
[TriggerRate,L1Pass,PSPass,RealHLTPrescale,Seed,StartLS,EndLS] = TriggerRates[key]
|
| 249 |
|
| 250 |
OverallPrescale = RealPrescale[key]
|
| 251 |
|
| 252 |
PSCorrectedRate = TriggerRate * OverallPrescale
|
| 253 |
#print str(key)+" made it here with TriggerRate = "+str(TriggerRate)+", PSCorrectedRate = "+str(PSCorrectedRate)
|
| 254 |
|
| 255 |
|
| 256 |
if RefRunNum == 0: ## Use rate prediction functions
|
| 257 |
|
| 258 |
PSCorrectedExpectedRate = Config.GetExpectedRate(key,AvLiveLumi)
|
| 259 |
|
| 260 |
|
| 261 |
if PSCorrectedExpectedRate < 0: ##This means we don't have a prediction for this trigger
|
| 262 |
continue
|
| 263 |
ExpectedRate = round((PSCorrectedExpectedRate / OverallPrescale),2)
|
| 264 |
PerDiff=0
|
| 265 |
if ExpectedRate>0:
|
| 266 |
PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
|
| 267 |
if abs(PerDiff) > AllowedRateDiff/max(sqrt(TriggerRate),sqrt(ExpectedRate)):
|
| 268 |
Warn.append(True)
|
| 269 |
else:
|
| 270 |
Warn.append(False)
|
| 271 |
|
| 272 |
if TriggerRate < IgnoreThreshold and ExpectedRate < IgnoreThreshold:
|
| 273 |
continue
|
| 274 |
|
| 275 |
VC = ""
|
| 276 |
|
| 277 |
Data.append([key,TriggerRate,ExpectedRate,PerDiff,OverallPrescale,VC])
|
| 278 |
continue
|
| 279 |
|
| 280 |
else: ## Use a reference run
|
| 281 |
## cheap trick to only get triggers in list when in shifter mode
|
| 282 |
print "shifter mode=",int(Config.ShifterMode)
|
| 283 |
if int(Config.ShifterMode)==1:
|
| 284 |
PSCorrectedExpectedRate = Config.GetExpectedRate(key,AvLiveLumi)
|
| 285 |
if PSCorrectedExpectedRate < 0: ##This means we don't have a prediction for this trigger
|
| 286 |
continue
|
| 287 |
|
| 288 |
if not L1Prescale[HLTSeed[key]][StartLS] == 1 or not L1Prescale[HLTSeed[key]][EndLS] == 1:
|
| 289 |
continue
|
| 290 |
|
| 291 |
RefInstLumi = 0
|
| 292 |
RefIterator = 0
|
| 293 |
|
| 294 |
for Iterator2 in range(1,Iterator+1):
|
| 295 |
if abs(RefLumis[2][Iterator2*10] - AvInstLumi) < abs(RefInstLumi - AvInstLumi):
|
| 296 |
RefInstLumi = RefLumis[2][Iterator2*10]
|
| 297 |
RefIterator = int(Iterator2)
|
| 298 |
|
| 299 |
RefRate = RefRates[RefIterator]
|
| 300 |
RefAvDeliveredLumi = (RefLumis[3][(RefIterator+1)*10] - RefLumis[3][RefIterator*10])*1000/(23.3*10)
|
| 301 |
RefAvLiveLumi = (RefLumis[4][(RefIterator+1)*10] - RefLumis[4][RefIterator*10])*1000/(23.3*10)
|
| 302 |
|
| 303 |
try:
|
| 304 |
ScaledRefRate = round( ((RefRate[key][0]*RefRate[key][3])/OverallPrescale)*(AvLiveLumi/RefAvLiveLumi), 2)
|
| 305 |
except:
|
| 306 |
print "Maybe trigger "+str(key)+" does not exist in the reference run?"
|
| 307 |
|
| 308 |
if ScaledRefRate == 0:
|
| 309 |
PerDiff = 100
|
| 310 |
else:
|
| 311 |
PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
|
| 312 |
|
| 313 |
if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
|
| 314 |
continue
|
| 315 |
|
| 316 |
if abs(PerDiff) > AllowedRateDiff/max(sqrt(TriggerRate),sqrt(ScaledRefRate)):
|
| 317 |
Warn.append(True)
|
| 318 |
else:
|
| 319 |
Warn.append(False)
|
| 320 |
VC = ""
|
| 321 |
Data.append([key,TriggerRate,ScaledRefRate,PerDiff,OverallPrescale,VC])
|
| 322 |
continue
|
| 323 |
|
| 324 |
|
| 325 |
PrettyPrintTable(Header,Data,[80,10,10,10,10,20],Warn)
|
| 326 |
|
| 327 |
MoreTableInfo(PSColumnByLS,LiveLumiByLS,DeliveredLumiByLS,StartLS,EndLS)
|
| 328 |
|
| 329 |
def CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
|
| 330 |
print "checking trigger list"
|
| 331 |
|
| 332 |
def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
|
| 333 |
L1Zeros=[]
|
| 334 |
IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
|
| 335 |
for key in HeadParser.TriggerRates:
|
| 336 |
## Skip events in the skip list
|
| 337 |
skipTrig=False
|
| 338 |
##for trig in Config.ExcludeList:
|
| 339 |
##if not trigN.find(trig) == -1:
|
| 340 |
##skipTrig=True
|
| 341 |
##break
|
| 342 |
if skipTrig:
|
| 343 |
continue
|
| 344 |
## if no events pass the L1, add it to the L1Zeros list if not already there
|
| 345 |
if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
|
| 346 |
if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
|
| 347 |
|
| 348 |
L1Zeros.append(HeadParser.TriggerRates[key][4])
|
| 349 |
print "L1Zeros=", L1Zeros
|
| 350 |
|
| 351 |
if len(L1Zeros) == 0:
|
| 352 |
#print "It looks like no masked L1 bits seed trigger paths"
|
| 353 |
pass
|
| 354 |
else:
|
| 355 |
print "The following seeds are used to seed HLT bits but accept 0 events:"
|
| 356 |
#print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
|
| 357 |
for Seed in L1Zeros:
|
| 358 |
print Seed
|
| 359 |
|
| 360 |
if __name__=='__main__':
|
| 361 |
main()
|
