| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
from ReadConfig import RateMonConfig
|
| 4 |
import sys
|
| 5 |
import os
|
| 6 |
import cPickle as pickle
|
| 7 |
import getopt
|
| 8 |
import time
|
| 9 |
from StreamMonitor import *
|
| 10 |
from colors import *
|
| 11 |
try:
|
| 12 |
from TablePrint import *
|
| 13 |
except ImportError:
|
| 14 |
sys.stderr.write("Exception of environment variables. try:\nsource set.sh\n")
|
| 15 |
sys.exit(2)
|
| 16 |
|
| 17 |
from AddTableInfo_db import MoreTableInfo
|
| 18 |
from math import *
|
| 19 |
from DatabaseParser import *
|
| 20 |
from TablePrint import *
|
| 21 |
|
| 22 |
WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
|
| 23 |
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
|
| 24 |
RefRunNameTemplate = "RefRuns/%s/Run_%s.pk"
|
| 25 |
|
| 26 |
# define a function that clears the terminal screen
|
| 27 |
def clear():
|
| 28 |
print("\x1B[2J")
|
| 29 |
|
| 30 |
|
| 31 |
def usage():
|
| 32 |
print sys.argv[0]+" [Options]"
|
| 33 |
print "This script gets the current HLT trigger rates and compares them to a reference run or a fit to multiple runs"
|
| 34 |
print "Options: "
|
| 35 |
print "--AllowedPercDiff=<diff> Warn only if difference in trigger rate is greater than <diff>%"
|
| 36 |
print "--AllowedSigmaDiff=<diff> Warn only if difference in trigger rate is greater than <diff> standard deviations"
|
| 37 |
print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
|
| 38 |
print "--FindL1Zeros Look for physics paths with 0 L1 rate"
|
| 39 |
print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
|
| 40 |
print "--NumberLS=<#> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
|
| 41 |
print " or set LSSlidingWindow = -1"
|
| 42 |
print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate>"
|
| 43 |
print "--AllTriggers Prints the paths that are not compared by this script and their rate in the CompareRun"
|
| 44 |
print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
|
| 45 |
print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
|
| 46 |
print "--ShowPSTriggers Show prescaled triggers in rate comparison"
|
| 47 |
print "--sortBy=<field> Sort the triggers by field. Valid fields are: name, rate, rateDiff"
|
| 48 |
print "--force Override the check for collisions run"
|
| 49 |
print "--write Writes rates to .csv file"
|
| 50 |
print "--ShowAllBadRates Show a list of all triggers (not just those in the monitor list) with bad rates"
|
| 51 |
print "--help Print this help"
|
| 52 |
|
| 53 |
def pickYear():
|
| 54 |
global thisyear
|
| 55 |
thisyear="2012"
|
| 56 |
##print "Year set to ",thisyear
|
| 57 |
|
| 58 |
def main():
|
| 59 |
pickYear()
|
| 60 |
try:
|
| 61 |
opt, args = getopt.getopt(sys.argv[1:],"",["AllowedPercDiff=","AllowedSigmaDiff=","CompareRun=","FindL1Zeros",\
|
| 62 |
"FirstLS=","NumberLS=","IgnoreLowRate=","AllTriggers",\
|
| 63 |
"PrintLumi","RefRun=","ShowPSTriggers","force","sortBy=","write","ShowAllBadRates","help"])
|
| 64 |
except getopt.GetoptError, err:
|
| 65 |
print str(err)
|
| 66 |
usage()
|
| 67 |
sys.exit(2)
|
| 68 |
|
| 69 |
Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
|
| 70 |
for o,a in opt:
|
| 71 |
if o=="--ConfigFile":
|
| 72 |
Config.CFGfile=a
|
| 73 |
Config.ReadCFG()
|
| 74 |
|
| 75 |
|
| 76 |
if "NoV" in Config.FitFileName:
|
| 77 |
Config.NoVersion=True
|
| 78 |
#print "NoVersion=",Config.NoVersion
|
| 79 |
|
| 80 |
ShowSigmaAndPercDiff = Config.DefShowSigmaAndPercDiff
|
| 81 |
WarnOnSigmaDiff = Config.DefWarnOnSigmaDiff
|
| 82 |
AllowedRatePercDiff = Config.DefAllowRatePercDiff
|
| 83 |
AllowedRateSigmaDiff = Config.DefAllowRateSigmaDiff
|
| 84 |
CompareRunNum = ""
|
| 85 |
FindL1Zeros = False
|
| 86 |
FirstLS = 9999
|
| 87 |
NumLS = -10
|
| 88 |
IgnoreThreshold = Config.DefAllowIgnoreThresh
|
| 89 |
AllTriggers = Config.AllTriggers
|
| 90 |
PrintLumi = False
|
| 91 |
RefRunNum = int(Config.ReferenceRun)
|
| 92 |
ShowPSTriggers = True
|
| 93 |
Force = False
|
| 94 |
writeb = False
|
| 95 |
SortBy = "rate"
|
| 96 |
ShifterMode = int(Config.ShifterMode) # get this from the config, but can be overridden by other options
|
| 97 |
ShowAllBadRates = Config.ShowAllBadRates
|
| 98 |
MaxBadRates = Config.DefaultMaxBadRatesToShow
|
| 99 |
|
| 100 |
if Config.LSWindow > 0:
|
| 101 |
NumLS = -1*Config.LSWindow
|
| 102 |
|
| 103 |
for o,a in opt: # get options passed on the command line
|
| 104 |
if o=="--AllowedPercDiff":
|
| 105 |
AllowedRatePercDiff = float(a)
|
| 106 |
elif o=="--AllowedSigmaDiff":
|
| 107 |
AllowedRateSigmaDiff = float(a)
|
| 108 |
elif o=="--CompareRun":
|
| 109 |
CompareRunNum=int(a)
|
| 110 |
ShifterMode = False
|
| 111 |
elif o=="--FindL1Zeros":
|
| 112 |
FindL1Zeros = True
|
| 113 |
elif o=="--FirstLS":
|
| 114 |
FirstLS = int(a)
|
| 115 |
ShifterMode = False
|
| 116 |
elif o=="--NumberLS":
|
| 117 |
NumLS = int(a)
|
| 118 |
elif o=="--IgnoreLowRate":
|
| 119 |
IgnoreThreshold = float(a)
|
| 120 |
elif o=="--AllTriggers":
|
| 121 |
AllTriggers=True
|
| 122 |
ShifterMode = False
|
| 123 |
elif o=="--PrintLumi":
|
| 124 |
PrintLumi = True
|
| 125 |
elif o=="--RefRun":
|
| 126 |
RefRunNum=int(a)
|
| 127 |
elif o=="--ShowPSTriggers":
|
| 128 |
ShowPSTriggers=True
|
| 129 |
elif o=="--sortBy":
|
| 130 |
SortBy = a
|
| 131 |
elif o=="--force":
|
| 132 |
Force = True
|
| 133 |
elif o=="--write":
|
| 134 |
writeb = True
|
| 135 |
elif o=="--ShowAllBadRates":
|
| 136 |
ShowAllBadRates=True
|
| 137 |
elif o=="--help":
|
| 138 |
usage()
|
| 139 |
sys.exit(0)
|
| 140 |
else:
|
| 141 |
print "Invalid Option "+a
|
| 142 |
sys.exit(1)
|
| 143 |
|
| 144 |
|
| 145 |
RefLumisExists = False
|
| 146 |
"""
|
| 147 |
RefRunFile=RefRunNameTemplate % str(RefRunNum)
|
| 148 |
if RefRunNum > 0:
|
| 149 |
RefRates = {}
|
| 150 |
for Iterator in range(1,100):
|
| 151 |
if RefLumisExists: ## Quits at the end of a run
|
| 152 |
if max(RefLumis[0]) <= (Iterator+1)*10:
|
| 153 |
break
|
| 154 |
|
| 155 |
RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
|
| 156 |
print "RefRunFile=",RefRunFile
|
| 157 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 158 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 159 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 160 |
sys.exit(0)
|
| 161 |
|
| 162 |
if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
|
| 163 |
print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
|
| 164 |
print "Creating ..."
|
| 165 |
try:
|
| 166 |
RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
|
| 167 |
print "parsing"
|
| 168 |
except:
|
| 169 |
print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 170 |
continue
|
| 171 |
|
| 172 |
else: # otherwise load it from the file
|
| 173 |
RefParser = pickle.load( open( RefRunFile ) )
|
| 174 |
print "loading"
|
| 175 |
if not RefLumisExists:
|
| 176 |
RefLumis = RefParser.LumiInfo
|
| 177 |
RefLumisExists = True
|
| 178 |
|
| 179 |
try:
|
| 180 |
RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
|
| 181 |
LastSuccessfulIterator = Iterator
|
| 182 |
except:
|
| 183 |
print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
|
| 184 |
|
| 185 |
"""
|
| 186 |
RefRunFile = RefRunNameTemplate % (thisyear,RefRunNum)
|
| 187 |
RefParser = DatabaseParser()
|
| 188 |
##print "Reference Run: "+str(RefRunNum)
|
| 189 |
if RefRunNum > 0:
|
| 190 |
print "Getting RefRunFile",RefRunFile
|
| 191 |
if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
|
| 192 |
print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
|
| 193 |
print RefRunFile[:RefRunFile.rfind('/')]
|
| 194 |
sys.exit(0)
|
| 195 |
s
|
| 196 |
return
|
| 197 |
if not os.path.exists(RefRunFile):
|
| 198 |
print "RefRunFile does not exist"
|
| 199 |
# create the reference run file
|
| 200 |
try:
|
| 201 |
RefParser.RunNumber = RefRunNum
|
| 202 |
RefParser.ParseRunSetup()
|
| 203 |
print "RefParser is setup"
|
| 204 |
#RefParser.GetAllTriggerRatesByLS()
|
| 205 |
#RefParser.Save( RefRunFile )
|
| 206 |
except e:
|
| 207 |
print "PROBLEM GETTING REFERNCE RUN"
|
| 208 |
raise
|
| 209 |
else:
|
| 210 |
RefParser = pickle.load(open(RefRunFile))
|
| 211 |
|
| 212 |
# OK, Got the Reference Run
|
| 213 |
# Now get the most recent run
|
| 214 |
|
| 215 |
SaveRun = False
|
| 216 |
if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
|
| 217 |
CompareRunNum,isCol,isGood = GetLatestRunNumber()
|
| 218 |
|
| 219 |
if not isGood:
|
| 220 |
print "NO TRIGGER KEY FOUND for run ",CompareRunNum
|
| 221 |
##sys.exit(0)
|
| 222 |
|
| 223 |
if not isCol:
|
| 224 |
print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
|
| 225 |
print "Monitoring only stream A and Express"
|
| 226 |
#if not Force:
|
| 227 |
# sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
|
| 228 |
|
| 229 |
else:
|
| 230 |
print "Most Recent run is "+str(CompareRunNum)
|
| 231 |
else:
|
| 232 |
CompareRunNum,isCol,isGood = GetLatestRunNumber(CompareRunNum)
|
| 233 |
if not isGood:
|
| 234 |
print "NO TRIGGER KEY FOUND for run ", CompareRunNum
|
| 235 |
##sys.exit(0)
|
| 236 |
|
| 237 |
HeadParser = DatabaseParser()
|
| 238 |
HeadParser.RunNumber = CompareRunNum
|
| 239 |
|
| 240 |
try:
|
| 241 |
HeadParser.ParseRunSetup()
|
| 242 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
| 243 |
LastGoodLS=HeadParser.GetLastLS(isCol)+1
|
| 244 |
tempLastGoodLS=LastGoodLS
|
| 245 |
CurrRun=CompareRunNum
|
| 246 |
|
| 247 |
except:
|
| 248 |
HeadLumiRange=[]
|
| 249 |
LastGoodLS=-1
|
| 250 |
tempLastGoodLS=LastGoodLS-1
|
| 251 |
CurrRun=CompareRunNum
|
| 252 |
isGood=0
|
| 253 |
|
| 254 |
if len(HeadLumiRange) is 0:
|
| 255 |
print "No lumisections that are taking physics data 0"
|
| 256 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
| 257 |
if len(HeadLumiRange)>0:
|
| 258 |
isGood=1
|
| 259 |
isCol=0
|
| 260 |
##sys.exit(0)
|
| 261 |
|
| 262 |
## This reduces the sensitivity for a rate measurement to cause a warning during the beginning of a run
|
| 263 |
if len(HeadLumiRange) > 0 and len(HeadLumiRange) < 10:
|
| 264 |
AllowedRateSigmaDiff = AllowedRateSigmaDiff*10 / len(HeadLumiRange)
|
| 265 |
|
| 266 |
if PrintLumi:
|
| 267 |
for LS in HeadParser.LumiInfo[0]:
|
| 268 |
try:
|
| 269 |
if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
|
| 270 |
continue
|
| 271 |
print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
|
| 272 |
except:
|
| 273 |
print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
|
| 274 |
|
| 275 |
sys.exit(0)
|
| 276 |
|
| 277 |
if RefRunNum == 0:
|
| 278 |
RefRates = 0
|
| 279 |
RefLumis = 0
|
| 280 |
LastSuccessfulIterator = 0
|
| 281 |
|
| 282 |
### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
|
| 283 |
###isGood=1##if there is a trigger key
|
| 284 |
try:
|
| 285 |
while True:
|
| 286 |
if isGood:
|
| 287 |
tempLastGoodLS=LastGoodLS
|
| 288 |
LastGoodLS=HeadParser.GetLastLS(isCol)
|
| 289 |
##print "Last Good=",LastGoodLS, tempLastGoodLS
|
| 290 |
if LastGoodLS==tempLastGoodLS:
|
| 291 |
write(bcolors.OKBLUE)
|
| 292 |
print "Trying to get new Run"
|
| 293 |
write(bcolors.ENDC+"\n")
|
| 294 |
else:
|
| 295 |
RefMoreLumiArray = HeadParser.GetMoreLumiInfo()
|
| 296 |
isBeams=True
|
| 297 |
for lumisection in HeadLumiRange:
|
| 298 |
try:
|
| 299 |
if not (RefMoreLumiArray["b1pres"][lumisection] and RefMoreLumiArray["b2pres"][lumisection] and RefMoreLumiArray["b1stab"][lumisection] and RefMoreLumiArray["b2stab"][lumisection]):
|
| 300 |
isBeams=False
|
| 301 |
except:
|
| 302 |
isBeams=False
|
| 303 |
|
| 304 |
if not (isCol and isBeams):
|
| 305 |
MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
|
| 306 |
else:
|
| 307 |
if (len(HeadLumiRange)>0):
|
| 308 |
if not isSequential(HeadLumiRange):
|
| 309 |
print "Some lumisections have been skipped. Averaging over most recent sequential lumisections..."
|
| 310 |
sequential_chunk = getSequential(HeadLumiRange)
|
| 311 |
HeadLumiRange = sequential_chunk
|
| 312 |
RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,AllTriggers,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb,ShowAllBadRates,MaxBadRates)
|
| 313 |
if FindL1Zeros:
|
| 314 |
CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config)
|
| 315 |
else:
|
| 316 |
print "No lumisections that are taking physics data 1"
|
| 317 |
|
| 318 |
if not ShifterMode:
|
| 319 |
print "Expert Mode. Quitting."
|
| 320 |
sys.exit(0)
|
| 321 |
|
| 322 |
print "Sleeping for 1 minute before repeating "
|
| 323 |
|
| 324 |
for iSleep in range(20):
|
| 325 |
write(".")
|
| 326 |
sys.stdout.flush()
|
| 327 |
time.sleep(3)
|
| 328 |
write(" Updating\n")
|
| 329 |
sys.stdout.flush()
|
| 330 |
|
| 331 |
##print "\nminLS=",min(HeadLumiRange),"Last LS=",HeadParser.GetLastLS(isCol),"run=",HeadParser.RunNumber
|
| 332 |
###Get a new run if DAQ stops
|
| 333 |
##print "\nLastGoodLS=",LastGoodLS
|
| 334 |
|
| 335 |
##### NEED PLACEHOLDER TO COMPARE CURRENT RUN TO LATEST RUN #####
|
| 336 |
|
| 337 |
NewRun,isCol,isGood = GetLatestRunNumber(9999999) ## update to the latest run and lumi range
|
| 338 |
|
| 339 |
try:
|
| 340 |
maxLumi=max(HeadLumiRange)
|
| 341 |
except:
|
| 342 |
maxLumi=0
|
| 343 |
|
| 344 |
##### THESE ARE CONDITIONS TO GET NEW RUN #####
|
| 345 |
if maxLumi>(LastGoodLS+1) or not isGood or NewRun!=CurrRun:
|
| 346 |
print "Trying to get new Run"
|
| 347 |
try:
|
| 348 |
HeadParser = DatabaseParser()
|
| 349 |
HeadParser.RunNumber = NewRun
|
| 350 |
HeadParser.ParseRunSetup()
|
| 351 |
CurrRun,isCol,isGood=GetLatestRunNumber(9999999)
|
| 352 |
FirstLS=9999
|
| 353 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
| 354 |
if len(HeadLumiRange) is 0:
|
| 355 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
| 356 |
print "No lumisections that are taking physics data 2"
|
| 357 |
if len(HeadLumiRange)>0:
|
| 358 |
isGood=1
|
| 359 |
isCol=0
|
| 360 |
|
| 361 |
#tempLastGoodLS=LastGoodLS
|
| 362 |
#LastGoodLS=HeadParser.GetLastLS(isCol)
|
| 363 |
##print CurrRun, isCol, isGood
|
| 364 |
except:
|
| 365 |
isGood=0
|
| 366 |
isCol=0
|
| 367 |
print "failed"
|
| 368 |
|
| 369 |
else:
|
| 370 |
try:
|
| 371 |
HeadParser.ParseRunSetup()
|
| 372 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
|
| 373 |
if len(HeadLumiRange) is 0:
|
| 374 |
HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,False)
|
| 375 |
print "No lumisections that are taking physics data"
|
| 376 |
if len(HeadLumiRange)>0:
|
| 377 |
isGood=1
|
| 378 |
isCol=0
|
| 379 |
#LastGoodLS=HeadParser.GetLastLS(isCol)
|
| 380 |
except:
|
| 381 |
isGood=0
|
| 382 |
isCol=0
|
| 383 |
clear()
|
| 384 |
print "NO TRIGGER KEY FOUND YET for run", NewRun ,"repeating search"
|
| 385 |
|
| 386 |
|
| 387 |
except KeyboardInterrupt:
|
| 388 |
print "Quitting. Peace Out."
|
| 389 |
|
| 390 |
|
| 391 |
def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRatePercDiff,AllowedRateSigmaDiff,IgnoreThreshold,Config,AllTriggers,SortBy,WarnOnSigmaDiff,ShowSigmaAndPercDiff,writeb,ShowAllBadRates,MaxBadRates):
|
| 392 |
Data = []
|
| 393 |
Warn = []
|
| 394 |
IgnoredRates=[]
|
| 395 |
|
| 396 |
[HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
|
| 397 |
##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
|
| 398 |
HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
|
| 399 |
if Config.DoL1:
|
| 400 |
L1RatesALL=HeadParser.GetL1RatesALL(HeadLumiRange)
|
| 401 |
for L1seed in L1RatesALL.iterkeys():
|
| 402 |
HeadUnprescaledRates[L1seed]=L1RatesALL[L1seed]
|
| 403 |
|
| 404 |
[PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo
|
| 405 |
deadtimebeamactive=HeadParser.GetDeadTimeBeamActive(HeadLumiRange)
|
| 406 |
try:
|
| 407 |
pkl_file = open(Config.FitFileName, 'rb')
|
| 408 |
FitInput = pickle.load(pkl_file)
|
| 409 |
pkl_file.close()
|
| 410 |
##print "fit file name=",Config.FitFileName
|
| 411 |
|
| 412 |
except:
|
| 413 |
print "No fit file specified"
|
| 414 |
sys.exit(2)
|
| 415 |
|
| 416 |
###fitfile by L1seedchange
|
| 417 |
if Config.L1SeedChangeFit:
|
| 418 |
try:
|
| 419 |
PSfitfile=Config.FitFileName.replace("HLT_NoV","HLT_NoV_ByPS")
|
| 420 |
#print "Opening", PSfitfile
|
| 421 |
pkl_filePS = open(PSfitfile, 'rb')
|
| 422 |
FitInputPS = pickle.load(pkl_filePS)
|
| 423 |
pkl_filePS.close()
|
| 424 |
#print "fit file name=",Config.FitFileName
|
| 425 |
|
| 426 |
except:
|
| 427 |
print "No fit file by L1seed change specified. Have you run DatabaseRatePredictor with DoL1 in defaults.cfg set to true?"
|
| 428 |
sys.exit(2)
|
| 429 |
else:
|
| 430 |
FitInputPS={} ##define empty dict when not in use
|
| 431 |
try:
|
| 432 |
refrunfile="RefRuns/%s/Rates_HLT_10LS_JPAP.pkl" % (thisyear)
|
| 433 |
pkl_file = open(refrunfile, 'rb')
|
| 434 |
RefRatesInput = pickle.load(pkl_file)
|
| 435 |
pkl_file.close()
|
| 436 |
except:
|
| 437 |
RefRatesInput={}
|
| 438 |
#print "Didn't open ref file"
|
| 439 |
|
| 440 |
|
| 441 |
trig_list=Config.MonitorList
|
| 442 |
|
| 443 |
if Config.NoVersion:
|
| 444 |
trig_list=[]
|
| 445 |
|
| 446 |
for trigger in Config.MonitorList:
|
| 447 |
trig_list.append(StripVersion(trigger))
|
| 448 |
|
| 449 |
L1HLTseeds = HeadParser.GetL1HLTseeds()
|
| 450 |
if Config.DoL1:
|
| 451 |
for HLTkey in trig_list:
|
| 452 |
if "L1" in HLTkey:
|
| 453 |
continue
|
| 454 |
else:
|
| 455 |
try:
|
| 456 |
for L1seed in L1HLTseeds[HLTkey]:
|
| 457 |
if L1seed not in trig_list:
|
| 458 |
trig_list.append(L1seed)
|
| 459 |
except:
|
| 460 |
pass
|
| 461 |
|
| 462 |
for trigger in FitInput.iterkeys():
|
| 463 |
FitInput[StripVersion(trigger)] = FitInput.pop(trigger)
|
| 464 |
for trigger in HeadUnprescaledRates:
|
| 465 |
HeadUnprescaledRates[StripVersion(trigger)] = HeadUnprescaledRates.pop(trigger)
|
| 466 |
|
| 467 |
RefAvInstLumi = 0
|
| 468 |
found_ref_rates = True
|
| 469 |
for HeadName in HeadUnprescaledRates:
|
| 470 |
if HeadName not in trig_list and not AllTriggers and not ShowAllBadRates:
|
| 471 |
continue
|
| 472 |
if HeadName not in FitInput.keys() and not AllTriggers and not ShowAllBadRates:
|
| 473 |
continue
|
| 474 |
|
| 475 |
masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_Zero","HLT_BeamHalo"]
|
| 476 |
masked_trig = False
|
| 477 |
for mask in masked_triggers:
|
| 478 |
if str(mask) in HeadName:
|
| 479 |
masked_trig = True
|
| 480 |
if masked_trig:
|
| 481 |
continue
|
| 482 |
|
| 483 |
skipTrig=False
|
| 484 |
TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)
|
| 485 |
|
| 486 |
if RefParser.RunNumber == 0: ## Use rate prediction functions
|
| 487 |
PSCorrectedExpectedRate = Config.GetExpectedRate(HeadName,FitInput,FitInputPS,HeadAvLiveLumi,HeadAvDeliveredLumi,deadtimebeamactive,Config.L1SeedChangeFit,HeadLumiRange,PSColumnByLS)
|
| 488 |
VC = PSCorrectedExpectedRate[2]
|
| 489 |
try:
|
| 490 |
sigma = PSCorrectedExpectedRate[1]*sqrt(PSCorrectedExpectedRate[0])/(sqrt(len(HeadLumiRange))* HeadUnprescaledRates[HeadName][1])
|
| 491 |
ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
|
| 492 |
except:
|
| 493 |
sigma = 0.0
|
| 494 |
ExpectedRate = 0.0 ##This means we don't have a prediction for this trigger-- gets overwritten to "--" later
|
| 495 |
PerDiff = 0.0
|
| 496 |
SigmaDiff = 0.0
|
| 497 |
if HeadUnprescaledRates[HeadName][1] != 0:
|
| 498 |
VC="No prediction"
|
| 499 |
else:
|
| 500 |
VC="Path prescaled to 0"
|
| 501 |
|
| 502 |
if ExpectedRate > 0:
|
| 503 |
PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
|
| 504 |
else:
|
| 505 |
PerDiff = 0.0
|
| 506 |
|
| 507 |
if sigma > 0:
|
| 508 |
SigmaDiff = round( (TriggerRate - ExpectedRate)/sigma, 2)
|
| 509 |
else:
|
| 510 |
SigmaDiff = 0.0 #Zero sigma means that when there were no rates for this trigger when the fit was made
|
| 511 |
|
| 512 |
if TriggerRate < IgnoreThreshold and (ExpectedRate < IgnoreThreshold and ExpectedRate!=0):
|
| 513 |
continue
|
| 514 |
|
| 515 |
|
| 516 |
Data.append([HeadName, TriggerRate, ExpectedRate, PerDiff, SigmaDiff, round(HeadUnprescaledRates[HeadName][1],0),VC])
|
| 517 |
|
| 518 |
else: ## Use a reference run
|
| 519 |
RefInstLumi = 0
|
| 520 |
RefIterator = 0
|
| 521 |
RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
|
| 522 |
RefLen = -10
|
| 523 |
RefLSRange = RefParser.GetLSRange(RefStartIndex,RefLen)
|
| 524 |
|
| 525 |
RefUnprescaledRates = RefParser.UpdateRun(RefLSRange)
|
| 526 |
|
| 527 |
[RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefLSRange)
|
| 528 |
if Config.DoL1 and RefUnprescaledRates != {}:
|
| 529 |
RefL1RatesALL = RefParser.GetL1RatesALL(RefLSRange)
|
| 530 |
for L1seed in RefL1RatesALL.iterkeys():
|
| 531 |
RefUnprescaledRates[L1seed]=RefL1RatesALL[L1seed]
|
| 532 |
|
| 533 |
if RefUnprescaledRates == {}:
|
| 534 |
found_ref_rates = False
|
| 535 |
for path in HeadUnprescaledRates.iterkeys():
|
| 536 |
RefUnprescaledRates[path] = [0.0,0.0,0.0,0.0]
|
| 537 |
|
| 538 |
RefRate = -1
|
| 539 |
for k,v in RefUnprescaledRates.iteritems():
|
| 540 |
if HeadName == StripVersion(k):
|
| 541 |
RefRate = RefUnprescaledRates[k][2]
|
| 542 |
|
| 543 |
try:
|
| 544 |
ScaledRefRate = round( (RefRate*HeadAvLiveLumi/RefAvLiveLumi*(1-deadtimebeamactive)), 2 )
|
| 545 |
|
| 546 |
except ZeroDivisionError:
|
| 547 |
ScaledRefRate=0
|
| 548 |
|
| 549 |
SigmaDiff = 0
|
| 550 |
if ScaledRefRate == 0:
|
| 551 |
PerDiff = -999
|
| 552 |
else:
|
| 553 |
PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
|
| 554 |
|
| 555 |
if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
|
| 556 |
continue
|
| 557 |
|
| 558 |
VC = ""
|
| 559 |
Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,SigmaDiff,round((HeadUnprescaledRates[HeadName][1]),0),VC])
|
| 560 |
|
| 561 |
if not found_ref_rates:
|
| 562 |
print '\n*****************************************************************************************************************************************************'
|
| 563 |
print 'COULD NOT PARSE REFERENCE RUN! MOST LIKELY THIS IS BECAUSE THE REFERENCE RUN DOES NOT PASS THE QUALITY CUTS (DEADTIME < 100%, PHYSICS DECALRED, ETC.)'
|
| 564 |
print 'Setting all reference rates to zero...'
|
| 565 |
print '*****************************************************************************************************************************************************'
|
| 566 |
|
| 567 |
SortedData = []
|
| 568 |
if SortBy == "":
|
| 569 |
SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
|
| 570 |
elif SortBy == "name":
|
| 571 |
SortedData=sorted(Data, key=lambda entry: entry[0])
|
| 572 |
elif SortBy == "rate":
|
| 573 |
SortedData=sorted(Data, key=lambda entry: entry[1],reverse=True)
|
| 574 |
elif SortBy == "ratePercDiff":
|
| 575 |
SortedData=sorted(Data, key=lambda entry: abs(entry[3]),reverse=True)
|
| 576 |
elif SortBy == "rateSigmaDiff":
|
| 577 |
SortedData=sorted(Data, key=lambda entry: abs(entry[4]),reverse=True)
|
| 578 |
else:
|
| 579 |
print "Invalid sorting option %s\n"%SortBy
|
| 580 |
SortedData = Data
|
| 581 |
|
| 582 |
#check for triggers above the warning threshold
|
| 583 |
Warn=[]
|
| 584 |
core_data=[]
|
| 585 |
core_l1_seeds=[]
|
| 586 |
nBadRates = 0
|
| 587 |
#Loop for HLT triggers
|
| 588 |
for entry in SortedData:
|
| 589 |
bad_seeds_string = ""
|
| 590 |
if not entry[0].startswith('HLT'):
|
| 591 |
continue
|
| 592 |
bad_rate = (abs(entry[4]) > AllowedRateSigmaDiff and WarnOnSigmaDiff) or (abs(entry[3]) > AllowedRatePercDiff and not WarnOnSigmaDiff) or (abs(entry[3]) > AllowedRatePercDiff and RefParser.RunNumber > 0)
|
| 593 |
if entry[0] in trig_list or AllTriggers or (bad_rate and ShowAllBadRates and nBadRates < MaxBadRates):
|
| 594 |
core_data.append(entry)
|
| 595 |
if bad_rate or (bad_rate and ShowAllBadRates and nBadRates < MaxBadRates):
|
| 596 |
if Config.DoL1:
|
| 597 |
for seed in L1HLTseeds[entry[0]]:
|
| 598 |
if not seed in core_l1_seeds:
|
| 599 |
core_l1_seeds.append(seed)
|
| 600 |
seed_rate = [line[3] for line in SortedData if line[0] == seed]
|
| 601 |
if seed_rate:
|
| 602 |
bad_seed_rate = (abs(seed_rate[0]) > AllowedRatePercDiff)
|
| 603 |
if bad_seed_rate:
|
| 604 |
if bad_seeds_string != "":
|
| 605 |
bad_seeds_string += ", "
|
| 606 |
bad_seeds_string += seed
|
| 607 |
entry[6] = bad_seeds_string
|
| 608 |
Warn.append(True)
|
| 609 |
nBadRates += 1
|
| 610 |
else:
|
| 611 |
Warn.append(False)
|
| 612 |
|
| 613 |
##Loop for L1 seeds of HLT triggers with warnings
|
| 614 |
if Config.DoL1:
|
| 615 |
for entry in SortedData:
|
| 616 |
if not entry[0] in core_l1_seeds:
|
| 617 |
continue
|
| 618 |
core_data.append(entry)
|
| 619 |
bad_seed_rate = (abs(entry[3]) > AllowedRatePercDiff)
|
| 620 |
if bad_seed_rate:
|
| 621 |
Warn.append(True) #Currently, number of bad rates to show refers to bad HLT triggers (no limit on the number of bad L1 seeds to show)
|
| 622 |
else:
|
| 623 |
Warn.append(False)
|
| 624 |
|
| 625 |
for index,entry in enumerate(core_data):
|
| 626 |
if entry[6] == "No prediction (fit missing)": #Dont show 0s if we don't actually have a prediction; it's confusing
|
| 627 |
core_data[index] = [entry[0],entry[1],"--","--","--",entry[5],entry[6]]
|
| 628 |
if entry[0].startswith('L1'): #Don't show sigma value for L1 since we don't trust them
|
| 629 |
core_data[index] = [entry[0],entry[1],entry[2],entry[3],"--",entry[5],entry[6]]
|
| 630 |
|
| 631 |
try:
|
| 632 |
comment_width = max([30,max([len(col[6]) for col in core_data])+1])
|
| 633 |
except:
|
| 634 |
comment_width = 30
|
| 635 |
|
| 636 |
if RefParser.RunNumber > 0:
|
| 637 |
Header = ["Trigger Name", "Actual", "Ref Run", "% Diff", "Cur PS", "Comments"]
|
| 638 |
table_data = [[col[0], col[1], col[2], col[3], col[5], col[6]] for col in core_data]
|
| 639 |
PrettyPrintTable(Header,table_data,[80,10,10,10,10,comment_width],Warn)
|
| 640 |
elif ShowSigmaAndPercDiff == 1:
|
| 641 |
Header = ["Trigger Name", "Actual", "Expected","% Diff","Deviation", "Cur PS", "Comments"]
|
| 642 |
table_data=core_data
|
| 643 |
PrettyPrintTable(Header,table_data,[80,10,10,10,11,10,comment_width],Warn)
|
| 644 |
print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
|
| 645 |
elif WarnOnSigmaDiff == 1:
|
| 646 |
Header = ["Trigger Name", "Actual", "Expected","Deviation", "Cur PS", "Comments"]
|
| 647 |
table_data = [[col[0], col[1], col[2], col[4], col[5], col[6]] for col in core_data]
|
| 648 |
PrettyPrintTable(Header,table_data,[80,10,10,10,11,10,comment_width],Warn)
|
| 649 |
print 'Deviation is the difference between the actual and expected rates, in units of the expected standard deviation.'
|
| 650 |
else:
|
| 651 |
Header = ["Trigger Name", "Actual", "Expected", "% Diff", "Cur PS", "Comments"]
|
| 652 |
table_data = [[col[0], col[1], col[2], col[3], col[5], col[6]] for col in core_data]
|
| 653 |
PrettyPrintTable(Header,table_data,[80,10,10,10,10,comment_width],Warn)
|
| 654 |
|
| 655 |
if writeb:
|
| 656 |
prettyCSVwriter("rateMon_newmenu.csv",[80,10,10,10,10,20,comment_width],Header,core_data,Warn)
|
| 657 |
|
| 658 |
MoreTableInfo(HeadParser,HeadLumiRange,Config,True)
|
| 659 |
if RefParser.RunNumber > 0 and RefAvInstLumi > 0:
|
| 660 |
print "The average instantaneous lumi for the reference run is: %s e30\n" % (round(RefAvInstLumi,1))
|
| 661 |
|
| 662 |
if nBadRates == MaxBadRates:
|
| 663 |
write(bcolors.WARNING)
|
| 664 |
print "The number of paths with rates outside limits exceeds the maximum number to display; only the first %i with the highest rate are shown above." % (MaxBadRates)
|
| 665 |
write(bcolors.ENDC+"\n")
|
| 666 |
|
| 667 |
for warning in Warn:
|
| 668 |
if warning==True:
|
| 669 |
write(bcolors.WARNING)
|
| 670 |
print "If any trigger remains red for 5 minutes, Please consult the shift crew and if needed contact relevant experts"
|
| 671 |
print "More instructions at https://twiki.cern.ch/twiki/bin/view/CMS/TriggerShiftHLTGuide"
|
| 672 |
write(bcolors.ENDC+"\n")
|
| 673 |
break
|
| 674 |
|
| 675 |
|
| 676 |
def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedPercRateDiff,IgnoreThreshold,Config):
|
| 677 |
L1Zeros=[]
|
| 678 |
IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
|
| 679 |
for key in HeadParser.TriggerRates:
|
| 680 |
## Skip events in the skip list
|
| 681 |
skipTrig=False
|
| 682 |
##for trig in Config.ExcludeList:
|
| 683 |
##if not trigN.find(trig) == -1:
|
| 684 |
##skipTrig=True
|
| 685 |
##break
|
| 686 |
if skipTrig:
|
| 687 |
continue
|
| 688 |
## if no events pass the L1, add it to the L1Zeros list if not already there
|
| 689 |
if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
|
| 690 |
if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
|
| 691 |
|
| 692 |
L1Zeros.append(HeadParser.TriggerRates[key][4])
|
| 693 |
print "L1Zeros=", L1Zeros
|
| 694 |
|
| 695 |
if len(L1Zeros) == 0:
|
| 696 |
#print "It looks like no masked L1 bits seed trigger paths"
|
| 697 |
pass
|
| 698 |
else:
|
| 699 |
print "The following seeds are used to seed HLT bits but accept 0 events:"
|
| 700 |
#print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
|
| 701 |
for Seed in L1Zeros:
|
| 702 |
print Seed
|
| 703 |
|
| 704 |
def isSequential(t):
|
| 705 |
try:
|
| 706 |
if len(t)<2:
|
| 707 |
return True
|
| 708 |
except:
|
| 709 |
return True
|
| 710 |
for i,e in enumerate(t[1:]):
|
| 711 |
if not abs(e-t[i])==1:
|
| 712 |
return False
|
| 713 |
return True
|
| 714 |
|
| 715 |
def getSequential(range):
|
| 716 |
for i,j in zip(range[-2::-1],range[::-1]):
|
| 717 |
if j-i != 1:
|
| 718 |
range = range[range.index(j):]
|
| 719 |
break
|
| 720 |
return range
|
| 721 |
|
| 722 |
|
| 723 |
if __name__=='__main__':
|
| 724 |
global thisyear
|
| 725 |
main()
|
