UserCode/RateMonShiftTool_dev/RateMonitorShifter.py

#!/usr/bin/env python

#from AndrewGetRun import GetRun
from DatabaseParser import *
from ReadConfig import RateMonConfig
import sys
import os
import cPickle as pickle
import getopt
import time
from colors import *
from TablePrint import *
from AddTableInfo_db import MoreTableInfo
from math import *

WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"

RefRunNameTemplate = "RefRuns/Run_%s.pk"

# define a function that clears the terminal screen
def clear():
    print("\x1B[2J")


def usage():
    print sys.argv[0]+" [Options]"
    print "This script gets the current HLT trigger rates and compares them to a reference run"
    print "Options: "
    print "--AllowedDiff=<diff>                 Report only if difference in trigger rate is greater than <diff>%"
    print "--CompareRun=<Run #>                 Compare run <Run #> to the reference run (Default = Current Run)"
    print "--FindL1Zeros                        Look for physics paths with 0 L1 rate"    
    print "--FirstLS=<ls>                       Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
    print "--NumberLS=<#>                       Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
    print "                                        or set LSSlidingWindow = -1"  
    print "--IgnoreLowRate=<rate>               Ignore triggers with an actual and expected rate below <rate>"
    print "--ListIgnoredPaths                   Prints the paths that are not compared by this script and their rate in the CompareRun"
    print "--PrintLumi                          Prints Instantaneous, Delivered, and Live lumi by LS for the run"
    print "--RefRun=<Run #>                     Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
    print "--ShowPSTriggers                     Show prescaled triggers in rate comparison"
    print "--force                              Override the check for collisions run"
    print "--help                               Print this help"
    
def main():
    try:
        opt, args = getopt.getopt(sys.argv[1:],"",["AllowedDiff=","CompareRun=","FindL1Zeros",\
                                                   "FirstLS=","NumberLS=","IgnoreLowRate=","ListIgnoredPaths",\
                                                   "PrintLumi","RefRun=","ShowPSTriggers","force","help"])
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)

    Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
    for o,a in opt:
        if o=="--ConfigFile":
            Config.CFGfile=a
    Config.ReadCFG()

    AllowedRateDiff   = Config.DefAllowRateDiff
    CompareRunNum     = ""
    FindL1Zeros       = False
    FirstLS           = 9999
    NumLS             = -10
    IgnoreThreshold   = Config.DefAllowIgnoreThresh
    ListIgnoredPaths  = False
    PrintLumi         = False
    RefRunNum         = int(Config.ReferenceRun)
    ShowPSTriggers    = True
    Force             = False

    if int(Config.ShifterMode):
        print "ShifterMode!!"
    else:
        print "ExpertMode"

    if Config.LSWindow > 0:
        NumLS = -1*Config.LSWindow

    for o,a in opt: # get options passed on the command line
        if o=="--AllowedDiff":
            AllowedRateDiff = float(a)/100.0
        elif o=="--CompareRun":
            CompareRunNum=int(a)
        elif o=="--FindL1Zeros":
            FindL1Zeros = True
        elif o=="--FirstLS":
            FirstLS = int(a)
        elif o=="--NumberLS":
            NumLS = int(a)
        elif o=="--IgnoreLowRate":
            IgnoreThreshold = float(a)
        elif o=="--ListIgnoredPaths":
            ListIgnoredPaths=True
        elif o=="--PrintLumi":
            PrintLumi = True
        elif o=="--RefRun":
            RefRunNum=int(a)
        elif o=="--ShowPSTriggers":
            ShowPSTriggers=True
        elif o=="--force":
            Force = True
        elif o=="--help":
            usage()
            sys.exit(0)
        else:
            print "Invalid Option "+a
            sys.exit(1)

    
    RefLumisExists = False

    """
    if RefRunNum > 0:
        RefRates = {}
        for Iterator in range(1,100):
            if RefLumisExists:  ## Quits at the end of a run
                if max(RefLumis[0]) <= (Iterator+1)*10:
                    break

                    RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator )  # place to save the reference run info
            print "RefRunFile=",RefRunFile
            if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]):  # folder for ref run file must exist
                print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
                print RefRunFile[:RefRunFile.rfind('/')]
                sys.exit(0)

            if not os.path.exists(RefRunFile):  # if the reference run is not saved, get it from wbm
                print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
                print "Creating ..."
                try:
                    RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator*10, (Iterator+1)*10)
                    print "parsing"
                except:
                    print "GetRun failed from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
                    continue
                    
            else: # otherwise load it from the file
                RefParser = pickle.load( open( RefRunFile ) )
                print "loading"
            if not RefLumisExists:
                RefLumis = RefParser.LumiInfo
                RefLumisExists = True

            try:
                RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
                LastSuccessfulIterator = Iterator
            except:
                print "Failed to get rates from LS "+str(Iterator*10)+" to "+str((Iterator+1)*10)
    """

    RefRunFile = RefRunNameTemplate % RefRunNum
    RefParser = DatabaseParser()
    print "Reference Run: "+str(RefRunNum)
    if RefRunNum > 0:
        if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]):  # folder for ref run file must exist
            print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
            print RefRunFile[:RefRunFile.rfind('/')]
            sys.exit(0)   
            return
        if not os.path.exists(RefRunFile):
            # create the reference run file
            try:
                RefParser.RunNumber = RefRunNum
                RefParser.ParseRunSetup()
                #RefParser.GetAllTriggerRatesByLS()
                #RefParser.Save( RefRunFile )
            except e:
                print "PROBLEM GETTING REFERNCE RUN"
                raise  
        else:
            RefParser = pickle.load( open( RefRunFile ) )
            
    # OK, Got the Reference Run
    # Now get the most recent run

    SaveRun = False
    if CompareRunNum=="":  # if no run # specified on the CL, get the most recent run
        CompareRunNum,isCol = GetLatestRunNumber()

        if not isCol:
            print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
            print "Monitoring only stream A and Express"
            #if not Force:
            #    sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
        print "Most Recent run is "+str(CompareRunNum)
    else:
        CompareRunNum,isCol = GetLatestRunNumber(CompareRunNum)

    HeadRunFile = RefRunNameTemplate % CompareRunNum
    if os.path.exists(HeadRunFile):  #check if a run file for the run we want to compare already exists
        HeadParser = pickle.load( open( HeadRunFile ) )
    else:
        HeadParser = DatabaseParser()
        HeadParser.RunNumber = CompareRunNum
        HeadParser.ParseRunSetup()
        HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
    if PrintLumi:
        for LS in HeadParser.LumiInfo[0]:
            try:
                if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
                    continue
                print str(LS)+'  '+str(round(HeadParser.LumiInfo[2][LS],1))+'  '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])*1000/23.3,0))+'  '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])*1000/23.3,0))
            except:
                print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
                                                                                                                                 
        sys.exit(0)

    if RefRunNum == 0:
        RefRates = 0
        RefLumis = 0
        LastSuccessfulIterator = 0

### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
        #CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
   
    try:
        while True:
            if not isCol:
                clear()
                MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
            else:
                RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths)

                if FindL1Zeros:
                    CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
                if int(Config.ShifterMode):
                    print "Shifter Mode. Continuing"
                else:
                    print "Expert Mode. Quitting."
                    sys.exit(0)

            
            print "Sleeping for 1 minute before repeating  "
            for iSleep in range(20):
                write(".")
                sys.stdout.flush()
                time.sleep(3)
            write("  Updating")
            sys.stdout.flush()
        #end while True
    #end try
    except KeyboardInterrupt:
        print "Quitting. Peace Out."

            
def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths):

    Header = ["Trigger Name","Actual","Expected","% Inc","Cur PS","Comments"]
    Data   = []
    Warn   = []
    IgnoredRates=[]

    [HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
    ##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
    HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
    [PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo

    pkl_file = open("Fits/2011/Fit_HLT_10LS_Run176023to180252.pkl", 'rb')
    FitInput = pickle.load(pkl_file)
    pkl_file.close()
    
    pkl_file = open("RefRuns/2011/Rates_HLT_10LS_JPAP.pkl", 'rb')
    RefRatesInput = pickle.load(pkl_file)
    pkl_file.close()

    for HeadName in HeadUnprescaledRates:
##  SKIP triggers in the skip list
##         if not HeadTotalPrescales.has_key(HeadName): ## for whatever reason we have no prescale here, so skip (calibration paths)
##             continue
##         if not HeadTotalPrescales[HeadName]: ## prescale is thought to be 0
##             continue
        
## unless we are Listing Ignored paths only look at triggers in the .list file specifed in defaults.cfg
        if StripVersion(HeadName) not in Config.MonitorList and not ListIgnoredPaths:
            continue
                
            
        masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_L2", "HLT_Zero"]
        masked_trig = False
        for mask in masked_triggers:
            if str(mask) in HeadName:
                masked_trig = True
        if masked_trig:
            continue

        skipTrig=False
        TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)

        if RefParser.RunNumber == 0:  ## Use rate prediction functions
           
            ##PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),HeadAvInstLumi)
            PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),FitInput,RefRatesInput,HeadAvLiveLumi,HeadAvDeliveredLumi)

            if PSCorrectedExpectedRate[0] < 0:  ##This means we don't have a prediction for this trigger
                continue
##             if not HeadTotalPrescales[HeadName]:
##                 print HeadName+ " has total prescale 0"
##                 continue
            ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
            PerDiff=0
            if ExpectedRate>0:
                PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
                if abs(PerDiff) > max(AllowedRateDiff/max(sqrt(TriggerRate),sqrt(ExpectedRate)),AllowedRateDiff/2.0):
                    Warn.append(True)
                else:
                    Warn.append(False)
            else:
                Warn.append(False)

            if TriggerRate < IgnoreThreshold and ExpectedRate < IgnoreThreshold:
                continue

            VC = ""
            
            Data.append([HeadName,TriggerRate,ExpectedRate,PerDiff,round(HeadUnprescaledRates[HeadName][1],1),VC]) 

        else:  ## Use a reference run
            ## cheap trick to only get triggers in list when in shifter mode
            #print "shifter mode=",int(Config.ShifterMode)
            if int(Config.ShifterMode)==1:
                if not HeadParser.AvgL1Prescales[HeadParser.HLTSeed[HeadName]]==1:
                    continue
            
            RefInstLumi = 0
            RefIterator = 0

            RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
            RefLen   = -10

            ##[RefUnprescaledRates, RefTotalPrescales, RefL1Prescales, RefTriggerRates] = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
            RefUnprescaledRates = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
            [RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefParser.GetLSRange(RefStartIndex,RefLen))
            RefRate = -1
            for k,v in RefUnprescaledRates.iteritems():
                if StripVersion(HeadName) == StripVersion(k): # versions may not match
                    RefRate = v

            ScaledRefRate = round( RefRate*HeadAvLiveLumi/RefAvLiveLumi/(HeadUnprescaledRates[HeadName][1]), 2  )

            if ScaledRefRate == 0:
                PerDiff = 100
            else:
                PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)

            if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
                continue

            if abs(PerDiff) > AllowedRateDiff:
                Warn.append(True)
            else:
                Warn.append(False)
            VC = ""
            Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,round((HeadUnprescaledRates[HeadName][1]),1),VC])

    clear()    
    PrettyPrintTable(Header,Data,[80,10,10,10,10,20],Warn)

    MoreTableInfo(HeadParser,HeadLumiRange,Config)

def CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
    print "checking trigger list"

def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
    L1Zeros=[]
    IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
    for key in HeadParser.TriggerRates:
    ## Skip events in the skip list
        skipTrig=False
    ##for trig in Config.ExcludeList:
    ##if not trigN.find(trig) == -1:
    ##skipTrig=True
    ##break
        if skipTrig:
            continue
        ## if no events pass the L1, add it to the L1Zeros list if not already there
        if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
            if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
                
                L1Zeros.append(HeadParser.TriggerRates[key][4])
                print "L1Zeros=", L1Zeros
        
    if len(L1Zeros) == 0:
       #print "It looks like no masked L1 bits seed trigger paths"
        pass
    else:
        print "The following seeds are used to seed HLT bits but accept 0 events:"
    #print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
        for Seed in L1Zeros:
            print Seed
        
if __name__=='__main__':
    main()
Revision:	1.3
Committed:	Thu Apr 5 06:13:40 2012 UTC (13 years ago) by grchrist
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.2:	+0 -0 lines
State:	*FILE REMOVED*
Log Message:	masked L1 predictions for now
#	User	Rev	Content
1	grchrist	1.1	#!/usr/bin/env python
2
3	grchrist	1.2	#from AndrewGetRun import GetRun
4			from DatabaseParser import *
5			from ReadConfig import RateMonConfig
6	grchrist	1.1	import sys
7			import os
8			import cPickle as pickle
9			import getopt
10			import time
11			from colors import *
12	grchrist	1.2	from TablePrint import *
13			from AddTableInfo_db import MoreTableInfo
14			from math import *
15	grchrist	1.1
16			WBMPageTemplate = "http://cmswbm/cmsdb/servlet/RunSummary?RUN=%s&DB=cms_omds_lb"
17			WBMRunInfoPage = "https://cmswbm/cmsdb/runSummary/RunSummary_1.html"
18
19			RefRunNameTemplate = "RefRuns/Run_%s.pk"
20
21			# define a function that clears the terminal screen
22			def clear():
23			print("\x1B[2J")
24
25	grchrist	1.2
26	grchrist	1.1	def usage():
27			print sys.argv[0]+" [Options]"
28	grchrist	1.2	print "This script gets the current HLT trigger rates and compares them to a reference run"
29	grchrist	1.1	print "Options: "
30	grchrist	1.2	print "--AllowedDiff=<diff> Report only if difference in trigger rate is greater than <diff>%"
31	grchrist	1.1	print "--CompareRun=<Run #> Compare run <Run #> to the reference run (Default = Current Run)"
32	grchrist	1.2	print "--FindL1Zeros Look for physics paths with 0 L1 rate"
33	grchrist	1.1	print "--FirstLS=<ls> Specify the first lumisection to consider. This will set LSSlidingWindow to -1"
34	grchrist	1.2	print "--NumberLS=<#> Specify the last lumisection to consider. Make sure LastLS > LSSlidingWindow"
35	grchrist	1.1	print " or set LSSlidingWindow = -1"
36	grchrist	1.2	print "--IgnoreLowRate=<rate> Ignore triggers with an actual and expected rate below <rate>"
37			print "--ListIgnoredPaths Prints the paths that are not compared by this script and their rate in the CompareRun"
38			print "--PrintLumi Prints Instantaneous, Delivered, and Live lumi by LS for the run"
39			print "--RefRun=<Run #> Specifies <Run #> as the reference run to use (Default in defaults.cfg)"
40			print "--ShowPSTriggers Show prescaled triggers in rate comparison"
41			print "--force Override the check for collisions run"
42	grchrist	1.1	print "--help Print this help"
43
44			def main():
45			try:
46	grchrist	1.2	opt, args = getopt.getopt(sys.argv[1:],"",["AllowedDiff=","CompareRun=","FindL1Zeros",\
47			"FirstLS=","NumberLS=","IgnoreLowRate=","ListIgnoredPaths",\
48			"PrintLumi","RefRun=","ShowPSTriggers","force","help"])
49	grchrist	1.1	except getopt.GetoptError, err:
50			print str(err)
51			usage()
52			sys.exit(2)
53
54			Config = RateMonConfig(os.path.abspath(os.path.dirname(sys.argv[0])))
55			for o,a in opt:
56			if o=="--ConfigFile":
57			Config.CFGfile=a
58			Config.ReadCFG()
59	grchrist	1.2
60	grchrist	1.1	AllowedRateDiff = Config.DefAllowRateDiff
61	grchrist	1.2	CompareRunNum = ""
62			FindL1Zeros = False
63			FirstLS = 9999
64			NumLS = -10
65	grchrist	1.1	IgnoreThreshold = Config.DefAllowIgnoreThresh
66	grchrist	1.2	ListIgnoredPaths = False
67			PrintLumi = False
68	grchrist	1.1	RefRunNum = int(Config.ReferenceRun)
69	grchrist	1.2	ShowPSTriggers = True
70			Force = False
71
72			if int(Config.ShifterMode):
73			print "ShifterMode!!"
74			else:
75			print "ExpertMode"
76
77	grchrist	1.1	if Config.LSWindow > 0:
78	grchrist	1.2	NumLS = -1*Config.LSWindow
79	grchrist	1.1
80			for o,a in opt: # get options passed on the command line
81	grchrist	1.2	if o=="--AllowedDiff":
82			AllowedRateDiff = float(a)/100.0
83	grchrist	1.1	elif o=="--CompareRun":
84	grchrist	1.2	CompareRunNum=int(a)
85			elif o=="--FindL1Zeros":
86			FindL1Zeros = True
87			elif o=="--FirstLS":
88			FirstLS = int(a)
89			elif o=="--NumberLS":
90			NumLS = int(a)
91	grchrist	1.1	elif o=="--IgnoreLowRate":
92			IgnoreThreshold = float(a)
93			elif o=="--ListIgnoredPaths":
94			ListIgnoredPaths=True
95	grchrist	1.2	elif o=="--PrintLumi":
96			PrintLumi = True
97			elif o=="--RefRun":
98			RefRunNum=int(a)
99			elif o=="--ShowPSTriggers":
100			ShowPSTriggers=True
101			elif o=="--force":
102			Force = True
103	grchrist	1.1	elif o=="--help":
104			usage()
105			sys.exit(0)
106			else:
107			print "Invalid Option "+a
108			sys.exit(1)
109
110	grchrist	1.2
111			RefLumisExists = False
112	grchrist	1.1
113	grchrist	1.2	"""
114			if RefRunNum > 0:
115			RefRates = {}
116			for Iterator in range(1,100):
117			if RefLumisExists: ## Quits at the end of a run
118			if max(RefLumis[0]) <= (Iterator+1)*10:
119			break
120	grchrist	1.1
121	grchrist	1.2	RefRunFile = RefRunNameTemplate % str( RefRunNum*100 + Iterator ) # place to save the reference run info
122			print "RefRunFile=",RefRunFile
123	grchrist	1.1	if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
124			print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
125			print RefRunFile[:RefRunFile.rfind('/')]
126			sys.exit(0)
127
128	grchrist	1.2	if not os.path.exists(RefRunFile): # if the reference run is not saved, get it from wbm
129			print "Reference Run File for run "+str(RefRunNum)+" iterator "+str(Iterator)+" does not exist"
130			print "Creating ..."
131			try:
132			RefParser = GetRun(RefRunNum, RefRunFile, True, Iterator10, (Iterator+1)10)
133			print "parsing"
134			except:
135			print "GetRun failed from LS "+str(Iterator10)+" to "+str((Iterator+1)10)
136			continue
137
138	grchrist	1.1	else: # otherwise load it from the file
139	grchrist	1.2	RefParser = pickle.load( open( RefRunFile ) )
140			print "loading"
141			if not RefLumisExists:
142			RefLumis = RefParser.LumiInfo
143			RefLumisExists = True
144
145			try:
146			RefRates[Iterator] = RefParser.TriggerRates # get the trigger rates from the reference run
147			LastSuccessfulIterator = Iterator
148			except:
149			print "Failed to get rates from LS "+str(Iterator10)+" to "+str((Iterator+1)10)
150			"""
151
152			RefRunFile = RefRunNameTemplate % RefRunNum
153			RefParser = DatabaseParser()
154			print "Reference Run: "+str(RefRunNum)
155			if RefRunNum > 0:
156			if not os.path.exists(RefRunFile[:RefRunFile.rfind('/')]): # folder for ref run file must exist
157			print "Reference run folder does not exist, please create" # should probably create programmatically, but for now force user to create
158			print RefRunFile[:RefRunFile.rfind('/')]
159			sys.exit(0)
160			return
161			if not os.path.exists(RefRunFile):
162			# create the reference run file
163			try:
164			RefParser.RunNumber = RefRunNum
165			RefParser.ParseRunSetup()
166			#RefParser.GetAllTriggerRatesByLS()
167			#RefParser.Save( RefRunFile )
168			except e:
169			print "PROBLEM GETTING REFERNCE RUN"
170			raise
171			else:
172			RefParser = pickle.load( open( RefRunFile ) )
173
174			# OK, Got the Reference Run
175			# Now get the most recent run
176
177			SaveRun = False
178			if CompareRunNum=="": # if no run # specified on the CL, get the most recent run
179			CompareRunNum,isCol = GetLatestRunNumber()
180
181			if not isCol:
182			print "Most Recent run, "+str(CompareRunNum)+", is NOT collisions"
183			print "Monitoring only stream A and Express"
184			#if not Force:
185			# sys.exit(0) # maybe we should walk back and try to find a collisions run, but for now just exit
186			print "Most Recent run is "+str(CompareRunNum)
187			else:
188			CompareRunNum,isCol = GetLatestRunNumber(CompareRunNum)
189
190			HeadRunFile = RefRunNameTemplate % CompareRunNum
191			if os.path.exists(HeadRunFile): #check if a run file for the run we want to compare already exists
192			HeadParser = pickle.load( open( HeadRunFile ) )
193			else:
194			HeadParser = DatabaseParser()
195			HeadParser.RunNumber = CompareRunNum
196			HeadParser.ParseRunSetup()
197			HeadLumiRange = HeadParser.GetLSRange(FirstLS,NumLS,isCol)
198			if PrintLumi:
199			for LS in HeadParser.LumiInfo[0]:
200			try:
201			if (LS < FirstLS or LS > LastLS) and not FirstLS==999999:
202			continue
203			print str(LS)+' '+str(round(HeadParser.LumiInfo[2][LS],1))+' '+str(round((HeadParser.LumiInfo[3][LS] - HeadParser.LumiInfo[3][LS-1])1000/23.3,0))+' '+str(round((HeadParser.LumiInfo[4][LS] - HeadParser.LumiInfo[4][LS-1])1000/23.3,0))
204			except:
205			print "Lumisection "+str(LS-1)+" was not parsed from the LumiSections page"
206
207			sys.exit(0)
208
209			if RefRunNum == 0:
210			RefRates = 0
211			RefLumis = 0
212			LastSuccessfulIterator = 0
213
214			### Now actually compare the rates, make tables and look at L1. Loops for ShifterMode
215			#CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
216
217			try:
218			while True:
219			if not isCol:
220			clear()
221			MoreTableInfo(HeadParser,HeadLumiRange,Config,False)
222	grchrist	1.1	else:
223	grchrist	1.2	RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths)
224	grchrist	1.1
225	grchrist	1.2	if FindL1Zeros:
226			CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config)
227			if int(Config.ShifterMode):
228			print "Shifter Mode. Continuing"
229			else:
230			print "Expert Mode. Quitting."
231			sys.exit(0)
232	grchrist	1.1
233	grchrist	1.2
234			print "Sleeping for 1 minute before repeating "
235			for iSleep in range(20):
236			write(".")
237			sys.stdout.flush()
238			time.sleep(3)
239			write(" Updating")
240			sys.stdout.flush()
241			#end while True
242			#end try
243			except KeyboardInterrupt:
244			print "Quitting. Peace Out."
245	grchrist	1.1
246	grchrist	1.2
247			def RunComparison(HeadParser,RefParser,HeadLumiRange,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config,ListIgnoredPaths):
248	grchrist	1.1
249	grchrist	1.2	Header = ["Trigger Name","Actual","Expected","% Inc","Cur PS","Comments"]
250			Data = []
251			Warn = []
252			IgnoredRates=[]
253
254			[HeadAvInstLumi,HeadAvLiveLumi,HeadAvDeliveredLumi,HeadAvDeadTime,HeadPSCols] = HeadParser.GetAvLumiInfo(HeadLumiRange)
255			##[HeadUnprescaledRates, HeadTotalPrescales, HeadL1Prescales, HeadTriggerRates] = HeadParser.UpdateRun(HeadLumiRange)
256			HeadUnprescaledRates = HeadParser.UpdateRun(HeadLumiRange)
257			[PSColumnByLS,InstLumiByLS,DeliveredLumiByLS,LiveLumiByLS,DeadTimeByLS,PhysicsByLS,ActiveByLS] = HeadParser.LumiInfo
258
259			pkl_file = open("Fits/2011/Fit_HLT_10LS_Run176023to180252.pkl", 'rb')
260			FitInput = pickle.load(pkl_file)
261			pkl_file.close()
262
263			pkl_file = open("RefRuns/2011/Rates_HLT_10LS_JPAP.pkl", 'rb')
264			RefRatesInput = pickle.load(pkl_file)
265			pkl_file.close()
266
267			for HeadName in HeadUnprescaledRates:
268			## SKIP triggers in the skip list
269			## if not HeadTotalPrescales.has_key(HeadName): ## for whatever reason we have no prescale here, so skip (calibration paths)
270			## continue
271			## if not HeadTotalPrescales[HeadName]: ## prescale is thought to be 0
272			## continue
273
274			## unless we are Listing Ignored paths only look at triggers in the .list file specifed in defaults.cfg
275			if StripVersion(HeadName) not in Config.MonitorList and not ListIgnoredPaths:
276			continue
277
278	grchrist	1.1
279	grchrist	1.2
280			masked_triggers = ["AlCa_", "DST_", "HLT_L1", "HLT_L2", "HLT_Zero"]
281			masked_trig = False
282			for mask in masked_triggers:
283			if str(mask) in HeadName:
284			masked_trig = True
285			if masked_trig:
286			continue
287
288			skipTrig=False
289			TriggerRate = round(HeadUnprescaledRates[HeadName][2],2)
290
291			if RefParser.RunNumber == 0: ## Use rate prediction functions
292
293			##PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),HeadAvInstLumi)
294			PSCorrectedExpectedRate = Config.GetExpectedRate(StripVersion(HeadName),FitInput,RefRatesInput,HeadAvLiveLumi,HeadAvDeliveredLumi)
295
296			if PSCorrectedExpectedRate[0] < 0: ##This means we don't have a prediction for this trigger
297			continue
298			## if not HeadTotalPrescales[HeadName]:
299			## print HeadName+ " has total prescale 0"
300			## continue
301			ExpectedRate = round((PSCorrectedExpectedRate[0] / HeadUnprescaledRates[HeadName][1]),2)
302			PerDiff=0
303			if ExpectedRate>0:
304			PerDiff = int(round( (TriggerRate-ExpectedRate)/ExpectedRate,2 )*100)
305			if abs(PerDiff) > max(AllowedRateDiff/max(sqrt(TriggerRate),sqrt(ExpectedRate)),AllowedRateDiff/2.0):
306			Warn.append(True)
307			else:
308			Warn.append(False)
309	grchrist	1.1	else:
310	grchrist	1.2	Warn.append(False)
311	grchrist	1.1
312	grchrist	1.2	if TriggerRate < IgnoreThreshold and ExpectedRate < IgnoreThreshold:
313			continue
314	grchrist	1.1
315	grchrist	1.2	VC = ""
316
317			Data.append([HeadName,TriggerRate,ExpectedRate,PerDiff,round(HeadUnprescaledRates[HeadName][1],1),VC])
318	grchrist	1.1
319	grchrist	1.2	else: ## Use a reference run
320			## cheap trick to only get triggers in list when in shifter mode
321			#print "shifter mode=",int(Config.ShifterMode)
322			if int(Config.ShifterMode)==1:
323			if not HeadParser.AvgL1Prescales[HeadParser.HLTSeed[HeadName]]==1:
324			continue
325	grchrist	1.1
326	grchrist	1.2	RefInstLumi = 0
327			RefIterator = 0
328
329			RefStartIndex = ClosestIndex(HeadAvInstLumi,RefParser.GetAvLumiPerRange())
330			RefLen = -10
331
332			##[RefUnprescaledRates, RefTotalPrescales, RefL1Prescales, RefTriggerRates] = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
333			RefUnprescaledRates = RefParser.UpdateRun(RefParser.GetLSRange(RefStartIndex,RefLen))
334			[RefAvInstLumi,RefAvLiveLumi,RefAvDeliveredLumi,RefAvDeadTime,RefPSCols] = RefParser.GetAvLumiInfo(RefParser.GetLSRange(RefStartIndex,RefLen))
335			RefRate = -1
336			for k,v in RefUnprescaledRates.iteritems():
337			if StripVersion(HeadName) == StripVersion(k): # versions may not match
338			RefRate = v
339
340			ScaledRefRate = round( RefRate*HeadAvLiveLumi/RefAvLiveLumi/(HeadUnprescaledRates[HeadName][1]), 2 )
341
342			if ScaledRefRate == 0:
343			PerDiff = 100
344	grchrist	1.1	else:
345	grchrist	1.2	PerDiff = int( round( (TriggerRate - ScaledRefRate)/ScaledRefRate , 2)*100)
346	grchrist	1.1
347	grchrist	1.2	if TriggerRate < IgnoreThreshold and ScaledRefRate < IgnoreThreshold:
348			continue
349	grchrist	1.1
350	grchrist	1.2	if abs(PerDiff) > AllowedRateDiff:
351			Warn.append(True)
352			else:
353			Warn.append(False)
354			VC = ""
355			Data.append([HeadName,TriggerRate,ScaledRefRate,PerDiff,round((HeadUnprescaledRates[HeadName][1]),1),VC])
356
357			clear()
358			PrettyPrintTable(Header,Data,[80,10,10,10,10,20],Warn)
359
360			MoreTableInfo(HeadParser,HeadLumiRange,Config)
361
362			def CheckTriggerList(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
363			print "checking trigger list"
364
365			def CheckL1Zeros(HeadParser,RefRunNum,RefRates,RefLumis,LastSuccessfulIterator,ShowPSTriggers,AllowedRateDiff,IgnoreThreshold,Config):
366			L1Zeros=[]
367			IgnoreBits = ["L1_PreCollisions","L1_InterBunch_Bsc","L1_BeamHalo","L1_BeamGas_Hf"]
368			for key in HeadParser.TriggerRates:
369			## Skip events in the skip list
370			skipTrig=False
371			##for trig in Config.ExcludeList:
372			##if not trigN.find(trig) == -1:
373			##skipTrig=True
374			##break
375			if skipTrig:
376			continue
377			## if no events pass the L1, add it to the L1Zeros list if not already there
378			if HeadParser.TriggerRates[key][1]==0 and not HeadParser.TriggerRates[key][4] in L1Zeros:
379			if HeadParser.TriggerRates[key][4].find('L1_BeamHalo')==-1 and HeadParser.TriggerRates[key][4].find('L1_PreCollisions')==-1 and HeadParser.TriggerRates[key][4].find('L1_InterBunch_Bsc')==-1:
380
381			L1Zeros.append(HeadParser.TriggerRates[key][4])
382			print "L1Zeros=", L1Zeros
383
384			if len(L1Zeros) == 0:
385			#print "It looks like no masked L1 bits seed trigger paths"
386			pass
387			else:
388			print "The following seeds are used to seed HLT bits but accept 0 events:"
389			#print "The average lumi of this run is: "+str(round(HeadParser.LumiInfo[6],1))+"e30"
390			for Seed in L1Zeros:
391			print Seed
392	grchrist	1.1
393			if __name__=='__main__':
394			main()