MyAnalysis/tools/getHLTkey.py

#!/usr/bin/env python 
import os
from sys import stderr, exit
from RecoLuminosity.LumiDB import selectionParser
import commands
import fileinput
import string

#
# HLT key and run query: E.P., 27 July 2010
# Specific trigger information added: bdahmes, 23 Aug 2010 
# query to the Run Registry taken from a script by Giovanni Petrucianni
#


from optparse import OptionParser
parser = OptionParser(usage="usage: %prog [options]")
parser.add_option("--firstRun",  dest="firstRun",  help="first run", type="int", metavar="RUN", default="1")
parser.add_option("--lastRun",   dest="lastRun",   help="last run",  type="int", metavar="RUN", default="9999999")
parser.add_option("--groupName", dest="groupName", help="select runs of name like NAME", metavar="NAME", default="Collisions%")
parser.add_option("--rrurl",     dest="rrurl",     help="run registry xmlrpc url", metavar="URL", default="http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc")
parser.add_option("--HLTkey",    dest="HLTkey",    help="name of the HLTkey e.g. /cdaq/physics/Run2010/v3.1/HLT_1.6E30/V1",metavar="HLT")
parser.add_option("--perKey",    action="store_true",default=False,dest="perKey",help="list the runs per HLT key",metavar="perKey")
parser.add_option("--json",      dest="json",      help="JSON file for good runs", type="string", default="None")
parser.add_option("--HLTpaths",  dest="HLTpaths",  help="Comma-separated list of HLT paths (a,b,c)", default="None")
parser.add_option("--latex",     action="store_true", default=False, dest="latex",help="table output in LaTeX format",metavar="latex")
parser.add_option("--runsPerRow",dest="runsPerRow",default=-1, type="int",help="Runs printed before new table row",metavar="NUMBER")
(options, args) = parser.parse_args()

def max(i,j):
    if i > j:
        return i
    else:
        return j

def queryRR():
    stderr.write("Querying run registry for range [%d, %d], group name like %s ...\n" % (options.firstRun, options.lastRun, options.groupName))
    import xmlrpclib
    import xml.dom.minidom
    server = xmlrpclib.ServerProxy(options.rrurl)
    run_data = server.DataExporter.export('RUN', 'GLOBAL', 'xml_datasets', "{runNumber} >= %d AND {runNumber} <= %d AND {groupName} like '%s' AND {datasetName} = '/Global/Online/ALL'"  % (options.firstRun, options.lastRun, options.groupName))
    ret = {}
    xml_data = xml.dom.minidom.parseString(run_data)
    xml_runs = xml_data.documentElement.getElementsByTagName("RUN_DATASET")
    for xml_run in xml_runs:
        ret[xml_run.getElementsByTagName("RUN_NUMBER")[0].firstChild.nodeValue] = xml_run.getElementsByTagName("RUN_HLTKEY")[0].firstChild.nodeValue
    return ret

tempHLTfile = 'tempHLT.log'
def pathInfo(path):
    searchString = " " + path + " "
    result = [line for line in
              open(tempHLTfile) if searchString in line]
    if len(result) > 0:
        return result[0]
    else:
        return ""

runKeys = queryRR()
prescaleTable = {}
runs = runKeys.keys(); runs.sort()
paths = []
runsPerRow = int(options.runsPerRow)

if options.HLTpaths != "None":
    pathString = options.HLTpaths
    beginString = 0
    ctr = 0 
    while beginString < len(pathString):
        ctr += 1
        nextString = pathString[beginString:-1].find(',')
        endString = beginString + nextString
        if endString < beginString:
            endString = len(pathString)
        newPath = pathString[beginString:endString]
        beginString = endString + 1
        paths.append(newPath)

if options.json != "None":
    jfile  = open(options.json,'r')
    filteredRunList = []
    goodLS = ''
    parsingResult = '' 
    goodLS = jfile.read()
    parsingResult = selectionParser.selectionParser(goodLS)
    if not parsingResult:
        print 'Failed to parse the input JSON file',ifilename
        raise 
    goodRuns = parsingResult.runs()
    for run in runs:
        key = runKeys[run]
        for anotherRun in goodRuns:
            if int(run) == anotherRun:
                filteredRunList.append(run)
    runs = filteredRunList

if options.perKey:
    runsPerKey={}
    for run in runs:
        key = runKeys[run]
        if not key in runsPerKey.keys():
            tmpruns=[]
            tmpruns.append(run)
            runsPerKey[key] = tmpruns
        else:
            runsPerKey[key].append(run)

        theKeys = runsPerKey.keys()
    # Resort the menus based on run number
    resortedKeys = []
    totalKeys = len(theKeys)
    while len(resortedKeys) != totalKeys:
        earliestRun = 999999
        earliestKey = ''
        for key in theKeys:
            first = int(runsPerKey[key][0])
            if first < earliestRun:
                earliestKey = key
                earliestRun = first
        resortedKeys.append(earliestKey)
        theKeys.remove(earliestKey)
    theKeys = resortedKeys

    # Look for special features based on interesting paths
    # NOTE: DQM is ignored (avoids seeing path appear more than once)
    featuresPerKey = {}
    if options.HLTpaths != "None":
        tempHLTfile = "tempHLT.log"
        for key in theKeys:
            mySetup = "touch " + tempHLTfile + " ; rm " + tempHLTfile
            os.system(mySetup)
            myGet = "edmConfigFromDB --orcoff --configName " + key + " --noes --services PrescaleService | hltDumpStream > " + tempHLTfile
            os.system(myGet)
            infoString = ''
            for path in paths:
                info = pathInfo(path).split()
                i = 1
                foundPrescale = False
                while i < len(info):
                    if info[i].isdigit():
                        if foundPrescale:
                            info.pop(i)
                        else:
                            foundPrescale = True
                            i += 1
                    else:
                        i += 1
                if len(info) > 4:
                    lastColumn = info[len(info)-1].split()
                    info[3] = lastColumn[len(lastColumn)-1]
                    while len(info) > 4:
                        dummy = info.pop()
                for item in info:
                    infoString += item.strip() + ':'
                infoString = infoString[:len(infoString)-1] + ';'
            # if options.latex:
            #     infoString = string.replace(infoString,'_','\\_')
            # print infoString
            featuresPerKey[key] = infoString
            myCleanup = "rm " + tempHLTfile
            os.system(myCleanup)
    
    pathFeatures = {}
    pathPrescale = {}
    pathL1seed = {}
    updatedFeatures = {}
    updatedPrescale = {}
    updatedL1seed = {}
    for path in paths:
        pathFeatures[path]    = ''
        pathPrescale[path]    = ''
        pathL1seed[path]      = ''
        updatedFeatures[path] = 'X'
        updatedPrescale[path] = ''
        updatedL1seed[path]   = ''

    beginString = ''
    separatorString = ''
    endString = ''
    if options.latex:
        separatorString = '&'
        endString = '\\\\'

    maxLen = 0 
    for key in theKeys:
        theLen = len(key)
        if theLen > maxLen:
            maxLen = theLen

    for key in theKeys:
        runList = []
        theruns = runsPerKey[key]
        topr = ""
        for r in theruns:
            topr += r + ","
            if runsPerRow > 0 and topr.count(',') == runsPerRow and theruns.index(r) != (len(theruns) - 1):
                runList.append(topr)
                topr = ""
        topr = topr[:len(topr)-1]
        runList.append(topr)

        # Always assume a path has been disabled, and prepare to be wrong
        for path in paths:
            updatedFeatures[path] = 'X'

        if options.HLTpaths != "None":
            parseMenuInfo = featuresPerKey[key].split(';',len(paths))
            for pathBlock in parseMenuInfo:
                parsePathInfo = pathBlock.split(':')
                if len(parsePathInfo) > 1:
                    thePath = parsePathInfo[len(parsePathInfo)-3]
                    # thePath = string.replace(thePath,'\\_','_')
                    updatedFeatures[thePath] = 'U'
                    if parsePathInfo[0] == '~':
                        updatedFeatures[thePath] = 'G'
                    updatedPrescale[thePath] = parsePathInfo[len(parsePathInfo)-2]
                    updatedL1seed[thePath] = parsePathInfo[len(parsePathInfo)-1]

        xtraSpace = ''
        for i in range(0,maxLen-len(key)):
            xtraSpace += ' '
        outputKey = key
        if options.latex:
            outputKey = string.replace(key,'_','\\_')
        print beginString,outputKey,xtraSpace,separatorString,
        featureString = []
        for path in paths:
            stringPrefix = ''
            if updatedFeatures[path] != pathFeatures[path]:
                pathFeatures[path] = updatedFeatures[path]
                stringPrefix += '(' + pathFeatures[path] + ')' 
            L1andPrescale = ''
            if updatedPrescale[path] != pathPrescale[path] or updatedL1seed[path] != pathL1seed[path]:
                if updatedPrescale[path] == pathPrescale[path]:
                    L1andPrescale = '(' + updatedL1seed[path] + ')'
                elif updatedL1seed[path] == pathL1seed[path]:
                    L1andPrescale = '(' + updatedPrescale[path] + ')'
                else:
                    L1andPrescale = '(' + updatedL1seed[path] + ',' + updatedPrescale[path] + ')'
                pathPrescale[path] = updatedPrescale[path]
                pathL1seed[path]   = updatedL1seed[path]
            pathString = ''
            if len(stringPrefix) > 0 or len(L1andPrescale) > 0:
                pathString = path
                if len(stringPrefix) > 0:
                    pathString = stringPrefix + pathString
                if len(L1andPrescale) > 0:
                    pathString += L1andPrescale
                if options.latex:
                    pathString = string.replace(pathString,'_','\\_')
            featureString.append(pathString)
        oldLength = len(featureString)
        for i in range(0,oldLength):
            idx = oldLength - 1 - i
            if len(featureString[idx]) == 0:
                featureString.pop(idx)
        for i in range(0,max(len(featureString),len(runList))):
            if i > 0:
                print beginString,' ', 
                for j in range(0,maxLen):
                    print '',
                print separatorString,
            if i >= len(runList):
                print featureString[i],separatorString,' ',
            elif i >= len(featureString):
                print ' ', separatorString,runList[i],
            else:
                print featureString[i],separatorString,runList[i],
            print endString
            
    exit(1)
                        
if options.HLTkey:
        HLTkey = options.HLTkey
        print "List of runs taken with HLT key = ",HLTkey 
for run in runs:
    key = runKeys[run]

    if not options.HLTkey:
       print run,key    
    else:
        if key == options.HLTkey:
           print run
Revision:	1.2
Committed:	Wed Nov 16 19:30:31 2011 UTC (13 years, 5 months ago) by ferencek
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.1:	+1 -1 lines
Log Message:	updated RunRegistry API URL
#	User	Rev	Content
1	ferencek	1.1	#!/usr/bin/env python
2			import os
3			from sys import stderr, exit
4			from RecoLuminosity.LumiDB import selectionParser
5			import commands
6			import fileinput
7			import string
8
9			#
10			# HLT key and run query: E.P., 27 July 2010
11			# Specific trigger information added: bdahmes, 23 Aug 2010
12			# query to the Run Registry taken from a script by Giovanni Petrucianni
13			#
14
15
16			from optparse import OptionParser
17			parser = OptionParser(usage="usage: %prog [options]")
18			parser.add_option("--firstRun", dest="firstRun", help="first run", type="int", metavar="RUN", default="1")
19			parser.add_option("--lastRun", dest="lastRun", help="last run", type="int", metavar="RUN", default="9999999")
20			parser.add_option("--groupName", dest="groupName", help="select runs of name like NAME", metavar="NAME", default="Collisions%")
21	ferencek	1.2	parser.add_option("--rrurl", dest="rrurl", help="run registry xmlrpc url", metavar="URL", default="http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc")
22	ferencek	1.1	parser.add_option("--HLTkey", dest="HLTkey", help="name of the HLTkey e.g. /cdaq/physics/Run2010/v3.1/HLT_1.6E30/V1",metavar="HLT")
23			parser.add_option("--perKey", action="store_true",default=False,dest="perKey",help="list the runs per HLT key",metavar="perKey")
24			parser.add_option("--json", dest="json", help="JSON file for good runs", type="string", default="None")
25			parser.add_option("--HLTpaths", dest="HLTpaths", help="Comma-separated list of HLT paths (a,b,c)", default="None")
26			parser.add_option("--latex", action="store_true", default=False, dest="latex",help="table output in LaTeX format",metavar="latex")
27			parser.add_option("--runsPerRow",dest="runsPerRow",default=-1, type="int",help="Runs printed before new table row",metavar="NUMBER")
28			(options, args) = parser.parse_args()
29
30			def max(i,j):
31			if i > j:
32			return i
33			else:
34			return j
35
36			def queryRR():
37			stderr.write("Querying run registry for range [%d, %d], group name like %s ...\n" % (options.firstRun, options.lastRun, options.groupName))
38			import xmlrpclib
39			import xml.dom.minidom
40			server = xmlrpclib.ServerProxy(options.rrurl)
41			run_data = server.DataExporter.export('RUN', 'GLOBAL', 'xml_datasets', "{runNumber} >= %d AND {runNumber} <= %d AND {groupName} like '%s' AND {datasetName} = '/Global/Online/ALL'" % (options.firstRun, options.lastRun, options.groupName))
42			ret = {}
43			xml_data = xml.dom.minidom.parseString(run_data)
44			xml_runs = xml_data.documentElement.getElementsByTagName("RUN_DATASET")
45			for xml_run in xml_runs:
46			ret[xml_run.getElementsByTagName("RUN_NUMBER")[0].firstChild.nodeValue] = xml_run.getElementsByTagName("RUN_HLTKEY")[0].firstChild.nodeValue
47			return ret
48
49			tempHLTfile = 'tempHLT.log'
50			def pathInfo(path):
51			searchString = " " + path + " "
52			result = [line for line in
53			open(tempHLTfile) if searchString in line]
54			if len(result) > 0:
55			return result[0]
56			else:
57			return ""
58
59			runKeys = queryRR()
60			prescaleTable = {}
61			runs = runKeys.keys(); runs.sort()
62			paths = []
63			runsPerRow = int(options.runsPerRow)
64
65			if options.HLTpaths != "None":
66			pathString = options.HLTpaths
67			beginString = 0
68			ctr = 0
69			while beginString < len(pathString):
70			ctr += 1
71			nextString = pathString[beginString:-1].find(',')
72			endString = beginString + nextString
73			if endString < beginString:
74			endString = len(pathString)
75			newPath = pathString[beginString:endString]
76			beginString = endString + 1
77			paths.append(newPath)
78
79			if options.json != "None":
80			jfile = open(options.json,'r')
81			filteredRunList = []
82			goodLS = ''
83			parsingResult = ''
84			goodLS = jfile.read()
85			parsingResult = selectionParser.selectionParser(goodLS)
86			if not parsingResult:
87			print 'Failed to parse the input JSON file',ifilename
88			raise
89			goodRuns = parsingResult.runs()
90			for run in runs:
91			key = runKeys[run]
92			for anotherRun in goodRuns:
93			if int(run) == anotherRun:
94			filteredRunList.append(run)
95			runs = filteredRunList
96
97			if options.perKey:
98			runsPerKey={}
99			for run in runs:
100			key = runKeys[run]
101			if not key in runsPerKey.keys():
102			tmpruns=[]
103			tmpruns.append(run)
104			runsPerKey[key] = tmpruns
105			else:
106			runsPerKey[key].append(run)
107
108			theKeys = runsPerKey.keys()
109			# Resort the menus based on run number
110			resortedKeys = []
111			totalKeys = len(theKeys)
112			while len(resortedKeys) != totalKeys:
113			earliestRun = 999999
114			earliestKey = ''
115			for key in theKeys:
116			first = int(runsPerKey[key][0])
117			if first < earliestRun:
118			earliestKey = key
119			earliestRun = first
120			resortedKeys.append(earliestKey)
121			theKeys.remove(earliestKey)
122			theKeys = resortedKeys
123
124			# Look for special features based on interesting paths
125			# NOTE: DQM is ignored (avoids seeing path appear more than once)
126			featuresPerKey = {}
127			if options.HLTpaths != "None":
128			tempHLTfile = "tempHLT.log"
129			for key in theKeys:
130			mySetup = "touch " + tempHLTfile + " ; rm " + tempHLTfile
131			os.system(mySetup)
132			myGet = "edmConfigFromDB --orcoff --configName " + key + " --noes --services PrescaleService \| hltDumpStream > " + tempHLTfile
133			os.system(myGet)
134			infoString = ''
135			for path in paths:
136			info = pathInfo(path).split()
137			i = 1
138			foundPrescale = False
139			while i < len(info):
140			if info[i].isdigit():
141			if foundPrescale:
142			info.pop(i)
143			else:
144			foundPrescale = True
145			i += 1
146			else:
147			i += 1
148			if len(info) > 4:
149			lastColumn = info[len(info)-1].split()
150			info[3] = lastColumn[len(lastColumn)-1]
151			while len(info) > 4:
152			dummy = info.pop()
153			for item in info:
154			infoString += item.strip() + ':'
155			infoString = infoString[:len(infoString)-1] + ';'
156			# if options.latex:
157			# infoString = string.replace(infoString,'_','\\_')
158			# print infoString
159			featuresPerKey[key] = infoString
160			myCleanup = "rm " + tempHLTfile
161			os.system(myCleanup)
162
163			pathFeatures = {}
164			pathPrescale = {}
165			pathL1seed = {}
166			updatedFeatures = {}
167			updatedPrescale = {}
168			updatedL1seed = {}
169			for path in paths:
170			pathFeatures[path] = ''
171			pathPrescale[path] = ''
172			pathL1seed[path] = ''
173			updatedFeatures[path] = 'X'
174			updatedPrescale[path] = ''
175			updatedL1seed[path] = ''
176
177			beginString = ''
178			separatorString = ''
179			endString = ''
180			if options.latex:
181			separatorString = '&'
182			endString = '\\\\'
183
184			maxLen = 0
185			for key in theKeys:
186			theLen = len(key)
187			if theLen > maxLen:
188			maxLen = theLen
189
190			for key in theKeys:
191			runList = []
192			theruns = runsPerKey[key]
193			topr = ""
194			for r in theruns:
195			topr += r + ","
196			if runsPerRow > 0 and topr.count(',') == runsPerRow and theruns.index(r) != (len(theruns) - 1):
197			runList.append(topr)
198			topr = ""
199			topr = topr[:len(topr)-1]
200			runList.append(topr)
201
202			# Always assume a path has been disabled, and prepare to be wrong
203			for path in paths:
204			updatedFeatures[path] = 'X'
205
206			if options.HLTpaths != "None":
207			parseMenuInfo = featuresPerKey[key].split(';',len(paths))
208			for pathBlock in parseMenuInfo:
209			parsePathInfo = pathBlock.split(':')
210			if len(parsePathInfo) > 1:
211			thePath = parsePathInfo[len(parsePathInfo)-3]
212			# thePath = string.replace(thePath,'\\_','_')
213			updatedFeatures[thePath] = 'U'
214			if parsePathInfo[0] == '~':
215			updatedFeatures[thePath] = 'G'
216			updatedPrescale[thePath] = parsePathInfo[len(parsePathInfo)-2]
217			updatedL1seed[thePath] = parsePathInfo[len(parsePathInfo)-1]
218
219			xtraSpace = ''
220			for i in range(0,maxLen-len(key)):
221			xtraSpace += ' '
222			outputKey = key
223			if options.latex:
224			outputKey = string.replace(key,'_','\\_')
225			print beginString,outputKey,xtraSpace,separatorString,
226			featureString = []
227			for path in paths:
228			stringPrefix = ''
229			if updatedFeatures[path] != pathFeatures[path]:
230			pathFeatures[path] = updatedFeatures[path]
231			stringPrefix += '(' + pathFeatures[path] + ')'
232			L1andPrescale = ''
233			if updatedPrescale[path] != pathPrescale[path] or updatedL1seed[path] != pathL1seed[path]:
234			if updatedPrescale[path] == pathPrescale[path]:
235			L1andPrescale = '(' + updatedL1seed[path] + ')'
236			elif updatedL1seed[path] == pathL1seed[path]:
237			L1andPrescale = '(' + updatedPrescale[path] + ')'
238			else:
239			L1andPrescale = '(' + updatedL1seed[path] + ',' + updatedPrescale[path] + ')'
240			pathPrescale[path] = updatedPrescale[path]
241			pathL1seed[path] = updatedL1seed[path]
242			pathString = ''
243			if len(stringPrefix) > 0 or len(L1andPrescale) > 0:
244			pathString = path
245			if len(stringPrefix) > 0:
246			pathString = stringPrefix + pathString
247			if len(L1andPrescale) > 0:
248			pathString += L1andPrescale
249			if options.latex:
250			pathString = string.replace(pathString,'_','\\_')
251			featureString.append(pathString)
252			oldLength = len(featureString)
253			for i in range(0,oldLength):
254			idx = oldLength - 1 - i
255			if len(featureString[idx]) == 0:
256			featureString.pop(idx)
257			for i in range(0,max(len(featureString),len(runList))):
258			if i > 0:
259			print beginString,' ',
260			for j in range(0,maxLen):
261			print '',
262			print separatorString,
263			if i >= len(runList):
264			print featureString[i],separatorString,' ',
265			elif i >= len(featureString):
266			print ' ', separatorString,runList[i],
267			else:
268			print featureString[i],separatorString,runList[i],
269			print endString
270
271			exit(1)
272
273			if options.HLTkey:
274			HLTkey = options.HLTkey
275			print "List of runs taken with HLT key = ",HLTkey
276			for run in runs:
277			key = runKeys[run]
278
279			if not options.HLTkey:
280			print run,key
281			else:
282			if key == options.HLTkey:
283			print run