| 1 |
#!/usr/bin/env python
|
| 2 |
|
| 3 |
#
|
| 4 |
# $Id: getRunInfo.py,v 1.3.2.5 2008/10/29 20:46:59 vadler Exp $
|
| 5 |
#
|
| 6 |
|
| 7 |
## CMSSW/DQM/SiStripMonitorClient/scripts/getRunInfo.py
|
| 8 |
#
|
| 9 |
# For a given run, this script collects information useful for SiStrip DQM
|
| 10 |
# from web sources.
|
| 11 |
# Questions and comments to: volker.adler@cern.ch
|
| 12 |
|
| 13 |
|
| 14 |
import sys
|
| 15 |
import os
|
| 16 |
import string
|
| 17 |
import urllib
|
| 18 |
import datetime
|
| 19 |
|
| 20 |
# Constants
|
| 21 |
|
| 22 |
# numbers
|
| 23 |
INT_offset = 8
|
| 24 |
# strings
|
| 25 |
STR_SiStrip = 'SIST'
|
| 26 |
STR_wwwDBSData = 'https://cmsweb.cern.ch/dbs_discovery/getData'
|
| 27 |
LSTR_dbsInstances = ['cms_dbs_caf_analysis_01',
|
| 28 |
'cms_dbs_prod_global' ]
|
| 29 |
STR_headDatasets = 'datasets'
|
| 30 |
STR_headFiles = 'available data files'
|
| 31 |
DICT_htmlTags = {}
|
| 32 |
DICT_htmlTags['GLOBAL_NAME'] = 'global name '
|
| 33 |
DICT_htmlTags['STATUS'] = 'status '
|
| 34 |
DICT_htmlTags['IN_DBS'] = 'in DBS '
|
| 35 |
DICT_htmlTags['SUBSYSTEMS'] = 'subsystems '
|
| 36 |
DICT_htmlTags['EVENTS'] = '# of triggers '
|
| 37 |
DICT_htmlTags['START_TIME'] = 'start time (local) '
|
| 38 |
DICT_htmlTags['END_TIME'] = 'end time (local) '
|
| 39 |
DICT_htmlTags['L1KEY'] = 'L1 key '
|
| 40 |
DICT_htmlTags['HLTKEY'] = 'HLT key '
|
| 41 |
DICT_htmlTags['L1SOURCES'] = 'L1 sources '
|
| 42 |
DICT_htmlTags['RUN_RATE'] = 'event rate (Hz) '
|
| 43 |
DICT_htmlTags['STOP_REASON'] = 'stop reason '
|
| 44 |
DICT_htmlTags['SHIFTER'] = 'DQM shifter '
|
| 45 |
DICT_htmlTags['CREATE_USER'] = 'entry created by '
|
| 46 |
DICT_htmlTags['CREATE_TIME'] = 'entry creation time '
|
| 47 |
DICT_htmlTags['ONLINE_COMMENT'] = 'DQM online shifter\'s comment '
|
| 48 |
DICT_htmlTags['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment'
|
| 49 |
|
| 50 |
# Globals
|
| 51 |
|
| 52 |
global Str_run
|
| 53 |
global Dict_cmsmonRunRegistry
|
| 54 |
global DictDict_dbsDatasets
|
| 55 |
global DictDict_dbsEvents
|
| 56 |
global Dict_dbsDatasets
|
| 57 |
global Dict_maxLenDbsDatasets
|
| 58 |
# initialise
|
| 59 |
Str_run = sys.argv[1]
|
| 60 |
Dict_cmsmonRunRegistry = {}
|
| 61 |
DictDict_dbsDatasets = {}
|
| 62 |
DictDict_dbsEvents = {}
|
| 63 |
Dict_dbsDatasets = {}
|
| 64 |
Dict_maxLenDbsDatasets = {}
|
| 65 |
|
| 66 |
## FUNCTIONS
|
| 67 |
|
| 68 |
## Func_GetHtmlTags(str_text)
|
| 69 |
#
|
| 70 |
# Gets HTML tags from a string
|
| 71 |
def Func_GetHtmlTags(str_text):
|
| 72 |
""" Func_GetHtmlTags(str_text):
|
| 73 |
Gets HTML tags from a string
|
| 74 |
"""
|
| 75 |
dict_tags = {}
|
| 76 |
# first look for tags w/ values
|
| 77 |
lstr_split = str_text.split('</')
|
| 78 |
for str_split in lstr_split[1:]:
|
| 79 |
str_key = str_split.split('>')[0]
|
| 80 |
dict_tags[str_key] = str_key in dict_tags
|
| 81 |
# second look for tags w/o values
|
| 82 |
lstr_split = str_text.split('/>')
|
| 83 |
for str_split in lstr_split[:-1]:
|
| 84 |
str_key = str_split.split('<')[-1].split()[0]
|
| 85 |
dict_tags[str_key] = str_key in dict_tags
|
| 86 |
return dict_tags
|
| 87 |
|
| 88 |
## Func_GetHtmlTagValue(str_tag, str_text)
|
| 89 |
#
|
| 90 |
# Gets the value of the n-th oocurence a given HTML tag from a string
|
| 91 |
def Func_GetHtmlTagValue(str_tag, str_text, int_index = 1):
|
| 92 |
""" Func_GetHtmlTagValue(str_tag, str_text):
|
| 93 |
Gets the value of the n-th oocurence a given HTML tag from a string
|
| 94 |
"""
|
| 95 |
if int_index > str_text.count('<'+str_tag):
|
| 96 |
return ''
|
| 97 |
str_1 = str_text.split('<'+str_tag)[int_index]
|
| 98 |
if str_1[0] != '>':
|
| 99 |
if str_1.split('>')[0][-1] == '/':
|
| 100 |
return ''
|
| 101 |
return str_1.split('>',1)[1].split('</'+str_tag+'>')[0]
|
| 102 |
|
| 103 |
## Func_GetHtmlTagValues(str_text)
|
| 104 |
#
|
| 105 |
# Gets HTML tag values from a string
|
| 106 |
def Func_GetHtmlTagValues(str_text):
|
| 107 |
""" Func_GetHtmlTagValues(str_text):
|
| 108 |
Gets HTML tag values from a string
|
| 109 |
"""
|
| 110 |
lstr_split = str_text.split('</')
|
| 111 |
lstr_values = []
|
| 112 |
for str_split in lstr_split[:-1]:
|
| 113 |
lstr_values.append(str_split.split('>')[-1])
|
| 114 |
return lstr_values
|
| 115 |
|
| 116 |
## Func_GetHtmlTagValueAttr(str_tag, str_text)
|
| 117 |
#
|
| 118 |
# Gets the (last) attribute of a given HTML tag value from a string
|
| 119 |
def Func_GetHtmlTagValueAttr(str_value, str_text):
|
| 120 |
""" Func_GetHtmlTagValueAttr(str_value, str_text):
|
| 121 |
Gets the (last) attributes of a given HTML tag value from a string
|
| 122 |
"""
|
| 123 |
return str_text.split('\">'+str_value+'<')[0].split('=\"')[-1]
|
| 124 |
|
| 125 |
## Func_FillInfoRunRegistry()
|
| 126 |
#
|
| 127 |
# Retrieves run info from RunRegistry and fills it into containers
|
| 128 |
def Func_FillInfoRunRegistry():
|
| 129 |
""" Func_FillInfoRunRegistry():
|
| 130 |
Retrieves run info from RunRegistry and fills it into containers
|
| 131 |
"""
|
| 132 |
str_cmsmonRunRegistry = urllib.urlencode({'format':'xml', 'intpl':'xml', 'qtype':'RUN_NUMBER', 'sortname':'RUN_NUMBER'})
|
| 133 |
file_cmsmonRunRegistry = urllib.urlopen("http://pccmsdqm04.cern.ch/runregistry/runregisterdata", str_cmsmonRunRegistry)
|
| 134 |
str_cmsmonRunRegistryLong = ''
|
| 135 |
for str_cmsmonRunRegistry in file_cmsmonRunRegistry.readlines():
|
| 136 |
str_cmsmonRunRegistryLong += str_cmsmonRunRegistry.splitlines()[0]
|
| 137 |
bool_foundRun = False
|
| 138 |
str_cmsmonRun = ''
|
| 139 |
for int_runIndex in range(1,int(str_cmsmonRunRegistryLong.split('<RUNS')[1].split('>')[0].split('total=\"')[1].split('\"')[0])):
|
| 140 |
str_cmsmonRun = Func_GetHtmlTagValue('RUN', str_cmsmonRunRegistryLong, int_runIndex)
|
| 141 |
if Func_GetHtmlTagValue('NUMBER', str_cmsmonRun) == Str_run:
|
| 142 |
bool_foundRun = True
|
| 143 |
break
|
| 144 |
if not bool_foundRun:
|
| 145 |
print '> getRunInfo.py > run ' + Str_run + ' not found in run registry'
|
| 146 |
return False
|
| 147 |
dict_cmsmonHtmlTags = Func_GetHtmlTags(str_cmsmonRun)
|
| 148 |
for str_cmsmonHtmlTag in dict_cmsmonHtmlTags.keys():
|
| 149 |
if dict_cmsmonHtmlTags[str_cmsmonHtmlTag] == False:
|
| 150 |
Dict_cmsmonRunRegistry[str_cmsmonHtmlTag] = Func_GetHtmlTagValue(str_cmsmonHtmlTag, str_cmsmonRun)
|
| 151 |
if Dict_cmsmonRunRegistry['SUBSYSTEMS'].find(STR_SiStrip) < 0:
|
| 152 |
print '> getRunInfo.py > SiStrip was not in this run'
|
| 153 |
return False
|
| 154 |
return True
|
| 155 |
|
| 156 |
## Func_FillInfoDBS(str_dbsInstance)
|
| 157 |
#
|
| 158 |
# Retrieves run info from DBS and fills it into containers
|
| 159 |
def Func_FillInfoDBS(str_dbsInstance):
|
| 160 |
""" Func_FillInfoDBS(str_dbsInstance)
|
| 161 |
Retrieves run info from DBS and fills it into containers
|
| 162 |
"""
|
| 163 |
str_dbsRuns = urllib.urlencode({'ajax':'0', '_idx':'0', 'pagerStep':'0', 'userMode':'user', 'release':'Any', 'tier':'Any', 'dbsInst':str_dbsInstance, 'primType':'Any', 'primD':'Any', 'minRun':Str_run, 'maxRun':Str_run})
|
| 164 |
file_dbsRuns = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getRunsFromRange", str_dbsRuns)
|
| 165 |
lstr_dbsRuns = []
|
| 166 |
lstr_dbsDatasets = []
|
| 167 |
dict_dbsDatasets = {}
|
| 168 |
dict_dbsEvents = {}
|
| 169 |
for str_dbsRuns in file_dbsRuns.readlines():
|
| 170 |
lstr_dbsRuns.append(str_dbsRuns)
|
| 171 |
if str_dbsRuns.find(STR_wwwDBSData) >= 0:
|
| 172 |
if str_dbsRuns.split('&proc=')[1].find('&') >= 0:
|
| 173 |
lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1].split('&')[0])
|
| 174 |
else:
|
| 175 |
lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1])
|
| 176 |
int_maxLenDbsDatasets = 0
|
| 177 |
for str_dbsDataset in lstr_dbsDatasets:
|
| 178 |
str_dbsLFN = urllib.urlencode({'dbsInst':str_dbsInstance, 'blockName':'*', 'dataset':str_dbsDataset, 'userMode':'user', 'run':Str_run})
|
| 179 |
file_dbsLFN = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFNlist", str_dbsLFN)
|
| 180 |
lstr_dbsLFN = []
|
| 181 |
int_events = 0
|
| 182 |
for str_dbsLFN in file_dbsLFN.readlines():
|
| 183 |
lstr_dbsLFN.append(str_dbsLFN)
|
| 184 |
if str_dbsLFN.find('contians') >= 0 and str_dbsLFN.find('file(s)'): # FIXME: be careful, this typo might be corrected sometimes on the web page...
|
| 185 |
dict_dbsDatasets[str_dbsDataset] = str_dbsLFN.split()[1]
|
| 186 |
if str_dbsLFN.startswith('/store/data/'):
|
| 187 |
int_events += int(Func_GetHtmlTagValue('td' ,lstr_dbsLFN[len(lstr_dbsLFN)-4]))
|
| 188 |
dict_dbsEvents[str_dbsDataset] = str(int_events)
|
| 189 |
if len(str_dbsDataset) > int_maxLenDbsDatasets:
|
| 190 |
int_maxLenDbsDatasets = len(str_dbsDataset)
|
| 191 |
DictDict_dbsDatasets[str_dbsInstance] = dict_dbsDatasets
|
| 192 |
DictDict_dbsEvents[str_dbsInstance] = dict_dbsEvents
|
| 193 |
Dict_dbsDatasets[str_dbsInstance] = lstr_dbsDatasets
|
| 194 |
Dict_maxLenDbsDatasets[str_dbsInstance] = int_maxLenDbsDatasets
|
| 195 |
|
| 196 |
## MAIN PROGRAM
|
| 197 |
|
| 198 |
print
|
| 199 |
print '> getRunInfo.py > information on run \t*** %s ***' %(Str_run)
|
| 200 |
print
|
| 201 |
|
| 202 |
# Get run information from the web
|
| 203 |
|
| 204 |
# get run RunRegistry entries
|
| 205 |
bool_runRegistry = Func_FillInfoRunRegistry()
|
| 206 |
|
| 207 |
# get run DBS entries
|
| 208 |
for str_dbsInstance in LSTR_dbsInstances:
|
| 209 |
Func_FillInfoDBS(str_dbsInstance)
|
| 210 |
|
| 211 |
# Print information
|
| 212 |
|
| 213 |
# from run registry
|
| 214 |
if bool_runRegistry:
|
| 215 |
print
|
| 216 |
print '> getRunInfo.py > * information from run registry *'
|
| 217 |
print
|
| 218 |
for str_htmlTag in DICT_htmlTags.keys():
|
| 219 |
if str_htmlTag in Dict_cmsmonRunRegistry:
|
| 220 |
print '> getRunInfo.py > %s: %s' %(DICT_htmlTags[str_htmlTag],Dict_cmsmonRunRegistry[str_htmlTag])
|
| 221 |
|
| 222 |
# from DBS
|
| 223 |
print
|
| 224 |
print '> getRunInfo.py > * information from DBS *'
|
| 225 |
print
|
| 226 |
for str_dbsInstance in LSTR_dbsInstances:
|
| 227 |
print '> getRunInfo.py > DBS instance: %s' %(str_dbsInstance)
|
| 228 |
if str_dbsInstance == LSTR_dbsInstances[0]:
|
| 229 |
print ' (This is the instance used at CAF!)'
|
| 230 |
str_print = '> getRunInfo.py > ' + STR_headDatasets
|
| 231 |
for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(STR_headDatasets)):
|
| 232 |
str_print += ' '
|
| 233 |
str_print += ' '
|
| 234 |
int_length = len(str_print)
|
| 235 |
print '%s%s' %(str_print,STR_headFiles)
|
| 236 |
str_print = ' '
|
| 237 |
for int_i in range(int_length-16+len(STR_headFiles)/2+INT_offset+8):
|
| 238 |
str_print += '-'
|
| 239 |
print str_print
|
| 240 |
for str_dbsDataset in Dict_dbsDatasets[str_dbsInstance]:
|
| 241 |
str_print = ' ' + str_dbsDataset
|
| 242 |
for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(str_dbsDataset)):
|
| 243 |
str_print += ' '
|
| 244 |
str_print += ' '
|
| 245 |
for int_i in range(len(STR_headFiles)/2-len(DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset])):
|
| 246 |
str_print += ' '
|
| 247 |
str_print += DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset] + ' ('
|
| 248 |
for int_i in range(INT_offset-len(DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])):
|
| 249 |
str_print += ' '
|
| 250 |
print '%s%s events)' %(str_print,DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])
|
| 251 |
print
|
