| 1 |
mangano |
1.1 |
#!/usr/bin/env python
|
| 2 |
|
|
|
| 3 |
|
|
# LM: version date: 01/02/2010 --> fixed dataset search and added json output file (optional)
|
| 4 |
|
|
# LM: updated 03/04/2010 --> adapted to new runreg api (and dcs status info)
|
| 5 |
|
|
|
| 6 |
|
|
# include XML-RPC client library
|
| 7 |
|
|
# RR API uses XML-RPC webservices interface for data access
|
| 8 |
|
|
import xmlrpclib,sys,ConfigParser,os,string,commands,time,re
|
| 9 |
|
|
# for json support
|
| 10 |
|
|
try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
|
| 11 |
|
|
import json
|
| 12 |
|
|
except:
|
| 13 |
|
|
try:
|
| 14 |
|
|
import simplejson as json
|
| 15 |
|
|
except:
|
| 16 |
|
|
print "Please set a crab environment in order to get the proper JSON lib"
|
| 17 |
|
|
sys.exit(1)
|
| 18 |
|
|
|
| 19 |
|
|
global QF_Req,ls_temp_data,QF_ALL_SYS,EXCEPTION,EXRUN
|
| 20 |
|
|
EXCEPTION=False
|
| 21 |
|
|
EXRUN=-1
|
| 22 |
|
|
|
| 23 |
|
|
def invert_intervals(intervals,min_val=1,max_val=9999):
|
| 24 |
|
|
# first order and merge in case
|
| 25 |
|
|
if not intervals:
|
| 26 |
|
|
return []
|
| 27 |
|
|
intervals=merge_intervals(intervals)
|
| 28 |
|
|
intervals = sorted(intervals, key = lambda x: x[0])
|
| 29 |
|
|
result = []
|
| 30 |
|
|
if min_val==-1:
|
| 31 |
|
|
# defin min and max
|
| 32 |
|
|
(a,b)=intervals[0]
|
| 33 |
|
|
min_val=a
|
| 34 |
|
|
if max_val==-1:
|
| 35 |
|
|
(a,b)=intervals[len(intervals)-1]
|
| 36 |
|
|
max_val=b
|
| 37 |
|
|
|
| 38 |
|
|
curr_min=min_val
|
| 39 |
|
|
for (x,y) in intervals:
|
| 40 |
|
|
if x>curr_min:
|
| 41 |
|
|
result.append((curr_min,x-1))
|
| 42 |
|
|
curr_min=y+1
|
| 43 |
|
|
if curr_min<max_val:
|
| 44 |
|
|
result.append((curr_min,max_val))
|
| 45 |
|
|
|
| 46 |
|
|
# print min_val,max_val
|
| 47 |
|
|
return result
|
| 48 |
|
|
|
| 49 |
|
|
def merge_intervals(intervals):
|
| 50 |
|
|
if not intervals:
|
| 51 |
|
|
return []
|
| 52 |
|
|
intervals = sorted(intervals, key = lambda x: x[0])
|
| 53 |
|
|
result = []
|
| 54 |
|
|
(a, b) = intervals[0]
|
| 55 |
|
|
for (x, y) in intervals[1:]:
|
| 56 |
|
|
if x <= b:
|
| 57 |
|
|
b = max(b, y)
|
| 58 |
|
|
else:
|
| 59 |
|
|
result.append((a, b))
|
| 60 |
|
|
(a, b) = (x, y)
|
| 61 |
|
|
result.append((a, b))
|
| 62 |
|
|
return result
|
| 63 |
|
|
|
| 64 |
|
|
def remove_html_tags(data):
|
| 65 |
|
|
p = re.compile(r'<.*?>')
|
| 66 |
|
|
newdata=p.sub('', data)
|
| 67 |
|
|
newdata=newdata.replace(" ","")
|
| 68 |
|
|
return newdata
|
| 69 |
|
|
|
| 70 |
|
|
def remove_extra_spaces(data):
|
| 71 |
|
|
result= re.sub(r'\s', '', data)
|
| 72 |
|
|
return result
|
| 73 |
|
|
|
| 74 |
|
|
def searchrun(runno):
|
| 75 |
|
|
global QF_Req,ls_temp_data,QF_ALL_SYS,EXCEPTION,EXRUN
|
| 76 |
|
|
intervallist=[]
|
| 77 |
|
|
selectls=""
|
| 78 |
|
|
|
| 79 |
|
|
for line in ls_temp_data.split("\n"):
|
| 80 |
|
|
if runno in line:
|
| 81 |
|
|
# print line
|
| 82 |
|
|
try:
|
| 83 |
|
|
if "%%%BAD LS INFO BEGIN%%%" in line:
|
| 84 |
|
|
selectls=line.split("%%%BAD LS INFO BEGIN%%%")[1]
|
| 85 |
|
|
selectls=selectls.split("%%%BAD LS INFO END%%%")[0]
|
| 86 |
|
|
selectls=remove_html_tags(selectls)
|
| 87 |
|
|
selectls=remove_extra_spaces(selectls)
|
| 88 |
|
|
# print selectls
|
| 89 |
|
|
for tag in QF_ALL_SYS:
|
| 90 |
|
|
selectls=selectls.replace(tag+":","\n"+tag+":")
|
| 91 |
|
|
# print selectls
|
| 92 |
|
|
|
| 93 |
|
|
for line in selectls.split("\n"):
|
| 94 |
|
|
try:
|
| 95 |
|
|
tag=line.split(":")[0]
|
| 96 |
|
|
intervals=line.split(":")[1]
|
| 97 |
|
|
except:
|
| 98 |
|
|
continue
|
| 99 |
|
|
if tag in QF_Req.keys():
|
| 100 |
|
|
if QF_Req[tag]=="GOOD":
|
| 101 |
|
|
for interval in intervals.split(","):
|
| 102 |
|
|
if "ALL" in interval:
|
| 103 |
|
|
lmin=1
|
| 104 |
|
|
lmax=9999
|
| 105 |
|
|
else:
|
| 106 |
|
|
strmin=interval.split('-')[0]
|
| 107 |
|
|
strmax=interval.split('-')[1]
|
| 108 |
|
|
lmin=int(strmin)
|
| 109 |
|
|
if "END" in strmax:
|
| 110 |
|
|
lmax=9999
|
| 111 |
|
|
else:
|
| 112 |
|
|
lmax=int(strmax)
|
| 113 |
|
|
intervallist.append((lmin,lmax))
|
| 114 |
|
|
except:
|
| 115 |
|
|
EXCEPTION=True
|
| 116 |
|
|
EXRUN=int(runno)
|
| 117 |
|
|
intervallist=merge_intervals(intervallist)
|
| 118 |
|
|
# print runno, intervallist
|
| 119 |
|
|
return intervallist
|
| 120 |
|
|
|
| 121 |
|
|
|
| 122 |
|
|
|
| 123 |
|
|
#main starts here#
|
| 124 |
|
|
|
| 125 |
|
|
QF_Req={}
|
| 126 |
|
|
GOODRUN={}
|
| 127 |
|
|
compactList = {}
|
| 128 |
|
|
|
| 129 |
|
|
QF_ALL_SYS=["Hcal","Track","Strip","Egam","Es","Dt","Csc","Pix","Muon","Rpc","Castor","Jmet","Ecal","L1t","Hlt","NONE"]
|
| 130 |
|
|
QF_ALL_STAT=["GOOD","BAD","EXCL","NONE"]
|
| 131 |
|
|
DCS_ALL=['Bpix','Fpix','Tibtid','TecM','TecP','Tob','Ebminus','Ebplus','EeMinus','EePlus','EsMinus','EsPlus','HbheA','HbheB','HbheC','H0','Hf','Dtminus','Dtplus','Dt0','CscMinus','CscPlus','Rpc','Castor',"NONE"]
|
| 132 |
|
|
|
| 133 |
|
|
# reading config file
|
| 134 |
|
|
CONFIGFILE='runreg.cfg'
|
| 135 |
|
|
CONFIG = ConfigParser.ConfigParser()
|
| 136 |
|
|
print 'Reading configuration file from ',CONFIGFILE
|
| 137 |
|
|
CONFIG.read(CONFIGFILE)
|
| 138 |
|
|
|
| 139 |
|
|
DATASET=CONFIG.get('Common','Dataset')
|
| 140 |
|
|
GROUP=CONFIG.get('Common','Group')
|
| 141 |
|
|
ADDRESS=CONFIG.get('Common','RunReg')
|
| 142 |
|
|
RUNMIN=CONFIG.get('Common','Runmin')
|
| 143 |
|
|
RUNMAX=CONFIG.get('Common','Runmax')
|
| 144 |
|
|
QFLAGS=CONFIG.get('Common','QFLAGS')
|
| 145 |
|
|
LSPARSE=CONFIG.get('Common','LSCOMMENT')
|
| 146 |
|
|
DCSSTAT=CONFIG.get('Common','DCS')
|
| 147 |
|
|
DCSLIST=string.split(DCSSTAT,',')
|
| 148 |
|
|
|
| 149 |
|
|
LSCOMMENT=True
|
| 150 |
|
|
if "TRUE" in LSPARSE.upper() or "1" in LSPARSE.upper() or "YES" in LSPARSE.upper():
|
| 151 |
|
|
LSCOMMENT=True
|
| 152 |
|
|
elif "FALSE" in LSPARSE.upper() or "0" in LSPARSE.upper() or "NO" in LSPARSE.upper():
|
| 153 |
|
|
LSCOMMENT=False
|
| 154 |
|
|
else:
|
| 155 |
|
|
print "Error in parsing LSCOMMENT cfg parameter: LSPARSE"
|
| 156 |
|
|
sys.exit(1)
|
| 157 |
|
|
|
| 158 |
|
|
QFlist=string.split(QFLAGS,',')
|
| 159 |
|
|
for QF in QFlist:
|
| 160 |
|
|
syst=string.split(QF,":")[0]
|
| 161 |
|
|
value=string.split(QF,":")[1]
|
| 162 |
|
|
if syst not in QF_ALL_SYS or value not in QF_ALL_STAT:
|
| 163 |
|
|
print "QFLAG not valid:",syst,value
|
| 164 |
|
|
sys.exit(1)
|
| 165 |
|
|
QF_Req[syst]=value
|
| 166 |
|
|
|
| 167 |
|
|
for dcs in DCSLIST:
|
| 168 |
|
|
if dcs not in DCS_ALL:
|
| 169 |
|
|
print "DCS not valid:",dcs
|
| 170 |
|
|
sys.exit(1)
|
| 171 |
|
|
|
| 172 |
|
|
|
| 173 |
|
|
CFGLIST=CONFIG.items('Common')
|
| 174 |
|
|
JSONFILE=CONFIG.get('Common','JSONFILE')
|
| 175 |
|
|
|
| 176 |
|
|
# report the request
|
| 177 |
|
|
|
| 178 |
|
|
print "You asked for the runreg info in the run range:"+RUNMIN+"-"+RUNMAX
|
| 179 |
|
|
print "for dataset: "+DATASET
|
| 180 |
|
|
print "with the following quality flags:"
|
| 181 |
|
|
for SS in QF_Req.keys():
|
| 182 |
|
|
print SS, QF_Req[SS]
|
| 183 |
|
|
print "and with the following DCS status:"
|
| 184 |
|
|
for dcs in DCSLIST:
|
| 185 |
|
|
print dcs
|
| 186 |
|
|
print "Manual bad LS in comment column:",LSCOMMENT
|
| 187 |
|
|
#sys.exit(1)
|
| 188 |
|
|
|
| 189 |
|
|
# get handler to RR XML-RPC server
|
| 190 |
|
|
FULLADDRESS=ADDRESS+"/xmlrpc"
|
| 191 |
|
|
print "RunRegistry from: ",FULLADDRESS
|
| 192 |
|
|
server = xmlrpclib.ServerProxy(FULLADDRESS)
|
| 193 |
|
|
|
| 194 |
|
|
# build up selection in RUN table
|
| 195 |
|
|
sel_runtable="{groupName} ='"+GROUP+"' and {runNumber} >= "+RUNMIN+" and {runNumber} <= "+RUNMAX+" and {datasetName} LIKE '"+DATASET+"'"
|
| 196 |
|
|
|
| 197 |
|
|
# the lumisection selection is on the Express dataset:
|
| 198 |
|
|
sel_dstable="{groupName} ='"+GROUP+"' and {runNumber} >= "+RUNMIN+" and {runNumber} <= "+RUNMAX+" and {datasetName} LIKE '%Express%'"
|
| 199 |
|
|
|
| 200 |
|
|
for key in QF_Req.keys():
|
| 201 |
|
|
if key != "NONE" and QF_Req[key]!="NONE":
|
| 202 |
|
|
sel_runtable+=" and {cmp"+key+"} = '"+QF_Req[key]+"'"
|
| 203 |
|
|
sel_dstable+=" and {cmp"+key+"} = '"+QF_Req[key]+"'"
|
| 204 |
|
|
#print sel_runtable
|
| 205 |
|
|
|
| 206 |
|
|
# build up selection in RUNLUMISECTION table
|
| 207 |
|
|
sel_dcstable="{groupName} ='"+GROUP+"' and {runNumber} >= "+RUNMIN+" and {runNumber} <= "+RUNMAX
|
| 208 |
|
|
for dcs in DCSLIST:
|
| 209 |
|
|
if dcs !="NONE":
|
| 210 |
|
|
sel_dcstable+=" and {parDcs"+dcs+"} = 1"
|
| 211 |
|
|
# = 'True'"
|
| 212 |
|
|
# print sel_dcstable
|
| 213 |
|
|
|
| 214 |
|
|
Tries=0
|
| 215 |
|
|
print " "
|
| 216 |
|
|
while Tries<10:
|
| 217 |
|
|
try:
|
| 218 |
|
|
print "Accessing run registry...."
|
| 219 |
|
|
dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json', sel_dcstable)
|
| 220 |
|
|
run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
|
| 221 |
|
|
ls_temp_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_datasets', sel_dstable)
|
| 222 |
|
|
break
|
| 223 |
|
|
except:
|
| 224 |
|
|
print "Something wrong in accessing runregistry, retrying in 3s...."
|
| 225 |
|
|
Tries=Tries+1
|
| 226 |
|
|
time.sleep(3)
|
| 227 |
|
|
if Tries==10:
|
| 228 |
|
|
print "Run registry unaccessible.....exiting now"
|
| 229 |
|
|
sys.exit(1)
|
| 230 |
|
|
|
| 231 |
|
|
#print dcs_data
|
| 232 |
|
|
#print run_data
|
| 233 |
|
|
#print ls_temp_data
|
| 234 |
|
|
# find LS info in comment
|
| 235 |
|
|
|
| 236 |
|
|
|
| 237 |
|
|
|
| 238 |
|
|
LISTOFRUN=[]
|
| 239 |
|
|
for line in run_data.split("\n"):
|
| 240 |
|
|
run=line.split(',')[0]
|
| 241 |
|
|
if run.isdigit():
|
| 242 |
|
|
LISTOFRUN.append(run)
|
| 243 |
|
|
|
| 244 |
|
|
|
| 245 |
|
|
selected_dcs={}
|
| 246 |
|
|
jsonlist=json.loads(dcs_data)
|
| 247 |
|
|
|
| 248 |
|
|
|
| 249 |
|
|
for element in jsonlist:
|
| 250 |
|
|
if element in LISTOFRUN:
|
| 251 |
|
|
# first search manual ls certification
|
| 252 |
|
|
if LSCOMMENT:
|
| 253 |
|
|
# using LS intervals in comment
|
| 254 |
|
|
manualbad_int=searchrun(element)
|
| 255 |
|
|
# make a badlumi list
|
| 256 |
|
|
dcsbad_int=invert_intervals(jsonlist[element])
|
| 257 |
|
|
combined=[]
|
| 258 |
|
|
for interval in manualbad_int:
|
| 259 |
|
|
combined.append(interval)
|
| 260 |
|
|
for interval in dcsbad_int:
|
| 261 |
|
|
combined.append(interval)
|
| 262 |
|
|
combined=merge_intervals(combined)
|
| 263 |
|
|
combined=invert_intervals(combined)
|
| 264 |
|
|
selected_dcs[element]=combined
|
| 265 |
|
|
else:
|
| 266 |
|
|
# using only DCS info
|
| 267 |
|
|
selected_dcs[element]=jsonlist[element]
|
| 268 |
|
|
# combined include bith manual LS and DCS LS
|
| 269 |
|
|
|
| 270 |
|
|
#JSONOUT=json.dumps(selected_dcs)
|
| 271 |
|
|
# WARNING: Don't use selected_dcs before dumping into file, it gets screwed up (don't know why!!)
|
| 272 |
|
|
if JSONFILE != "NONE":
|
| 273 |
|
|
lumiSummary = open(JSONFILE, 'w')
|
| 274 |
|
|
json.dump(selected_dcs, lumiSummary)
|
| 275 |
|
|
lumiSummary.close()
|
| 276 |
|
|
print " "
|
| 277 |
|
|
print "-------------------------------------------"
|
| 278 |
|
|
print "Json file: ",JSONFILE," written."
|
| 279 |
|
|
|
| 280 |
|
|
|
| 281 |
|
|
# buildup cms snippet
|
| 282 |
|
|
selectlumi="process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange(\n"
|
| 283 |
|
|
ranges = []
|
| 284 |
|
|
runs_to_print = selected_dcs.keys()
|
| 285 |
|
|
runs_to_print.sort()
|
| 286 |
|
|
for run in runs_to_print:
|
| 287 |
|
|
blocks = selected_dcs[run]
|
| 288 |
|
|
blocks.sort()
|
| 289 |
|
|
prevblock = [-2,-2]
|
| 290 |
|
|
for lsrange in blocks:
|
| 291 |
|
|
if lsrange[0] == prevblock[1]+1:
|
| 292 |
|
|
print "Run: ",run,"- This lumi starts at ", lsrange[0], " previous ended at ", prevblock[1]+1, " so I should merge"
|
| 293 |
|
|
prevblock[1] = lsrange[1]
|
| 294 |
|
|
ranges[-1] = "\t'%s:%d-%s:%d',\n" % (run, prevblock[0],
|
| 295 |
|
|
run, prevblock[1])
|
| 296 |
|
|
else:
|
| 297 |
|
|
ranges.append("\t'%s:%d-%s:%d',\n" % (run, lsrange[0],
|
| 298 |
|
|
run, lsrange[1]))
|
| 299 |
|
|
prevblock = lsrange
|
| 300 |
|
|
selectlumi += "".join(ranges)
|
| 301 |
|
|
selectlumi += ")"
|
| 302 |
|
|
|
| 303 |
|
|
|
| 304 |
|
|
print "-------------------------------------------"
|
| 305 |
|
|
print " "
|
| 306 |
|
|
print "CFG snippet to select:"
|
| 307 |
|
|
print selectlumi
|
| 308 |
|
|
|
| 309 |
|
|
if EXCEPTION:
|
| 310 |
|
|
print "WARNING: Something wrong in manual lumisection selection tag for run: "+str(EXRUN)
|
