| 1 |
#! /usr/bin/env python
|
| 2 |
|
| 3 |
from IMProv.IMProvQuery import IMProvQuery
|
| 4 |
from IMProv.IMProvLoader import loadIMProvFile
|
| 5 |
|
| 6 |
# script to parse Crab XML report obtained with crab -status
|
| 7 |
#
|
| 8 |
# needs to be run in crab_..../share directory after
|
| 9 |
# crab -status -USER.xml_report=RReport.xml
|
| 10 |
#
|
| 11 |
# status codes are explained in
|
| 12 |
# CRAB_2_2_X/ProdCommon/ProdCommon/BossLite/Scheduler/GLiteLBQuery.py
|
| 13 |
# i.e.
|
| 14 |
#
|
| 15 |
# statusMap = {
|
| 16 |
# 'Undefined':'UN',
|
| 17 |
# 'Submitted':'SU',
|
| 18 |
# 'Waiting':'SW',
|
| 19 |
# 'Ready':'SR',
|
| 20 |
# 'Scheduled':'SS',
|
| 21 |
# 'Running':'R',
|
| 22 |
# 'Done':'SD',
|
| 23 |
# 'Cleared':'E',
|
| 24 |
# 'Aborted':'A',
|
| 25 |
# 'Cancelled':'K',
|
| 26 |
# 'Unknown':'UN',
|
| 27 |
# 'Done(failed)':'DA'
|
| 28 |
# }
|
| 29 |
# more status code come from CrabServer and are here
|
| 30 |
# https://twiki.cern.ch/twiki/bin/view/CMS/TaskTracking#Notes
|
| 31 |
#
|
| 32 |
|
| 33 |
# map from new (left) to old (right) code
|
| 34 |
statusMap = {
|
| 35 |
'Undefined':'U',
|
| 36 |
'Created':'C',
|
| 37 |
'Submitting':'B',
|
| 38 |
'Submitted':'B',
|
| 39 |
'Waiting':'S',
|
| 40 |
'Ready':'S',
|
| 41 |
'Scheduled':'S',
|
| 42 |
'Running':'R',
|
| 43 |
'Done':'D',
|
| 44 |
'Done (Failed)':'D',
|
| 45 |
'Cleared':'Y',
|
| 46 |
'Retrieved':'Y',
|
| 47 |
'Killing':'K',
|
| 48 |
'Killed':'K',
|
| 49 |
'CannotSubmit':'A',
|
| 50 |
'Aborted':'A',
|
| 51 |
'NotSubmitted':'A',
|
| 52 |
'Cancelled':'K',
|
| 53 |
'Cancelled by user':'K',
|
| 54 |
'Unknown':'U',
|
| 55 |
'Done(failed)':'D'
|
| 56 |
}
|
| 57 |
|
| 58 |
def queryStatusXML(filename):
|
| 59 |
|
| 60 |
try:
|
| 61 |
report = loadIMProvFile(filename)
|
| 62 |
except StandardError, ex:
|
| 63 |
msg = "Error reading CRAB Status Report: %s\n" % filename
|
| 64 |
msg += str(ex)
|
| 65 |
raise RuntimeError, msg
|
| 66 |
|
| 67 |
query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
|
| 68 |
Jobs = query(report)
|
| 69 |
|
| 70 |
return Jobs
|
| 71 |
|
| 72 |
def printCrabStatusFromReport(filename):
|
| 73 |
Jobs = queryStatusXML(filename)
|
| 74 |
print "Crab Id: StatusScheduler | Status | ProcessStatus | State | GridId |"
|
| 75 |
for j in Jobs:
|
| 76 |
crabId = int(j.attrs.get("jobId",None))
|
| 77 |
statusScheduler = str(j.attrs.get("statusScheduler",None))
|
| 78 |
status = str(j.attrs.get("status",None))
|
| 79 |
processStatus = str(j.attrs.get("processStatus",None))
|
| 80 |
state = str(j.attrs.get("state",None))
|
| 81 |
gridId = str(j.attrs.get("schedulerId",None))
|
| 82 |
|
| 83 |
# print crabId, processStatus, statusScheduler, status, state, gridId
|
| 84 |
print "%d : %s | %s | %s | %s | %s " % (crabId,statusScheduler,status,processStatus,state,gridId)
|
| 85 |
|
| 86 |
"""
|
| 87 |
# remap into old status codes from BOSS for use in JobRobot
|
| 88 |
if state == 'SubRequested' : status = 'Submitting'
|
| 89 |
if state == 'Terminated' : status = 'Done'
|
| 90 |
ost = statusMap[statusScheduler]
|
| 91 |
# old bossId starts from 0
|
| 92 |
bossId = crabId-1
|
| 93 |
print ("%d|%1s||%s|%d|") % (bossId, ost, gridId, crabId)
|
| 94 |
"""
|
| 95 |
|
| 96 |
def crabStatusFromReport(filename):
|
| 97 |
Jobs = queryStatusXML(filename)
|
| 98 |
#statusField = "statusScheduler"
|
| 99 |
#statusField = "state"
|
| 100 |
statusField = "status"
|
| 101 |
summary = {}
|
| 102 |
nJobs = 0
|
| 103 |
for j in Jobs:
|
| 104 |
jobStatus = str(j.attrs.get(statusField,None))
|
| 105 |
if summary.has_key(jobStatus): summary[jobStatus] += 1
|
| 106 |
else: summary[jobStatus] = 1
|
| 107 |
nJobs += 1
|
| 108 |
|
| 109 |
for item in summary: summary[item] = 100.*summary[item]/nJobs
|
| 110 |
|
| 111 |
return summary
|
| 112 |
|
| 113 |
if __name__ == '__main__':
|
| 114 |
|
| 115 |
import optparse,sys
|
| 116 |
usage_str = "Usage: %prog [options] arg\n"
|
| 117 |
usage_str += "Example: crabStatusFromReport -f XMLREPORT\n"
|
| 118 |
usage_str += " crabStatusFromReport -f XMLREPORT all"
|
| 119 |
parser = optparse.OptionParser(usage=usage_str)
|
| 120 |
parser.add_option("-f","-r","--file","--report", dest="xmlreport", metavar="XMLREPORT", help="extract info from XMLREPORT")
|
| 121 |
|
| 122 |
(input, args) = parser.parse_args()
|
| 123 |
|
| 124 |
if not input.xmlreport: parser.error('must set XML report file name')
|
| 125 |
print "Using",input.xmlreport
|
| 126 |
|
| 127 |
if 'all' in args:
|
| 128 |
printCrabStatusFromReport(input.xmlreport)
|
| 129 |
else:
|
| 130 |
summary = crabStatusFromReport(input.xmlreport)
|
| 131 |
for key in summary:
|
| 132 |
print "%s: %.0f%%" % (key,summary[key])
|
| 133 |
|
| 134 |
sys.exit(0)
|
