CRAB/python/parseCrabFjr.py

#!/usr/bin/env python

import sys, getopt, string

from FwkJobRep.ReportParser import readJobReport
from DashboardAPI import apmonSend, apmonFree


def main(argv) :
    """
    parseCrabFjr

    - parse CRAB FrameworkJobReport on WN: { 'protocol' : { 'action' : [attempted,succedeed,total-size,total-time,min-time,max-time] , ... } , ... }
    - report parameters to DashBoard using DashBoardApi.py: for all 'read' actions of all protocols, report MBPS
    - return ExitStatus and dump of DashBoard report separated by semi-colon to WN wrapper script

    required parameters:
    --input            :       input FJR xml file
    --MonitorID        :       DashBoard MonitorID
    --MonitorJobID     :       DashBoard MonitorJobID

    optional parameters:
    --help             :       help
    --debug            :       debug statements
    
    """

    # defaults
    input = ''
    MonitorID = ''
    MonitorJobID = ''
    debug = 0

    try:
        opts, args = getopt.getopt(argv, "", ["input=", "MonitorID=", "MonitorJobID=", "debug", "help"])
    except getopt.GetoptError:
        print main.__doc__
        sys.exit(2)

    # check command line parameter
    for opt, arg in opts :
        if opt  == "--help" :
            print main.__doc__
            sys.exit()
        elif opt == "--input" :
            input = arg
        elif opt == "--MonitorID" :
            MonitorID = arg
        elif opt == "--MonitorJobID" :
            MonitorJobID = arg
        elif opt == "--debug" :
            debug = 1
            
    if input == '' or MonitorID == '' or MonitorJobID == '':
        print main.__doc__
        sys.exit()

    # load FwkJobRep
    jobReport = readJobReport(input)[0]

    exit_satus = ''
    
    # get ExitStatus of last error
    if len(jobReport.errors) != 0 :
        exit_status = str(jobReport.errors[-1]['ExitStatus'])
    else :
        exit_status = str(0)

    # get i/o statistics
    storageStatistics = str(jobReport.storageStatistics)

    # dashboard report dictionary
    dashboard_report = {}

    # check if storageStatistics is valid
    if storageStatistics.find('Storage statistics:') != -1 :
        # report form: { 'protocol' : { 'action' : [attempted,succedeed,total-size,total-time,min-time,max-time] , ... } , ... }
        report = {}
        data = storageStatistics.split('Storage statistics:')[1]
        data_fields = data.split(';')
        for data_field in data_fields:
            # parse: format protocol/action = attepted/succedeed/total-size/total-time/min-time/max-time
            key = data_field.split('=')[0].strip()
            item = data_field.split('=')[1].strip()
            protocol = str(key.split('/')[0].strip())
            action = str(key.split('/')[1].strip())
            item_array = item.split('/')
            attempted = str(item_array[0].strip())
            succeeded = str(item_array[1].strip())
            total_size = str(item_array[2].strip().split('MB')[0])
            total_time = str(item_array[3].strip().split('ms')[0])
            min_time = str(item_array[4].strip().split('ms')[0])
            max_time = str(item_array[5].strip().split('ms')[0])
            # add to report
            if protocol in report.keys() :
                if action in report[protocol].keys() :
                    print 'protocol/action:',protocol,'/',action,'listed twice in report, taking the first'
                else :
                    report[protocol][action] = [attempted,succeeded,total_size,total_time,min_time,max_time]
            else :
                report[protocol] = {'action' : [attempted,succeeded,total_size,total_time,min_time,max_time] }

        if debug :
            for protocol in report.keys() :
                print 'protocol:',protocol
                for action in report[protocol].keys() :
                    print 'action:',action,'measurement:',report[protocol][action]

        # extract information to be sent to DashBoard
        # per protocol and for action=read, calculate MBPS
        # dashboard key is io_action
        dashboard_report['MonitorID'] = MonitorID
        dashboard_report['MonitorJobID'] = MonitorJobID
        for protocol in report.keys() :
            if 'read' in report[protocol].keys() :
                try:
                    size = float(report[protocol]['read'][2])
                    time = float(report[protocol]['read'][3])
                    dashboard_report['io_'+protocol] = size/time*1000
                except:
                    pass

        if debug :
            print dashboard_report

        # send to DashBoard
        apmonSend(MonitorID, MonitorJobID, dashboard_report)
        apmonFree()

    # prepare exit string
    exit_string = str(exit_status)
    for key in dashboard_report.keys() :
        exit_string += ';' + str(key) + '=' + str(dashboard_report[key])

    return exit_string


if __name__ == '__main__' :
    exit_status = main(sys.argv[1:])
    # output for wrapper script
    print exit_status

    
Revision:	1.2
Committed:	Tue Oct 10 19:34:59 2006 UTC (18 years, 6 months ago) by gutsche
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_1_4_0_pre3
Changes since 1.1:	+85 -13 lines
Log Message:	added first parsing of i/o part in FJR and added DashBoard report functionality
#	User	Rev	Content
1	gutsche	1.1	#!/usr/bin/env python
2
3			import sys, getopt, string
4
5			from FwkJobRep.ReportParser import readJobReport
6	gutsche	1.2	from DashboardAPI import apmonSend, apmonFree
7
8	gutsche	1.1
9			def main(argv) :
10			"""
11			parseCrabFjr
12
13	gutsche	1.2	- parse CRAB FrameworkJobReport on WN: { 'protocol' : { 'action' : [attempted,succedeed,total-size,total-time,min-time,max-time] , ... } , ... }
14			- report parameters to DashBoard using DashBoardApi.py: for all 'read' actions of all protocols, report MBPS
15			- return ExitStatus and dump of DashBoard report separated by semi-colon to WN wrapper script
16	gutsche	1.1
17			required parameters:
18			--input : input FJR xml file
19	gutsche	1.2	--MonitorID : DashBoard MonitorID
20			--MonitorJobID : DashBoard MonitorJobID
21	gutsche	1.1
22			optional parameters:
23			--help : help
24			--debug : debug statements
25
26			"""
27
28			# defaults
29			input = ''
30	gutsche	1.2	MonitorID = ''
31			MonitorJobID = ''
32	gutsche	1.1	debug = 0
33
34			try:
35	gutsche	1.2	opts, args = getopt.getopt(argv, "", ["input=", "MonitorID=", "MonitorJobID=", "debug", "help"])
36	gutsche	1.1	except getopt.GetoptError:
37			print main.__doc__
38			sys.exit(2)
39
40			# check command line parameter
41			for opt, arg in opts :
42			if opt == "--help" :
43			print main.__doc__
44			sys.exit()
45			elif opt == "--input" :
46			input = arg
47	gutsche	1.2	elif opt == "--MonitorID" :
48			MonitorID = arg
49			elif opt == "--MonitorJobID" :
50			MonitorJobID = arg
51	gutsche	1.1	elif opt == "--debug" :
52			debug = 1
53
54	gutsche	1.2	if input == '' or MonitorID == '' or MonitorJobID == '':
55	gutsche	1.1	print main.__doc__
56			sys.exit()
57
58			# load FwkJobRep
59			jobReport = readJobReport(input)[0]
60
61	gutsche	1.2	exit_satus = ''
62	gutsche	1.1
63			# get ExitStatus of last error
64			if len(jobReport.errors) != 0 :
65	gutsche	1.2	exit_status = str(jobReport.errors[-1]['ExitStatus'])
66	gutsche	1.1	else :
67	gutsche	1.2	exit_status = str(0)
68	gutsche	1.1
69			# get i/o statistics
70			storageStatistics = str(jobReport.storageStatistics)
71
72	gutsche	1.2	# dashboard report dictionary
73			dashboard_report = {}
74
75			# check if storageStatistics is valid
76			if storageStatistics.find('Storage statistics:') != -1 :
77			# report form: { 'protocol' : { 'action' : [attempted,succedeed,total-size,total-time,min-time,max-time] , ... } , ... }
78			report = {}
79			data = storageStatistics.split('Storage statistics:')[1]
80			data_fields = data.split(';')
81			for data_field in data_fields:
82			# parse: format protocol/action = attepted/succedeed/total-size/total-time/min-time/max-time
83			key = data_field.split('=')[0].strip()
84			item = data_field.split('=')[1].strip()
85			protocol = str(key.split('/')[0].strip())
86			action = str(key.split('/')[1].strip())
87			item_array = item.split('/')
88			attempted = str(item_array[0].strip())
89			succeeded = str(item_array[1].strip())
90			total_size = str(item_array[2].strip().split('MB')[0])
91			total_time = str(item_array[3].strip().split('ms')[0])
92			min_time = str(item_array[4].strip().split('ms')[0])
93			max_time = str(item_array[5].strip().split('ms')[0])
94			# add to report
95			if protocol in report.keys() :
96			if action in report[protocol].keys() :
97			print 'protocol/action:',protocol,'/',action,'listed twice in report, taking the first'
98			else :
99			report[protocol][action] = [attempted,succeeded,total_size,total_time,min_time,max_time]
100			else :
101			report[protocol] = {'action' : [attempted,succeeded,total_size,total_time,min_time,max_time] }
102
103			if debug :
104			for protocol in report.keys() :
105			print 'protocol:',protocol
106			for action in report[protocol].keys() :
107			print 'action:',action,'measurement:',report[protocol][action]
108
109			# extract information to be sent to DashBoard
110			# per protocol and for action=read, calculate MBPS
111			# dashboard key is io_action
112			dashboard_report['MonitorID'] = MonitorID
113			dashboard_report['MonitorJobID'] = MonitorJobID
114			for protocol in report.keys() :
115			if 'read' in report[protocol].keys() :
116			try:
117			size = float(report[protocol]['read'][2])
118			time = float(report[protocol]['read'][3])
119			dashboard_report['io_'+protocol] = size/time*1000
120			except:
121			pass
122
123			if debug :
124			print dashboard_report
125
126			# send to DashBoard
127			apmonSend(MonitorID, MonitorJobID, dashboard_report)
128			apmonFree()
129
130			# prepare exit string
131			exit_string = str(exit_status)
132			for key in dashboard_report.keys() :
133			exit_string += ';' + str(key) + '=' + str(dashboard_report[key])
134
135			return exit_string
136	gutsche	1.1
137
138			if __name__ == '__main__' :
139	gutsche	1.2	exit_status = main(sys.argv[1:])
140			# output for wrapper script
141			print exit_status
142	gutsche	1.1
143