CRAB/python/FjrParser.py

import os, commands, re
import xml.dom.minidom
from xml.dom.minidom import Node

# we use this function for popen calls so that we can control verbosity
def getstatusoutput (cmd):
    (stat, output) = commands.getstatusoutput(cmd)
    return stat, output

def get_fjrs (directory):
    cmd = '/bin/ls ' + directory + '/res/*.xml'
    (stat, fjrs) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:",  fjrs
        return []
    return fjrs.split('\n')

def get_nodes ():
    cmd = "wget --no-check-certificate -O- -q https://cmsweb.cern.ch/phedex/datasvc/xml/prod/nodes"
    (stat, nodes) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", nodes
        raise RuntimeError('command ' + cmd + ' execution error')
    return nodes

def parse_xml(file):
    return xml.dom.minidom.parse(file)

def parse_nodes():
  datasvc_nodes = get_nodes()
  return xml.dom.minidom.parseString(datasvc_nodes)

def is_goodfile (doc):
    ns = 0 
    for node in doc.getElementsByTagName("FrameworkJobReport"):
        key = node.attributes.keys()[0].encode('ascii')
        value = node.attributes[key].value
        if value == "Success":
            for node2 in doc.getElementsByTagName("FrameworkError"):
                exitStatus = node2.attributes["ExitStatus"].value
                type = node2.attributes["Type"].value
                if exitStatus == "0" and (type == "WrapperExitCode" or type == "ExeExitCode"):
                    ns = ns + 1
    if (ns > 1): return True
    return False

def has_local_stageout (doc):
    for node in doc.getElementsByTagName("FrameworkJobReport"):
        key =  node.attributes.keys()[0].encode('ascii')
        value = node.attributes[key].value
        if value == "Failed":
            for node2 in doc.getElementsByTagName("FrameworkError"):
                exitStatus = node2.attributes["ExitStatus"].value
                type = node2.attributes["Type"].value
                if exitStatus == "60308" and type == "WrapperExitCode":
                    node.attributes[key].value = "Success"
                    node2.attributes["ExitStatus"].value = "0"
                    return True
    return False

def get_filenames (doc):
    lfn = ""
    for node in doc.getElementsByTagName("LFN"):
        if node.parentNode.tagName == "AnalysisFile":
            lfn = node.attributes["Value"].value.strip()

    pfn = ""
    for node in doc.getElementsByTagName("PFN"):
        if node.parentNode.tagName == "AnalysisFile":
            pfn = node.attributes["Value"].value.strip()

    surl = ""
    for node in doc.getElementsByTagName("SurlForGrid"):
        if node.parentNode.tagName == "AnalysisFile":
            surl = node.attributes["Value"].value.strip()

    return (lfn, pfn, surl)

def local_stageout_filenames_from_datasvc (doc, nodes):
    # convert SEName into node
    seName = ""
    for node in doc.getElementsByTagName("SEName"):
        if node.parentNode.tagName == "File":
            seName = node.firstChild.nodeValue.strip()
    if seName == "":
        print ">>> could not find SEName in fjr, aborting retrieval"
        raise RuntimeError('Failed to find SE name in fjr')
    nodeName = ""
    for node in nodes.getElementsByTagName("node"):
        se = ""
        name = ""
        for key in node.attributes.keys():
            if key.encode("ascii") == "se":
                se = node.attributes[key].value
            if key.encode("ascii") == "name":
                name = node.attributes[key].value
        if se == seName:
            nodeName = name
            break
    if verbosity > 0:
        print ">>> local stageout nodeName =", nodeName
    lfn = ""
    for node in doc.getElementsByTagName("LFN"):
        if node.parentNode.tagName == "File":
            lfn = node.firstChild.nodeValue.strip()
    cmd = "wget --no-check-certificate -O- -q \"https://cmsweb.cern.ch/phedex/datasvc/xml/prod/lfn2pfn?node=" + nodeName + "&lfn=" + lfn + "&protocol=srmv2\""
    (stat, pfnXml) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", pfnXml
        raise RuntimeError('command ' + cmd + ' execution error')
    try:
        pfnDoc = xml.dom.minidom.parseString(pfnXml)
    except:
        print ">>> aborting retrieval, could not parse pfn xml for node/lfn:", nodeName, lfn
        raise RuntimeError('xml parsing error')
    pfn = ""
    for node in pfnDoc.getElementsByTagName("mapping"):
        for key in node.attributes.keys():
            if key.encode("ascii") == "pfn":
                pfn = node.attributes[key].value.encode("ascii")
    return lfn, pfn

def cp_target (directory):
    # this is a bit trickier; we need to parse CMSSW.sh to get $endpoint
    cmd = "grep 'export endpoint=' " + directory + "/job/CMSSW.sh"
    (stat, grep_output) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", grep_output
        raise RuntimeError('Command ' + cmd + ' execution error')
    return grep_output.replace("export endpoint=", "")

def cp_ui_target(directory):
    path =  os.getcwd() + '/' + directory + '/res/'
    endpoint = 'file:/' + path
    return path, endpoint

def rewrite_fjr (file, doc, quiet=True):
    if not quiet:
        print ">>> rewriting fjr to indicate remote stageout success"
    (bkup_path, bkup_file) = os.path.split(file)
    bkup_path += "/retry_backup"
    if not quiet:
        print ">>> backup path is", bkup_path
    try: 
        stat_result = os.stat(bkup_path)
    except OSError as err: 
        if err.errno == os.errno.ENOENT:
            if not quiet:
                print ">>> backup directory does not exist, creating ..."
            os.mkdir(bkup_path)
        else:
            raise RuntimeError('Error: ' + err.errno)
    bkup_file = os.path.join(bkup_path, bkup_file)
    if not quiet:
        print ">>> \told fjr will be backed up to", bkup_file
    (bkup_cp_output_stat, bkup_cp_output) = getstatusoutput("mv " + file + " " + bkup_file)
    if bkup_cp_output_stat != 0:
        print ">>> could not back up fjr, error:", bkup_cp_output, "(fjr not rewritten)"
        raise RuntimeError('failed to backup fjr')

    out = open(file, "w")
    doc.writexml(out)
Revision:	1.1
Committed:	Wed Feb 29 17:16:25 2012 UTC (13 years, 2 months ago) by fanzago
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, HEAD
Log Message:	A simple fjr parsing framework that is used by retry_stageout.py & find_goodfiles.py
#	Content
1	import os, commands, re
2	import xml.dom.minidom
3	from xml.dom.minidom import Node
4
5	# we use this function for popen calls so that we can control verbosity
6	def getstatusoutput (cmd):
7	(stat, output) = commands.getstatusoutput(cmd)
8	return stat, output
9
10	def get_fjrs (directory):
11	cmd = '/bin/ls ' + directory + '/res/*.xml'
12	(stat, fjrs) = getstatusoutput(cmd)
13	if stat != 0:
14	print ">>> aborting retrieval, error:", fjrs
15	return []
16	return fjrs.split('\n')
17
18	def get_nodes ():
19	cmd = "wget --no-check-certificate -O- -q https://cmsweb.cern.ch/phedex/datasvc/xml/prod/nodes"
20	(stat, nodes) = getstatusoutput(cmd)
21	if stat != 0:
22	print ">>> aborting retrieval, error:", nodes
23	raise RuntimeError('command ' + cmd + ' execution error')
24	return nodes
25
26	def parse_xml(file):
27	return xml.dom.minidom.parse(file)
28
29	def parse_nodes():
30	datasvc_nodes = get_nodes()
31	return xml.dom.minidom.parseString(datasvc_nodes)
32
33	def is_goodfile (doc):
34	ns = 0
35	for node in doc.getElementsByTagName("FrameworkJobReport"):
36	key = node.attributes.keys()[0].encode('ascii')
37	value = node.attributes[key].value
38	if value == "Success":
39	for node2 in doc.getElementsByTagName("FrameworkError"):
40	exitStatus = node2.attributes["ExitStatus"].value
41	type = node2.attributes["Type"].value
42	if exitStatus == "0" and (type == "WrapperExitCode" or type == "ExeExitCode"):
43	ns = ns + 1
44	if (ns > 1): return True
45	return False
46
47	def has_local_stageout (doc):
48	for node in doc.getElementsByTagName("FrameworkJobReport"):
49	key = node.attributes.keys()[0].encode('ascii')
50	value = node.attributes[key].value
51	if value == "Failed":
52	for node2 in doc.getElementsByTagName("FrameworkError"):
53	exitStatus = node2.attributes["ExitStatus"].value
54	type = node2.attributes["Type"].value
55	if exitStatus == "60308" and type == "WrapperExitCode":
56	node.attributes[key].value = "Success"
57	node2.attributes["ExitStatus"].value = "0"
58	return True
59	return False
60
61	def get_filenames (doc):
62	lfn = ""
63	for node in doc.getElementsByTagName("LFN"):
64	if node.parentNode.tagName == "AnalysisFile":
65	lfn = node.attributes["Value"].value.strip()
66
67	pfn = ""
68	for node in doc.getElementsByTagName("PFN"):
69	if node.parentNode.tagName == "AnalysisFile":
70	pfn = node.attributes["Value"].value.strip()
71
72	surl = ""
73	for node in doc.getElementsByTagName("SurlForGrid"):
74	if node.parentNode.tagName == "AnalysisFile":
75	surl = node.attributes["Value"].value.strip()
76
77	return (lfn, pfn, surl)
78
79	def local_stageout_filenames_from_datasvc (doc, nodes):
80	# convert SEName into node
81	seName = ""
82	for node in doc.getElementsByTagName("SEName"):
83	if node.parentNode.tagName == "File":
84	seName = node.firstChild.nodeValue.strip()
85	if seName == "":
86	print ">>> could not find SEName in fjr, aborting retrieval"
87	raise RuntimeError('Failed to find SE name in fjr')
88	nodeName = ""
89	for node in nodes.getElementsByTagName("node"):
90	se = ""
91	name = ""
92	for key in node.attributes.keys():
93	if key.encode("ascii") == "se":
94	se = node.attributes[key].value
95	if key.encode("ascii") == "name":
96	name = node.attributes[key].value
97	if se == seName:
98	nodeName = name
99	break
100	if verbosity > 0:
101	print ">>> local stageout nodeName =", nodeName
102	lfn = ""
103	for node in doc.getElementsByTagName("LFN"):
104	if node.parentNode.tagName == "File":
105	lfn = node.firstChild.nodeValue.strip()
106	cmd = "wget --no-check-certificate -O- -q \"https://cmsweb.cern.ch/phedex/datasvc/xml/prod/lfn2pfn?node=" + nodeName + "&lfn=" + lfn + "&protocol=srmv2\""
107	(stat, pfnXml) = getstatusoutput(cmd)
108	if stat != 0:
109	print ">>> aborting retrieval, error:", pfnXml
110	raise RuntimeError('command ' + cmd + ' execution error')
111	try:
112	pfnDoc = xml.dom.minidom.parseString(pfnXml)
113	except:
114	print ">>> aborting retrieval, could not parse pfn xml for node/lfn:", nodeName, lfn
115	raise RuntimeError('xml parsing error')
116	pfn = ""
117	for node in pfnDoc.getElementsByTagName("mapping"):
118	for key in node.attributes.keys():
119	if key.encode("ascii") == "pfn":
120	pfn = node.attributes[key].value.encode("ascii")
121	return lfn, pfn
122
123	def cp_target (directory):
124	# this is a bit trickier; we need to parse CMSSW.sh to get $endpoint
125	cmd = "grep 'export endpoint=' " + directory + "/job/CMSSW.sh"
126	(stat, grep_output) = getstatusoutput(cmd)
127	if stat != 0:
128	print ">>> aborting retrieval, error:", grep_output
129	raise RuntimeError('Command ' + cmd + ' execution error')
130	return grep_output.replace("export endpoint=", "")
131
132	def cp_ui_target(directory):
133	path = os.getcwd() + '/' + directory + '/res/'
134	endpoint = 'file:/' + path
135	return path, endpoint
136
137	def rewrite_fjr (file, doc, quiet=True):
138	if not quiet:
139	print ">>> rewriting fjr to indicate remote stageout success"
140	(bkup_path, bkup_file) = os.path.split(file)
141	bkup_path += "/retry_backup"
142	if not quiet:
143	print ">>> backup path is", bkup_path
144	try:
145	stat_result = os.stat(bkup_path)
146	except OSError as err:
147	if err.errno == os.errno.ENOENT:
148	if not quiet:
149	print ">>> backup directory does not exist, creating ..."
150	os.mkdir(bkup_path)
151	else:
152	raise RuntimeError('Error: ' + err.errno)
153	bkup_file = os.path.join(bkup_path, bkup_file)
154	if not quiet:
155	print ">>> \told fjr will be backed up to", bkup_file
156	(bkup_cp_output_stat, bkup_cp_output) = getstatusoutput("mv " + file + " " + bkup_file)
157	if bkup_cp_output_stat != 0:
158	print ">>> could not back up fjr, error:", bkup_cp_output, "(fjr not rewritten)"
159	raise RuntimeError('failed to backup fjr')
160
161	out = open(file, "w")
162	doc.writexml(out)