CRAB/python/FjrParser.py

import os, commands, re
import xml.dom.minidom
from xml.dom.minidom import Node

# we use this function for popen calls so that we can control verbosity
def getstatusoutput (cmd):
    (stat, output) = commands.getstatusoutput(cmd)
    return stat, output

def get_fjrs (directory):
    cmd = '/bin/ls ' + directory + '/res/*.xml'
    (stat, fjrs) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:",  fjrs
        return []
    return fjrs.split('\n')

def get_nodes ():
    cmd = "wget --no-check-certificate -O- -q https://cmsweb.cern.ch/phedex/datasvc/xml/prod/nodes"
    (stat, nodes) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", nodes
        raise RuntimeError('command ' + cmd + ' execution error')
    return nodes

def parse_xml(file):
    return xml.dom.minidom.parse(file)

def parse_nodes():
  datasvc_nodes = get_nodes()
  return xml.dom.minidom.parseString(datasvc_nodes)

def is_goodfile (doc):
    ns = 0 
    for node in doc.getElementsByTagName("FrameworkJobReport"):
        key = node.attributes.keys()[0].encode('ascii')
        value = node.attributes[key].value
        if value == "Success":
            for node2 in doc.getElementsByTagName("FrameworkError"):
                exitStatus = node2.attributes["ExitStatus"].value
                type = node2.attributes["Type"].value
                if exitStatus == "0" and (type == "WrapperExitCode" or type == "ExeExitCode"):
                    ns = ns + 1
    if (ns > 1): return True
    return False

def has_local_stageout (doc):
    for node in doc.getElementsByTagName("FrameworkJobReport"):
        key =  node.attributes.keys()[0].encode('ascii')
        value = node.attributes[key].value
        if value == "Failed":
            for node2 in doc.getElementsByTagName("FrameworkError"):
                exitStatus = node2.attributes["ExitStatus"].value
                type = node2.attributes["Type"].value
                if exitStatus == "60308" and type == "WrapperExitCode":
                    node.attributes[key].value = "Success"
                    node2.attributes["ExitStatus"].value = "0"
                    return True
    return False

def get_filenames (doc):
    lfn = ""
    for node in doc.getElementsByTagName("LFN"):
        if node.parentNode.tagName == "AnalysisFile":
            lfn = node.attributes["Value"].value.strip()

    pfn = ""
    for node in doc.getElementsByTagName("PFN"):
        if node.parentNode.tagName == "AnalysisFile":
            pfn = node.attributes["Value"].value.strip()

    surl = ""
    for node in doc.getElementsByTagName("SurlForGrid"):
        if node.parentNode.tagName == "AnalysisFile":
            surl = node.attributes["Value"].value.strip()

    return (lfn, pfn, surl)

def local_stageout_filenames_from_datasvc (doc, nodes):
    # convert SEName into node
    seName = ""
    for node in doc.getElementsByTagName("SEName"):
        if node.parentNode.tagName == "File":
            seName = node.firstChild.nodeValue.strip()
    if seName == "":
        print ">>> could not find SEName in fjr, aborting retrieval"
        raise RuntimeError('Failed to find SE name in fjr')
    nodeName = ""
    for node in nodes.getElementsByTagName("node"):
        se = ""
        name = ""
        for key in node.attributes.keys():
            if key.encode("ascii") == "se":
                se = node.attributes[key].value
            if key.encode("ascii") == "name":
                name = node.attributes[key].value
        if se == seName:
            nodeName = name
            break
    if verbosity > 0:
        print ">>> local stageout nodeName =", nodeName
    lfn = ""
    for node in doc.getElementsByTagName("LFN"):
        if node.parentNode.tagName == "File":
            lfn = node.firstChild.nodeValue.strip()
    cmd = "wget --no-check-certificate -O- -q \"https://cmsweb.cern.ch/phedex/datasvc/xml/prod/lfn2pfn?node=" + nodeName + "&lfn=" + lfn + "&protocol=srmv2\""
    (stat, pfnXml) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", pfnXml
        raise RuntimeError('command ' + cmd + ' execution error')
    try:
        pfnDoc = xml.dom.minidom.parseString(pfnXml)
    except:
        print ">>> aborting retrieval, could not parse pfn xml for node/lfn:", nodeName, lfn
        raise RuntimeError('xml parsing error')
    pfn = ""
    for node in pfnDoc.getElementsByTagName("mapping"):
        for key in node.attributes.keys():
            if key.encode("ascii") == "pfn":
                pfn = node.attributes[key].value.encode("ascii")
    return lfn, pfn

def cp_target (directory):
    # this is a bit trickier; we need to parse CMSSW.sh to get $endpoint
    cmd = "grep 'export endpoint=' " + directory + "/job/CMSSW.sh"
    (stat, grep_output) = getstatusoutput(cmd)
    if stat != 0:
        print ">>> aborting retrieval, error:", grep_output
        raise RuntimeError('Command ' + cmd + ' execution error')
    return grep_output.replace("export endpoint=", "")

def cp_ui_target(directory):
    path =  os.getcwd() + '/' + directory + '/res/'
    endpoint = 'file:/' + path
    return path, endpoint

def rewrite_fjr (file, doc, quiet=True):
    if not quiet:
        print ">>> rewriting fjr to indicate remote stageout success"
    (bkup_path, bkup_file) = os.path.split(file)
    bkup_path += "/retry_backup"
    if not quiet:
        print ">>> backup path is", bkup_path
    try: 
        stat_result = os.stat(bkup_path)
    except OSError as err: 
        if err.errno == os.errno.ENOENT:
            if not quiet:
                print ">>> backup directory does not exist, creating ..."
            os.mkdir(bkup_path)
        else:
            raise RuntimeError('Error: ' + err.errno)
    bkup_file = os.path.join(bkup_path, bkup_file)
    if not quiet:
        print ">>> \told fjr will be backed up to", bkup_file
    (bkup_cp_output_stat, bkup_cp_output) = getstatusoutput("mv " + file + " " + bkup_file)
    if bkup_cp_output_stat != 0:
        print ">>> could not back up fjr, error:", bkup_cp_output, "(fjr not rewritten)"
        raise RuntimeError('failed to backup fjr')

    out = open(file, "w")
    doc.writexml(out)
Revision:	1.1
Committed:	Wed Feb 29 17:16:25 2012 UTC (13 years, 2 months ago) by fanzago
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, HEAD
Log Message:	A simple fjr parsing framework that is used by retry_stageout.py & find_goodfiles.py
#	User	Rev	Content
1	fanzago	1.1	import os, commands, re
2			import xml.dom.minidom
3			from xml.dom.minidom import Node
4
5			# we use this function for popen calls so that we can control verbosity
6			def getstatusoutput (cmd):
7			(stat, output) = commands.getstatusoutput(cmd)
8			return stat, output
9
10			def get_fjrs (directory):
11			cmd = '/bin/ls ' + directory + '/res/*.xml'
12			(stat, fjrs) = getstatusoutput(cmd)
13			if stat != 0:
14			print ">>> aborting retrieval, error:", fjrs
15			return []
16			return fjrs.split('\n')
17
18			def get_nodes ():
19			cmd = "wget --no-check-certificate -O- -q https://cmsweb.cern.ch/phedex/datasvc/xml/prod/nodes"
20			(stat, nodes) = getstatusoutput(cmd)
21			if stat != 0:
22			print ">>> aborting retrieval, error:", nodes
23			raise RuntimeError('command ' + cmd + ' execution error')
24			return nodes
25
26			def parse_xml(file):
27			return xml.dom.minidom.parse(file)
28
29			def parse_nodes():
30			datasvc_nodes = get_nodes()
31			return xml.dom.minidom.parseString(datasvc_nodes)
32
33			def is_goodfile (doc):
34			ns = 0
35			for node in doc.getElementsByTagName("FrameworkJobReport"):
36			key = node.attributes.keys()[0].encode('ascii')
37			value = node.attributes[key].value
38			if value == "Success":
39			for node2 in doc.getElementsByTagName("FrameworkError"):
40			exitStatus = node2.attributes["ExitStatus"].value
41			type = node2.attributes["Type"].value
42			if exitStatus == "0" and (type == "WrapperExitCode" or type == "ExeExitCode"):
43			ns = ns + 1
44			if (ns > 1): return True
45			return False
46
47			def has_local_stageout (doc):
48			for node in doc.getElementsByTagName("FrameworkJobReport"):
49			key = node.attributes.keys()[0].encode('ascii')
50			value = node.attributes[key].value
51			if value == "Failed":
52			for node2 in doc.getElementsByTagName("FrameworkError"):
53			exitStatus = node2.attributes["ExitStatus"].value
54			type = node2.attributes["Type"].value
55			if exitStatus == "60308" and type == "WrapperExitCode":
56			node.attributes[key].value = "Success"
57			node2.attributes["ExitStatus"].value = "0"
58			return True
59			return False
60
61			def get_filenames (doc):
62			lfn = ""
63			for node in doc.getElementsByTagName("LFN"):
64			if node.parentNode.tagName == "AnalysisFile":
65			lfn = node.attributes["Value"].value.strip()
66
67			pfn = ""
68			for node in doc.getElementsByTagName("PFN"):
69			if node.parentNode.tagName == "AnalysisFile":
70			pfn = node.attributes["Value"].value.strip()
71
72			surl = ""
73			for node in doc.getElementsByTagName("SurlForGrid"):
74			if node.parentNode.tagName == "AnalysisFile":
75			surl = node.attributes["Value"].value.strip()
76
77			return (lfn, pfn, surl)
78
79			def local_stageout_filenames_from_datasvc (doc, nodes):
80			# convert SEName into node
81			seName = ""
82			for node in doc.getElementsByTagName("SEName"):
83			if node.parentNode.tagName == "File":
84			seName = node.firstChild.nodeValue.strip()
85			if seName == "":
86			print ">>> could not find SEName in fjr, aborting retrieval"
87			raise RuntimeError('Failed to find SE name in fjr')
88			nodeName = ""
89			for node in nodes.getElementsByTagName("node"):
90			se = ""
91			name = ""
92			for key in node.attributes.keys():
93			if key.encode("ascii") == "se":
94			se = node.attributes[key].value
95			if key.encode("ascii") == "name":
96			name = node.attributes[key].value
97			if se == seName:
98			nodeName = name
99			break
100			if verbosity > 0:
101			print ">>> local stageout nodeName =", nodeName
102			lfn = ""
103			for node in doc.getElementsByTagName("LFN"):
104			if node.parentNode.tagName == "File":
105			lfn = node.firstChild.nodeValue.strip()
106			cmd = "wget --no-check-certificate -O- -q \"https://cmsweb.cern.ch/phedex/datasvc/xml/prod/lfn2pfn?node=" + nodeName + "&lfn=" + lfn + "&protocol=srmv2\""
107			(stat, pfnXml) = getstatusoutput(cmd)
108			if stat != 0:
109			print ">>> aborting retrieval, error:", pfnXml
110			raise RuntimeError('command ' + cmd + ' execution error')
111			try:
112			pfnDoc = xml.dom.minidom.parseString(pfnXml)
113			except:
114			print ">>> aborting retrieval, could not parse pfn xml for node/lfn:", nodeName, lfn
115			raise RuntimeError('xml parsing error')
116			pfn = ""
117			for node in pfnDoc.getElementsByTagName("mapping"):
118			for key in node.attributes.keys():
119			if key.encode("ascii") == "pfn":
120			pfn = node.attributes[key].value.encode("ascii")
121			return lfn, pfn
122
123			def cp_target (directory):
124			# this is a bit trickier; we need to parse CMSSW.sh to get $endpoint
125			cmd = "grep 'export endpoint=' " + directory + "/job/CMSSW.sh"
126			(stat, grep_output) = getstatusoutput(cmd)
127			if stat != 0:
128			print ">>> aborting retrieval, error:", grep_output
129			raise RuntimeError('Command ' + cmd + ' execution error')
130			return grep_output.replace("export endpoint=", "")
131
132			def cp_ui_target(directory):
133			path = os.getcwd() + '/' + directory + '/res/'
134			endpoint = 'file:/' + path
135			return path, endpoint
136
137			def rewrite_fjr (file, doc, quiet=True):
138			if not quiet:
139			print ">>> rewriting fjr to indicate remote stageout success"
140			(bkup_path, bkup_file) = os.path.split(file)
141			bkup_path += "/retry_backup"
142			if not quiet:
143			print ">>> backup path is", bkup_path
144			try:
145			stat_result = os.stat(bkup_path)
146			except OSError as err:
147			if err.errno == os.errno.ENOENT:
148			if not quiet:
149			print ">>> backup directory does not exist, creating ..."
150			os.mkdir(bkup_path)
151			else:
152			raise RuntimeError('Error: ' + err.errno)
153			bkup_file = os.path.join(bkup_path, bkup_file)
154			if not quiet:
155			print ">>> \told fjr will be backed up to", bkup_file
156			(bkup_cp_output_stat, bkup_cp_output) = getstatusoutput("mv " + file + " " + bkup_file)
157			if bkup_cp_output_stat != 0:
158			print ">>> could not back up fjr, error:", bkup_cp_output, "(fjr not rewritten)"
159			raise RuntimeError('failed to backup fjr')
160
161			out = open(file, "w")
162			doc.writexml(out)