COMP/PRODREQUEST/injectRequest.py

#!/usr/bin/env python
from os import getenv
from sys import argv
from xml.sax import parse
from xml.sax.handler import ContentHandler
from os.path import abspath, exists
from optparse import OptionParser

#from Utilities.InjectionUtilities import _call, createPSetHash, createPythonConfig

###
###
###
import popen2
import sys

def createPSetHash(cfgFile):
    hashFile = "%s.hash" % cfgFile
    pop = popen2.Popen4("EdmConfigHash < %s > %s " % (cfgFile, hashFile))
    pop.wait()
    exitStatus = pop.poll()
    if exitStatus:
        msg = "Error creating PSet Hash file:\n"
        msg += pop.fromchild.read()
        raise RuntimeError, msg

    content = file(hashFile).read()
    return content.strip()

def createPythonConfig(cfgFile):
    pycfgFile = cfgFile.replace(".cfg", ".pycfg")
    pop = popen2.Popen4("EdmConfigToPython < %s > %s " % (cfgFile, pycfgFile))
    pop.wait()
    exitStatus = pop.poll()
    if exitStatus:
        msg = "Error creating Python cfg file:\n"
        msg += pop.fromchild.read()
        raise RuntimeError, msg

    pop = popen2.Popen4("%s %s" % (sys.executable, pycfgFile))
    pop.wait()
    exitStatus = pop.poll()
    if exitStatus:
        msg = "Error importing Python cfg file:\n"
        msg += pop.fromchild.read()
        raise RuntimeError, msg

    return pycfgFile

from cStringIO import StringIO
import urllib2
import gzip 

def _encode(args):
    boundary = '----------=_PRODREQUEST_CGI_BOUNDARY_=-----------'
    (body, crlf) = ('', '\r\n')
    for key, value in args.items():
        body += '--' + boundary + crlf
        body += ('Content-disposition: form-data; name="%s"' % key) + crlf
        body += crlf + value + crlf
    body += '--' + boundary + '--' + crlf + crlf
    return ('multipart/form-data; boundary=' + boundary, body)

def _marshall(args, request):
    (type, body) = _encode(args)
    request.add_header ('Content-type', type)
    request.add_header ('Content-length', str(len(body)))
    request.add_data (body)

def _call (url, args):
    try:
        #request = urllib2.Request (self._cgiUrl, urllib.urlencode(args))
        request = urllib2.Request (url)
        request.add_header ('Accept-encoding', 'gzip')
        _marshall (args, request)
        result = urllib2.build_opener ().open (request)
        data = result.read()
        if result.headers.get ('Content-encoding', '') == 'gzip':
            data = gzip.GzipFile (fileobj=StringIO(data)).read ()
        return data
    except:
        pass

###
###
###


class RelValContentHandler (ContentHandler):
    def __init__ (self, version, relval, directory, hostAndPort):
        ContentHandler.__init__ (self)
        self.events = None
        self.testStarted = False
        self.workflowName = None
        self.cfgFile = None
        self.version = version
        self.relval = relval
        self.directory = directory
        self.hostAndPort = hostAndPort

    def startElement (self, name, attributes):
        if name == "Test":
            self.events = attributes.getValueByQName ("Events")
            self.workflowName = attributes.getValueByQName ("Name")
            self.testStarted = True
    
    def characters (self, data):
        if self.testStarted:
            if data.strip (" \t\n") != "":
                self.cfgFile = data.strip ("\n\t ")

    def endElement (self, name):
        if name == "Test":
            print self.events
            print self.workflowName
            print self.cfgFile
            self.inject ()
            self.testStarted = False
        pass

    def inject (self):
        prodrequestUrl="http://%s/ProdRequest/createPayloadWithEntry" % self.hostAndPort
        cfgfile = "%s/%s" % (self.directory, self.cfgFile) 
        
        if not exists (cfgfile):
            print cfgfile + " not found."
            pass
        
        cfg = file (cfgfile).read ()
        pycfgfile = createPythonConfig(cfgfile)
        pycfg = file (pycfgfile).read ()
        psetHash = createPSetHash (cfgfile)

        cfgname = "%s - %s" % (self.relval, self.version)
        _call (prodrequestUrl,
            {"name": self.workflowName,
            "parameterset": cfg,
            "parsed_parameterset": pycfg,
            "pset_hash": psetHash,
            "input_dataset": self.workflowName,
            "output_dataset": "output_dataset",
            "workflowName": cfgname,
            "version": self.version})
        
        
def injectRequest (confFilename, hostAndPort):
    confFilename = abspath (confFilename)
    print confFilename
    directory = confFilename.rsplit ("/", 1)[0]
    relval = confFilename.rsplit ("/", 1)[1].rsplit (".", 1)[0]
    cmsswVersion = getenv ("CMSSW_VERSION")
    if cmsswVersion:
        print "CMSSW version is " + cmsswVersion
    handler = RelValContentHandler (version=cmsswVersion, relval=relval, directory=directory, hostAndPort=hostAndPort)
    parser = parse (confFilename, handler)

if __name__ == "__main__":
    confFilename = argv[1]
    hostAndPort = "localhost:8030"
    optionParser = OptionParser ()
    optionParser.add_option ("-u", "--url", dest="url", 
                             metavar="URL", default="localhost:8030",
                             help="PRODREQUEST url")
    optlist, args = optionParser.parse_args ()
    for i in args:
        injectRequest (i, optlist.url)
Revision:	1.4
Committed:	Tue Feb 20 14:45:11 2007 UTC (18 years, 2 months ago) by eulisse
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	V00-03-09, V00-03-08, V00-03-07, V00-03-06, V00-03-05
Changes since 1.3:	+1 -2 lines
Log Message:	- Adds ability to specify Location / Priority hint for the request. - Adds a list to the list of requests as requested by Alessandra / Peter.
#	User	Rev	Content
1	eulisse	1.1	#!/usr/bin/env python
2			from os import getenv
3			from sys import argv
4			from xml.sax import parse
5			from xml.sax.handler import ContentHandler
6			from os.path import abspath, exists
7	eulisse	1.2	from optparse import OptionParser
8
9	eulisse	1.1	#from Utilities.InjectionUtilities import _call, createPSetHash, createPythonConfig
10
11			###
12			###
13			###
14			import popen2
15			import sys
16
17			def createPSetHash(cfgFile):
18			hashFile = "%s.hash" % cfgFile
19			pop = popen2.Popen4("EdmConfigHash < %s > %s " % (cfgFile, hashFile))
20			pop.wait()
21			exitStatus = pop.poll()
22			if exitStatus:
23			msg = "Error creating PSet Hash file:\n"
24			msg += pop.fromchild.read()
25			raise RuntimeError, msg
26
27			content = file(hashFile).read()
28			return content.strip()
29
30			def createPythonConfig(cfgFile):
31			pycfgFile = cfgFile.replace(".cfg", ".pycfg")
32			pop = popen2.Popen4("EdmConfigToPython < %s > %s " % (cfgFile, pycfgFile))
33			pop.wait()
34			exitStatus = pop.poll()
35			if exitStatus:
36			msg = "Error creating Python cfg file:\n"
37			msg += pop.fromchild.read()
38			raise RuntimeError, msg
39
40			pop = popen2.Popen4("%s %s" % (sys.executable, pycfgFile))
41			pop.wait()
42			exitStatus = pop.poll()
43			if exitStatus:
44			msg = "Error importing Python cfg file:\n"
45			msg += pop.fromchild.read()
46			raise RuntimeError, msg
47
48			return pycfgFile
49
50			from cStringIO import StringIO
51			import urllib2
52			import gzip
53
54			def _encode(args):
55			boundary = '----------=_PRODREQUEST_CGI_BOUNDARY_=-----------'
56			(body, crlf) = ('', '\r\n')
57			for key, value in args.items():
58			body += '--' + boundary + crlf
59			body += ('Content-disposition: form-data; name="%s"' % key) + crlf
60			body += crlf + value + crlf
61			body += '--' + boundary + '--' + crlf + crlf
62			return ('multipart/form-data; boundary=' + boundary, body)
63
64			def _marshall(args, request):
65			(type, body) = _encode(args)
66			request.add_header ('Content-type', type)
67			request.add_header ('Content-length', str(len(body)))
68			request.add_data (body)
69
70			def _call (url, args):
71			try:
72			#request = urllib2.Request (self._cgiUrl, urllib.urlencode(args))
73			request = urllib2.Request (url)
74			request.add_header ('Accept-encoding', 'gzip')
75			_marshall (args, request)
76			result = urllib2.build_opener ().open (request)
77			data = result.read()
78			if result.headers.get ('Content-encoding', '') == 'gzip':
79			data = gzip.GzipFile (fileobj=StringIO(data)).read ()
80			return data
81			except:
82			pass
83
84			###
85			###
86			###
87
88
89			class RelValContentHandler (ContentHandler):
90	eulisse	1.2	def __init__ (self, version, relval, directory, hostAndPort):
91	eulisse	1.1	ContentHandler.__init__ (self)
92			self.events = None
93			self.testStarted = False
94			self.workflowName = None
95			self.cfgFile = None
96			self.version = version
97			self.relval = relval
98			self.directory = directory
99	eulisse	1.2	self.hostAndPort = hostAndPort
100	eulisse	1.1
101			def startElement (self, name, attributes):
102			if name == "Test":
103			self.events = attributes.getValueByQName ("Events")
104			self.workflowName = attributes.getValueByQName ("Name")
105			self.testStarted = True
106
107			def characters (self, data):
108			if self.testStarted:
109			if data.strip (" \t\n") != "":
110			self.cfgFile = data.strip ("\n\t ")
111
112			def endElement (self, name):
113			if name == "Test":
114			print self.events
115			print self.workflowName
116			print self.cfgFile
117			self.inject ()
118			self.testStarted = False
119			pass
120
121	eulisse	1.3	def inject (self):
122			prodrequestUrl="http://%s/ProdRequest/createPayloadWithEntry" % self.hostAndPort
123	eulisse	1.1	cfgfile = "%s/%s" % (self.directory, self.cfgFile)
124
125			if not exists (cfgfile):
126			print cfgfile + " not found."
127			pass
128
129			cfg = file (cfgfile).read ()
130			pycfgfile = createPythonConfig(cfgfile)
131			pycfg = file (pycfgfile).read ()
132			psetHash = createPSetHash (cfgfile)
133
134			cfgname = "%s - %s" % (self.relval, self.version)
135			_call (prodrequestUrl,
136			{"name": self.workflowName,
137			"parameterset": cfg,
138			"parsed_parameterset": pycfg,
139			"pset_hash": psetHash,
140			"input_dataset": self.workflowName,
141			"output_dataset": "output_dataset",
142			"workflowName": cfgname,
143			"version": self.version})
144
145
146	eulisse	1.2	def injectRequest (confFilename, hostAndPort):
147	eulisse	1.1	confFilename = abspath (confFilename)
148			print confFilename
149			directory = confFilename.rsplit ("/", 1)[0]
150			relval = confFilename.rsplit ("/", 1)[1].rsplit (".", 1)[0]
151	eulisse	1.4	cmsswVersion = getenv ("CMSSW_VERSION")
152	eulisse	1.1	if cmsswVersion:
153			print "CMSSW version is " + cmsswVersion
154	eulisse	1.2	handler = RelValContentHandler (version=cmsswVersion, relval=relval, directory=directory, hostAndPort=hostAndPort)
155	eulisse	1.1	parser = parse (confFilename, handler)
156
157			if __name__ == "__main__":
158			confFilename = argv[1]
159	eulisse	1.3	hostAndPort = "localhost:8030"
160	eulisse	1.2	optionParser = OptionParser ()
161			optionParser.add_option ("-u", "--url", dest="url",
162	eulisse	1.3	metavar="URL", default="localhost:8030",
163	eulisse	1.2	help="PRODREQUEST url")
164			optlist, args = optionParser.parse_args ()
165			for i in args:
166			injectRequest (i, optlist.url)