CRAB/python/osg_bdii.py

#!/usr/bin/env python
import re, popen2, sys, copy

CR = '\r'
LF = '\n'
CRLF = CR+LF

def unwraplines(wrapped_list):
    r = re.compile('^ (.*)$')
    unwrapped_list = []
    for l in wrapped_list:
        m = r.match(l)
        if m:
            unwrapped_list[-1] += m.groups()[0]
        else:
            unwrapped_list.append(l.rstrip())

    return unwrapped_list


def runldapquery(filter, attribute, bdii):
    command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
    command += filter + ' ' + attribute
    pout,pin,perr = popen2.popen3(command)

    pout = pout.readlines()
    p = perr.readlines()

    pout = unwraplines(pout)
    if (p):
        for l in p: print l
        raise RuntimeError('ldapsearch call failed')

    return pout

def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
    se = '\'' + se + '\''
    pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)

#    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*)-cms')
    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')
    jm = []
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jm.count(item) == 0):
                jm.append(item)

    return jm


def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
    status = []
    jmlist = jm_from_se_bdii(se)

    for jm in jmlist:
        jm += "-cms"

        pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)

        r = re.compile('^GlueCEStateStatus: (.*)')
        for l in pout:
            m = r.match(l)
            if m:
                status.append(m.groups()[0])

    return status

def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
    pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)

    status = ''
    r = re.compile('^GlueCEStateStatus: (.*)')
    for l in pout:
        m = r.match(l)
        if m:
            status = m.groups()[0]

    return status

def concatoutput(pout):
    output = ''
    for l in pout:
        if l == '':
            output = output + LF
        output = output + l + LF

    return output

def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
    # Get the Jobmanager list
    jmlist = []

    query = " '(|"
    for se in selist:
        query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
    query = query + ")' "

    pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')

    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)


    query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(|"
    for l in jmlist:
        query += "(GlueCEInfoContactString=" + l + "-*)"

    query += "))' "

    pout = runldapquery(query, 'GlueCEInfoContactString', bdii)

    r = re.compile('^GlueCEInfoContactString: (.*:.*/jobmanager-.*)')
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)

    return jmlist

def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
    results_list = []
    r = re.compile('^GlueSiteDescription: (OSG.*)')
    s = re.compile('^GlueSiteUniqueID: (.*)')

    query = " '(|"
    for h in host_list:
        query = query + "(GlueSiteUniqueID=" + h + ")"
    query = query + ")' GlueSiteDescription"

    pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
    output = concatoutput(pout)

    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        osg = 0
        host = ""
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            n = s.match(detail)
            if m:
                osg = 1
            if n:
                host = n.groups()[0]
        if (osg == 1):
            results_list.append(host)

    return results_list

def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = "'(|"

    for h in host_list:
        query += "(GlueCEInfoContactString=" + h + ")"
    query += ")'"

    pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
    r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
    s = re.compile('GlueCEInfoContactString: (.*)')

    ClusterMap =  {}
    ClusterUniqueID = None
    CEInfoContact = None

    for l in pout:
        m = r.match(l)
        if m:
            ClusterUniqueID = m.groups()[0]
        m = s.match(l)
        if m:
            CEInfoContact = m.groups()[0]

        if (ClusterUniqueID and CEInfoContact):
            ClusterMap[ClusterUniqueID] = CEInfoContact
            ClusterUniqueID = None
            CEInfoContact = None

    query = "'(|"
    for c in ClusterMap.keys():
        query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
    query += ")'"

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                ClusterUniqueID = m2.groups()[0]
                host = ClusterMap[ClusterUniqueID]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = " '(|"
    for h in host_list:
        query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
    query = query + ")' "

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                host = m2.groups()[0]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jminfo_list = []
    host_list = []

    stat = re.compile('^GlueCEStateStatus: (.*)')
    host = re.compile('^GlueCEInfoHostName: (.*)')
    wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
    name = re.compile('^GlueCEUniqueID: (.*)')

    jmlist = getJMListFromSEList(selist)

    query = " '(&(objectClass=GlueCEState)(|"
    for jm in jmlist:
        query = query + "(GlueCEUniqueID=" + jm + ")"
    query = query + "))' "

    pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
    output = concatoutput(pout)

    stanza_list = output.split(LF+LF)
    for stanza in stanza_list:
        if len(stanza) > 1:
            status = 1
            wait_jobs = 0
            jmname = ''
            hostname = 0
            jminfo = {}

            details = stanza.split(LF)
            for det in details:
                mhost = host.match(det)
                if mhost: # hostname
                    host_list.append(mhost.groups()[0])
                    hostname = mhost.groups()[0]
                mstat = stat.match(det)
                if mstat: # e.g. Production
                    if not ((mstat.groups()[0] == 'Production') and (status == 1)):
                        status = 0
                mwait = wait.match(det)
                if mwait: # Waiting jobs
                    if (mwait.groups()[0] > wait_jobs):
                        wait_jobs = mwait.groups()[0]
                mname = name.match(det)
                if mname: # jm name
                    jmname = mname.groups()[0]

            jminfo["name"] = jmname
            jminfo["status"] = status
            jminfo["waiting_jobs"] = wait_jobs
            jminfo["host"] = hostname

            jminfo_list.append(copy.deepcopy(jminfo))

    # Narrow the list of host to include only OSG sites if requested
    osg_list = isOSGSite([x['host'] for x in jminfo_list])
    if not onlyOSG:
        CElist = [x['name'] for x in jminfo_list]
    else:
        CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]

    # Narrow the OSG host list to include only those with the specified software and architecture
#    softarch_list = getSoftwareAndArch(osg_list, software, arch)
    softarch_list = getSoftwareAndArch2(CElist, software, arch)

    # remove any non-OSG sites from the list
    jminfo_newlist = []

    for item in jminfo_list:
        for narrowed_item in softarch_list:
            if (item['name'] == narrowed_item):
                if (jminfo_newlist.count(item) == 0):
                    jminfo_newlist.append(item)

    return jminfo_newlist

# This function is used to sort lists of dictionaries
def compare_by (fieldname):
    def compare_two_dicts (a, b):
        return cmp(int(a[fieldname]), int(b[fieldname]))
    return compare_two_dicts

def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
    # Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
    jms.sort(compare_by('waiting_jobs'))
    jmlist = []
    r = re.compile('^(.*:.*/jobmanager-.*?)-(.*)')
    for jm in jms:
        fullname = jm['name']
        m = r.match(fullname)
        if m:
            name = m.groups()[0]
            if (jmlist.count(name) == 0): jmlist.append(name)

    return jmlist
            
def listAllSEs(bdii='exp-bdii.cern.ch'):
    host = re.compile('^GlueCEInfoDefaultSE: (.*)')
    
    seList = []

    query = ' "(GlueCEAccessControlBaseRule=VO:cms)" '
    pout = runldapquery(query, 'GlueCEInfoDefaultSE', bdii)
    output = concatoutput(pout)

    stanza_list = output.split(LF+LF)
    for stanza in stanza_list:
        if len(stanza) > 1:
            details = stanza.split(LF)
            for det in details:
                mhost = host.match(det)
                if mhost: # hostname
                    hostname = mhost.groups()[0]
                    if hostname not in seList:
                        seList.append(mhost.groups()[0])
    
    return seList                    
                    

if __name__ == '__main__':
    import pprint
    pprint.pprint(listAllSEs())
#    seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
#    seList = ['ccsrm.in2p3.fr', 'storm.ifca.es']
#    jmlist =  getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345", 'uscmsbd2.fnal.gov', True)
#    for jm in jmlist:
#        print jm
#   print jm_from_se_bdii(sys.argv[1])
Revision:	1.15
Committed:	Tue May 13 17:04:29 2008 UTC (16 years, 11 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_2_2_pre4, CRAB_2_2_2_pre3, CRAB_2_2_2_pre2, CRAB_2_2_2_pre1, CRAB_2_2_1, CRAB_2_2_1_pre6, CRAB_2_2_1_pre5, CRAB_2_2_1_pre4, PRODCOMMON_0_10_7_testCS2, CRAB_2_2_1_pre3, CRAB_2_2_1_pre2, CRAB_2_2_1_pre1, CRAB_2_2_0, CRAB_2_2_0_pre21
Changes since 1.14:	+29 -4 lines
Log Message:	Two hour default, new function in osg_bdii
#	User	Rev	Content
1	ewv	1.7	#!/usr/bin/env python
2			import re, popen2, sys, copy
3
4			CR = '\r'
5			LF = '\n'
6			CRLF = CR+LF
7
8			def unwraplines(wrapped_list):
9			r = re.compile('^ (.*)$')
10			unwrapped_list = []
11			for l in wrapped_list:
12			m = r.match(l)
13			if m:
14			unwrapped_list[-1] += m.groups()[0]
15			else:
16			unwrapped_list.append(l.rstrip())
17
18			return unwrapped_list
19
20
21			def runldapquery(filter, attribute, bdii):
22			command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
23			command += filter + ' ' + attribute
24			pout,pin,perr = popen2.popen3(command)
25
26			pout = pout.readlines()
27			p = perr.readlines()
28
29			pout = unwraplines(pout)
30			if (p):
31			for l in p: print l
32			raise RuntimeError('ldapsearch call failed')
33
34			return pout
35
36			def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
37			se = '\'' + se + '\''
38	ewv	1.13	pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)
39	ewv	1.7
40	ewv	1.8	# r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.*)-cms')
41			r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
42	ewv	1.7	jm = []
43			for l in pout:
44			m = r.match(l)
45			if m:
46			item = m.groups()[0]
47			if (jm.count(item) == 0):
48	ewv	1.8	jm.append(item)
49	ewv	1.7
50			return jm
51
52
53			def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
54			status = []
55			jmlist = jm_from_se_bdii(se)
56
57			for jm in jmlist:
58			jm += "-cms"
59
60	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)
61	ewv	1.7
62			r = re.compile('^GlueCEStateStatus: (.*)')
63			for l in pout:
64			m = r.match(l)
65			if m:
66			status.append(m.groups()[0])
67
68			return status
69
70			def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
71	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)
72	ewv	1.7
73			status = ''
74			r = re.compile('^GlueCEStateStatus: (.*)')
75			for l in pout:
76			m = r.match(l)
77			if m:
78			status = m.groups()[0]
79
80			return status
81
82			def concatoutput(pout):
83			output = ''
84			for l in pout:
85			if l == '':
86			output = output + LF
87			output = output + l + LF
88
89			return output
90
91			def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
92			# Get the Jobmanager list
93			jmlist = []
94
95	ewv	1.13	query = " '(\|"
96	ewv	1.7	for se in selist:
97	ewv	1.13	query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
98			query = query + ")' "
99	ewv	1.7
100			pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
101	ewv	1.8	r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
102
103	ewv	1.7	for l in pout:
104			m = r.match(l)
105			if m:
106			item = m.groups()[0]
107			if (jmlist.count(item) == 0):
108	ewv	1.8	jmlist.append(item)
109
110	ewv	1.10
111	ewv	1.13	query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(\|"
112	ewv	1.8	for l in jmlist:
113	ewv	1.13	query += "(GlueCEInfoContactString=" + l + "-*)"
114	ewv	1.8
115	ewv	1.13	query += "))' "
116	ewv	1.8
117			pout = runldapquery(query, 'GlueCEInfoContactString', bdii)
118
119			r = re.compile('^GlueCEInfoContactString: (.:./jobmanager-.*)')
120			for l in pout:
121			m = r.match(l)
122			if m:
123			item = m.groups()[0]
124			if (jmlist.count(item) == 0):
125			jmlist.append(item)
126	ewv	1.7
127			return jmlist
128
129			def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
130			results_list = []
131			r = re.compile('^GlueSiteDescription: (OSG.*)')
132			s = re.compile('^GlueSiteUniqueID: (.*)')
133
134	ewv	1.13	query = " '(\|"
135	ewv	1.7	for h in host_list:
136	ewv	1.13	query = query + "(GlueSiteUniqueID=" + h + ")"
137			query = query + ")' GlueSiteDescription"
138	ewv	1.7
139			pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
140			output = concatoutput(pout)
141
142			stanzas = output.split(LF + LF)
143			for stanza in stanzas:
144			osg = 0
145			host = ""
146			details = stanza.split(LF)
147			for detail in details:
148			m = r.match(detail)
149			n = s.match(detail)
150			if m:
151			osg = 1
152			if n:
153			host = n.groups()[0]
154			if (osg == 1):
155			results_list.append(host)
156
157			return results_list
158
159	ewv	1.10	def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
160			results_list = []
161
162			# Find installed CMSSW versions and Architecture
163			software = 'VO-cms-' + software
164			arch = 'VO-cms-' + arch
165
166			query = "'(\|"
167
168			for h in host_list:
169			query += "(GlueCEInfoContactString=" + h + ")"
170			query += ")'"
171
172			pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
173			r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
174			s = re.compile('GlueCEInfoContactString: (.*)')
175
176			ClusterMap = {}
177			ClusterUniqueID = None
178			CEInfoContact = None
179
180			for l in pout:
181			m = r.match(l)
182			if m:
183			ClusterUniqueID = m.groups()[0]
184			m = s.match(l)
185			if m:
186			CEInfoContact = m.groups()[0]
187
188			if (ClusterUniqueID and CEInfoContact):
189			ClusterMap[ClusterUniqueID] = CEInfoContact
190			ClusterUniqueID = None
191			CEInfoContact = None
192
193			query = "'(\|"
194			for c in ClusterMap.keys():
195			query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
196			query += ")'"
197
198			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
199			output = concatoutput(pout)
200
201			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
202			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
203			stanzas = output.split(LF + LF)
204			for stanza in stanzas:
205			software_installed = 0
206			architecture = 0
207			host = ''
208			details = stanza.split(LF)
209			for detail in details:
210			m = r.match(detail)
211			if m:
212			if (m.groups()[0] == software):
213			software_installed = 1
214			elif (m.groups()[0] == arch):
215			architecture = 1
216			m2 = s.match(detail)
217			if m2:
218			ClusterUniqueID = m2.groups()[0]
219			host = ClusterMap[ClusterUniqueID]
220
221			if ((software_installed == 1) and (architecture == 1)):
222			results_list.append(host)
223
224			return results_list
225	ewv	1.7
226			def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
227			results_list = []
228
229			# Find installed CMSSW versions and Architecture
230			software = 'VO-cms-' + software
231			arch = 'VO-cms-' + arch
232
233	ewv	1.13	query = " '(\|"
234	ewv	1.7	for h in host_list:
235	ewv	1.13	query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
236			query = query + ")' "
237	ewv	1.7
238			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
239			output = concatoutput(pout)
240
241			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
242			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
243			stanzas = output.split(LF + LF)
244			for stanza in stanzas:
245			software_installed = 0
246			architecture = 0
247			host = ''
248			details = stanza.split(LF)
249			for detail in details:
250			m = r.match(detail)
251			if m:
252			if (m.groups()[0] == software):
253			software_installed = 1
254			elif (m.groups()[0] == arch):
255			architecture = 1
256			m2 = s.match(detail)
257			if m2:
258			host = m2.groups()[0]
259
260			if ((software_installed == 1) and (architecture == 1)):
261			results_list.append(host)
262
263			return results_list
264
265			def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
266			jminfo_list = []
267			host_list = []
268
269			stat = re.compile('^GlueCEStateStatus: (.*)')
270			host = re.compile('^GlueCEInfoHostName: (.*)')
271			wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
272			name = re.compile('^GlueCEUniqueID: (.*)')
273
274			jmlist = getJMListFromSEList(selist)
275	ewv	1.8
276	ewv	1.13	query = " '(&(objectClass=GlueCEState)(\|"
277	ewv	1.7	for jm in jmlist:
278	ewv	1.13	query = query + "(GlueCEUniqueID=" + jm + ")"
279			query = query + "))' "
280	ewv	1.7
281			pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
282			output = concatoutput(pout)
283
284			stanza_list = output.split(LF+LF)
285			for stanza in stanza_list:
286			if len(stanza) > 1:
287			status = 1
288			wait_jobs = 0
289			jmname = ''
290			hostname = 0
291			jminfo = {}
292
293			details = stanza.split(LF)
294			for det in details:
295			mhost = host.match(det)
296			if mhost: # hostname
297			host_list.append(mhost.groups()[0])
298			hostname = mhost.groups()[0]
299			mstat = stat.match(det)
300			if mstat: # e.g. Production
301			if not ((mstat.groups()[0] == 'Production') and (status == 1)):
302			status = 0
303			mwait = wait.match(det)
304			if mwait: # Waiting jobs
305			if (mwait.groups()[0] > wait_jobs):
306			wait_jobs = mwait.groups()[0]
307			mname = name.match(det)
308			if mname: # jm name
309			jmname = mname.groups()[0]
310
311			jminfo["name"] = jmname
312			jminfo["status"] = status
313			jminfo["waiting_jobs"] = wait_jobs
314			jminfo["host"] = hostname
315
316			jminfo_list.append(copy.deepcopy(jminfo))
317
318			# Narrow the list of host to include only OSG sites if requested
319	ewv	1.13	osg_list = isOSGSite([x['host'] for x in jminfo_list])
320			if not onlyOSG:
321			CElist = [x['name'] for x in jminfo_list]
322	ewv	1.7	else:
323	ewv	1.13	CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]
324	ewv	1.7
325			# Narrow the OSG host list to include only those with the specified software and architecture
326	ewv	1.10	# softarch_list = getSoftwareAndArch(osg_list, software, arch)
327			softarch_list = getSoftwareAndArch2(CElist, software, arch)
328	ewv	1.7
329			# remove any non-OSG sites from the list
330			jminfo_newlist = []
331	ewv	1.10
332	ewv	1.7	for item in jminfo_list:
333			for narrowed_item in softarch_list:
334	ewv	1.10	if (item['name'] == narrowed_item):
335	ewv	1.8	if (jminfo_newlist.count(item) == 0):
336			jminfo_newlist.append(item)
337	ewv	1.7
338			return jminfo_newlist
339
340			# This function is used to sort lists of dictionaries
341			def compare_by (fieldname):
342			def compare_two_dicts (a, b):
343			return cmp(int(a[fieldname]), int(b[fieldname]))
344			return compare_two_dicts
345
346			def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
347			jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
348			# Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
349			jms.sort(compare_by('waiting_jobs'))
350			jmlist = []
351	ewv	1.8	r = re.compile('^(.:./jobmanager-.?)-(.)')
352	ewv	1.7	for jm in jms:
353	ewv	1.8	fullname = jm['name']
354			m = r.match(fullname)
355			if m:
356			name = m.groups()[0]
357			if (jmlist.count(name) == 0): jmlist.append(name)
358	ewv	1.7
359			return jmlist
360	ewv	1.15
361			def listAllSEs(bdii='exp-bdii.cern.ch'):
362			host = re.compile('^GlueCEInfoDefaultSE: (.*)')
363
364			seList = []
365
366			query = ' "(GlueCEAccessControlBaseRule=VO:cms)" '
367			pout = runldapquery(query, 'GlueCEInfoDefaultSE', bdii)
368			output = concatoutput(pout)
369
370			stanza_list = output.split(LF+LF)
371			for stanza in stanza_list:
372			if len(stanza) > 1:
373			details = stanza.split(LF)
374			for det in details:
375			mhost = host.match(det)
376			if mhost: # hostname
377			hostname = mhost.groups()[0]
378			if hostname not in seList:
379			seList.append(mhost.groups()[0])
380
381			return seList
382
383	ewv	1.7
384			if __name__ == '__main__':
385	ewv	1.15	import pprint
386			pprint.pprint(listAllSEs())
387			# seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
388	ewv	1.10	# seList = ['ccsrm.in2p3.fr', 'storm.ifca.es']
389	ewv	1.15	# jmlist = getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345", 'uscmsbd2.fnal.gov', True)
390			# for jm in jmlist:
391			# print jm
392	ewv	1.10	# print jm_from_se_bdii(sys.argv[1])