CRAB/python/osg_bdii.py

#!/usr/bin/env python
import re, popen2, sys, copy

CR = '\r'
LF = '\n'
CRLF = CR+LF

def unwraplines(wrapped_list):
    r = re.compile('^ (.*)$')
    unwrapped_list = []
    for l in wrapped_list:
        m = r.match(l)
        if m:
            unwrapped_list[-1] += m.groups()[0]
        else:
            unwrapped_list.append(l.rstrip())

    return unwrapped_list


def runldapquery(filter, attribute, bdii):
    command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
    command += filter + ' ' + attribute
    #print "Command",command
    pout,pin,perr = popen2.popen3(command)

    pout = pout.readlines()
    p = perr.readlines()

    pout = unwraplines(pout)
    if (p):
        for l in p: print l
        raise RuntimeError('ldapsearch call failed')

    return pout

def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
    se = '\'' + se + '\''
    pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)

    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')
    jm = []
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jm.count(item) == 0):
                jm.append(item)

    return jm


def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
    status = []
    jmlist = jm_from_se_bdii(se)

    for jm in jmlist:
        jm += "-cms"

        pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)

        r = re.compile('^GlueCEStateStatus: (.*)')
        for l in pout:
            m = r.match(l)
            if m:
                status.append(m.groups()[0])

    return status

def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
    pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)

    status = ''
    r = re.compile('^GlueCEStateStatus: (.*)')
    for l in pout:
        m = r.match(l)
        if m:
            status = m.groups()[0]

    return status

def concatoutput(pout):
    output = ''
    for l in pout:
        if l == '':
            output = output + LF
        output = output + l + LF

    return output

def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
    # Get the Jobmanager list
    jmlist = []

    query = " '(|"
    for se in selist:
        query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
    query = query + ")' "

    pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')

    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)


    query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(|"
    for l in jmlist:
        query += "(GlueCEInfoContactString=" + l + "-*)"

    query += "))' "

    pout = runldapquery(query, 'GlueCEInfoContactString', bdii)

    r = re.compile('^GlueCEInfoContactString: (.*:.*/jobmanager-.*)')
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)

    return jmlist

def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
    results_list = []
    r = re.compile('^GlueSiteDescription: (OSG.*)')
    s = re.compile('^GlueSiteUniqueID: (.*)')

    query = " '(|"
    for h in host_list:
        query = query + "(GlueSiteUniqueID=" + h + ")"
    query = query + ")' GlueSiteDescription"

    pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
    output = concatoutput(pout)

    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        osg = 0
        host = ""
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            n = s.match(detail)
            if m:
                osg = 1
            if n:
                host = n.groups()[0]
        if (osg == 1):
            results_list.append(host)
    return results_list

def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = "'(|"

    for h in host_list:
        query += "(GlueCEInfoContactString=" + h + ")"
    query += ")'"

    pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
    r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
    s = re.compile('GlueCEInfoContactString: (.*)')

    ClusterMap =  {}
    ClusterUniqueID = None
    CEInfoContact = None

    for l in pout:
        m = r.match(l)
        if m:
            ClusterUniqueID = m.groups()[0]
        m = s.match(l)
        if m:
            CEInfoContact = m.groups()[0]

        if (ClusterUniqueID and CEInfoContact):
            ClusterMap[ClusterUniqueID] = CEInfoContact
            ClusterUniqueID = None
            CEInfoContact = None

    query = "'(|"
    for c in ClusterMap.keys():
        query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
    query += ")'"

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                ClusterUniqueID = m2.groups()[0]
                host = ClusterMap[ClusterUniqueID]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = " '(|"
    for h in host_list:
        query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
    query = query + ")' "

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                host = m2.groups()[0]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jminfo_list = []
    host_list = []

    stat = re.compile('^GlueCEStateStatus: (.*)')
    host = re.compile('^GlueCEInfoHostName: (.*)')
    wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
    name = re.compile('^GlueCEUniqueID: (.*)')

    jmlist = getJMListFromSEList(selist)

    query = " '(&(objectClass=GlueCEState)(|"
    for jm in jmlist:
        query = query + "(GlueCEUniqueID=" + jm + ")"
    query = query + "))' "

    pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
    output = concatoutput(pout)

    stanza_list = output.split(LF+LF)
    for stanza in stanza_list:
        if len(stanza) > 1:
            status = 1
            wait_jobs = 0
            jmname = ''
            hostname = 0
            jminfo = {}

            details = stanza.split(LF)
            for det in details:
                mhost = host.match(det)
                if mhost: # hostname
                    host_list.append(mhost.groups()[0])
                    hostname = mhost.groups()[0]
                mstat = stat.match(det)
                if mstat: # e.g. Production
                    if not ((mstat.groups()[0] == 'Production') and (status == 1)):
                        status = 0
                mwait = wait.match(det)
                if mwait: # Waiting jobs
                    if (mwait.groups()[0] > wait_jobs):
                        wait_jobs = mwait.groups()[0]
                mname = name.match(det)
                if mname: # jm name
                    jmname = mname.groups()[0]

            jminfo["name"] = jmname
            jminfo["status"] = status
            jminfo["waiting_jobs"] = wait_jobs
            jminfo["host"] = hostname

            jminfo_list.append(copy.deepcopy(jminfo))

    # Narrow the list of host to include only OSG sites if requested
    osg_list = isOSGSite([x['host'] for x in jminfo_list])
    if not onlyOSG:
        CElist = [x['name'] for x in jminfo_list]
    else:
        CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]

    # Narrow the OSG host list to include only those with the specified software and architecture
#    softarch_list = getSoftwareAndArch(osg_list, software, arch)
    softarch_list = getSoftwareAndArch2(CElist, software, arch)

    # remove any non-OSG sites from the list
    jminfo_newlist = []

    for item in jminfo_list:
        for narrowed_item in softarch_list:
            if (item['name'] == narrowed_item):
                if (jminfo_newlist.count(item) == 0):
                    jminfo_newlist.append(item)

    return jminfo_newlist

# This function is used to sort lists of dictionaries
def compare_by (fieldname):
    def compare_two_dicts (a, b):
        return cmp(int(a[fieldname]), int(b[fieldname]))
    return compare_two_dicts

def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
    # Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
    jms.sort(compare_by('waiting_jobs'))
    jmlist = []
    r = re.compile('^(.*:.*/jobmanager-.*?)-(.*)')
    for jm in jms:
        fullname = jm['name']
        m = r.match(fullname)
        if m:
            name = m.groups()[0]
            if (jmlist.count(name) == 0): jmlist.append(name)

    return jmlist

def listAllCEs(software, arch, bdii='exp-bdii.cern.ch',onlyOSG=True):
    ''' List all GlueCEUniqueIDs that advertise support for CMS '''

#    RE_cename = re.compile('^GlueCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)', re.IGNORECASE)
    RE_cename = re.compile('^GlueCEUniqueID: (.*)', re.IGNORECASE)
    hostSplit  = re.compile(r'[^\w\.\-]')
    filt = "'(&(GlueCEUniqueID=*)(GlueCEAccessControlBaseRule=VO:cms))'"
    pout = runldapquery(filt, 'GlueCEUniqueID', bdii)
    ceList   = []
    hostList = []
    for l in pout:
        m = RE_cename.match(l)
        if m:
            item = m.groups()[0]
            hostname = hostSplit.split(item)[0]
            if (ceList.count(item) == 0):       ceList.append(item)
            if (hostList.count(hostname) == 0): hostList.append(hostname)

    if onlyOSG:
      osgCEs = []
      osgList =  isOSGSite(hostList, bdii)
      for ce in ceList:
        hostname = hostSplit.split(ce)[0]
        if hostname in osgList:
          osgCEs.append(ce)
    else:
      osgCEs = ceList

    softarch_list = getSoftwareAndArch2(osgCEs, software, arch)

    shortCeList   = [] # Convert to CE without queue
    RE_short = re.compile('^(.*:.*/jobmanager-.*?)-(.*)', re.IGNORECASE)
    for ce in softarch_list:
        m = RE_short.match(ce)
        if m:
            item = m.groups()[0]
            if (shortCeList.count(item) == 0):       shortCeList.append(item)

    return shortCeList

def listAllSEs(bdii='exp-bdii.cern.ch'):
    ''' List all SEs that are bound to (CEs that advertise support for CMS) '''

    RE_sename = re.compile('^GlueCESEBindGroupSEUniqueID: (.*)', re.IGNORECASE)
    seList = []
    filt = "'(|"
    ceList = listAllCEs(bdii)
    for ce in ceList:
        filtstring = '(GlueCESEBindGroupCEUniqueID=' + ce + ')'
        filt += filtstring
    filt += ")'"

    pout = runldapquery(filt, 'GlueCESEBindGroupSEUniqueID', bdii)
    for l in pout:
        m = RE_sename.match(l)
        if m:
            item = m.groups()[0]
            if (seList.count(item) == 0): seList.append(item)

    return seList


if __name__ == '__main__':
    import pprint
#    pprint.pprint(listAllSEs('uscmsbd2.fnal.gov'))
    pprint.pprint(listAllCEs( "CMSSW_1_6_11", "slc4_ia32_gcc345", onlyOSG=False))
#    pprint.pprint(listAllCEs(onlyOSG=False))

    seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
#    seList = ['ccsrm.in2p3.fr', 'storm.ifca.es']
#    jmlist =  getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345", 'uscmsbd2.fnal.gov', True)
#    for jm in jmlist:
#        print jm
#   print jm_from_se_bdii(sys.argv[1])
Revision:	1.18
Committed:	Mon Jun 9 18:44:16 2008 UTC (16 years, 10 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_3_1_pre2, CRAB_2_3_1_pre1, CRAB_2_3_0, CRAB_2_3_0_pre6, CRAB_2_3_0_pre1, CRAB_2_2_2_pre5
Branch point for:	CRAB_2_3_0_br
Changes since 1.17:	+11 -4 lines
Log Message:	Should work with Pythia/OSG
#	User	Rev	Content
1	ewv	1.7	#!/usr/bin/env python
2			import re, popen2, sys, copy
3
4			CR = '\r'
5			LF = '\n'
6			CRLF = CR+LF
7
8			def unwraplines(wrapped_list):
9			r = re.compile('^ (.*)$')
10			unwrapped_list = []
11			for l in wrapped_list:
12			m = r.match(l)
13			if m:
14			unwrapped_list[-1] += m.groups()[0]
15			else:
16			unwrapped_list.append(l.rstrip())
17
18			return unwrapped_list
19
20
21			def runldapquery(filter, attribute, bdii):
22			command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
23			command += filter + ' ' + attribute
24	ewv	1.17	#print "Command",command
25	ewv	1.7	pout,pin,perr = popen2.popen3(command)
26
27			pout = pout.readlines()
28			p = perr.readlines()
29
30			pout = unwraplines(pout)
31			if (p):
32			for l in p: print l
33			raise RuntimeError('ldapsearch call failed')
34
35			return pout
36
37			def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
38			se = '\'' + se + '\''
39	ewv	1.13	pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)
40	ewv	1.7
41	ewv	1.8	r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
42	ewv	1.7	jm = []
43			for l in pout:
44			m = r.match(l)
45			if m:
46			item = m.groups()[0]
47			if (jm.count(item) == 0):
48	ewv	1.8	jm.append(item)
49	ewv	1.7
50			return jm
51
52
53			def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
54			status = []
55			jmlist = jm_from_se_bdii(se)
56
57			for jm in jmlist:
58			jm += "-cms"
59
60	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)
61	ewv	1.7
62			r = re.compile('^GlueCEStateStatus: (.*)')
63			for l in pout:
64			m = r.match(l)
65			if m:
66			status.append(m.groups()[0])
67
68			return status
69
70			def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
71	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)
72	ewv	1.7
73			status = ''
74			r = re.compile('^GlueCEStateStatus: (.*)')
75			for l in pout:
76			m = r.match(l)
77			if m:
78			status = m.groups()[0]
79
80			return status
81
82			def concatoutput(pout):
83			output = ''
84			for l in pout:
85			if l == '':
86			output = output + LF
87			output = output + l + LF
88
89			return output
90
91			def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
92			# Get the Jobmanager list
93			jmlist = []
94
95	ewv	1.13	query = " '(\|"
96	ewv	1.7	for se in selist:
97	ewv	1.13	query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
98			query = query + ")' "
99	ewv	1.7
100			pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
101	ewv	1.8	r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
102
103	ewv	1.7	for l in pout:
104			m = r.match(l)
105			if m:
106			item = m.groups()[0]
107			if (jmlist.count(item) == 0):
108	ewv	1.8	jmlist.append(item)
109
110	ewv	1.10
111	ewv	1.13	query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(\|"
112	ewv	1.8	for l in jmlist:
113	ewv	1.13	query += "(GlueCEInfoContactString=" + l + "-*)"
114	ewv	1.8
115	ewv	1.13	query += "))' "
116	ewv	1.8
117			pout = runldapquery(query, 'GlueCEInfoContactString', bdii)
118
119			r = re.compile('^GlueCEInfoContactString: (.:./jobmanager-.*)')
120			for l in pout:
121			m = r.match(l)
122			if m:
123			item = m.groups()[0]
124			if (jmlist.count(item) == 0):
125			jmlist.append(item)
126	ewv	1.7
127			return jmlist
128
129			def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
130			results_list = []
131			r = re.compile('^GlueSiteDescription: (OSG.*)')
132			s = re.compile('^GlueSiteUniqueID: (.*)')
133
134	ewv	1.13	query = " '(\|"
135	ewv	1.7	for h in host_list:
136	ewv	1.13	query = query + "(GlueSiteUniqueID=" + h + ")"
137			query = query + ")' GlueSiteDescription"
138	ewv	1.7
139			pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
140			output = concatoutput(pout)
141
142			stanzas = output.split(LF + LF)
143			for stanza in stanzas:
144			osg = 0
145			host = ""
146			details = stanza.split(LF)
147			for detail in details:
148			m = r.match(detail)
149			n = s.match(detail)
150			if m:
151			osg = 1
152			if n:
153			host = n.groups()[0]
154			if (osg == 1):
155			results_list.append(host)
156			return results_list
157
158	ewv	1.10	def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
159			results_list = []
160
161			# Find installed CMSSW versions and Architecture
162			software = 'VO-cms-' + software
163			arch = 'VO-cms-' + arch
164
165			query = "'(\|"
166
167			for h in host_list:
168			query += "(GlueCEInfoContactString=" + h + ")"
169			query += ")'"
170
171			pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
172			r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
173			s = re.compile('GlueCEInfoContactString: (.*)')
174
175			ClusterMap = {}
176			ClusterUniqueID = None
177			CEInfoContact = None
178
179			for l in pout:
180			m = r.match(l)
181			if m:
182			ClusterUniqueID = m.groups()[0]
183			m = s.match(l)
184			if m:
185			CEInfoContact = m.groups()[0]
186
187			if (ClusterUniqueID and CEInfoContact):
188			ClusterMap[ClusterUniqueID] = CEInfoContact
189			ClusterUniqueID = None
190			CEInfoContact = None
191
192			query = "'(\|"
193			for c in ClusterMap.keys():
194			query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
195			query += ")'"
196
197			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
198			output = concatoutput(pout)
199
200			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
201			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
202			stanzas = output.split(LF + LF)
203			for stanza in stanzas:
204			software_installed = 0
205			architecture = 0
206			host = ''
207			details = stanza.split(LF)
208			for detail in details:
209			m = r.match(detail)
210			if m:
211			if (m.groups()[0] == software):
212			software_installed = 1
213			elif (m.groups()[0] == arch):
214			architecture = 1
215			m2 = s.match(detail)
216			if m2:
217			ClusterUniqueID = m2.groups()[0]
218			host = ClusterMap[ClusterUniqueID]
219
220			if ((software_installed == 1) and (architecture == 1)):
221			results_list.append(host)
222
223			return results_list
224	ewv	1.7
225			def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
226			results_list = []
227
228			# Find installed CMSSW versions and Architecture
229			software = 'VO-cms-' + software
230			arch = 'VO-cms-' + arch
231
232	ewv	1.13	query = " '(\|"
233	ewv	1.7	for h in host_list:
234	ewv	1.13	query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
235			query = query + ")' "
236	ewv	1.7
237			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
238			output = concatoutput(pout)
239
240			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
241			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
242			stanzas = output.split(LF + LF)
243			for stanza in stanzas:
244			software_installed = 0
245			architecture = 0
246			host = ''
247			details = stanza.split(LF)
248			for detail in details:
249			m = r.match(detail)
250			if m:
251			if (m.groups()[0] == software):
252			software_installed = 1
253			elif (m.groups()[0] == arch):
254			architecture = 1
255			m2 = s.match(detail)
256			if m2:
257			host = m2.groups()[0]
258
259			if ((software_installed == 1) and (architecture == 1)):
260			results_list.append(host)
261
262			return results_list
263
264			def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
265			jminfo_list = []
266			host_list = []
267
268			stat = re.compile('^GlueCEStateStatus: (.*)')
269			host = re.compile('^GlueCEInfoHostName: (.*)')
270			wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
271			name = re.compile('^GlueCEUniqueID: (.*)')
272
273			jmlist = getJMListFromSEList(selist)
274	ewv	1.8
275	ewv	1.13	query = " '(&(objectClass=GlueCEState)(\|"
276	ewv	1.7	for jm in jmlist:
277	ewv	1.13	query = query + "(GlueCEUniqueID=" + jm + ")"
278			query = query + "))' "
279	ewv	1.7
280			pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
281			output = concatoutput(pout)
282
283			stanza_list = output.split(LF+LF)
284			for stanza in stanza_list:
285			if len(stanza) > 1:
286			status = 1
287			wait_jobs = 0
288			jmname = ''
289			hostname = 0
290			jminfo = {}
291
292			details = stanza.split(LF)
293			for det in details:
294			mhost = host.match(det)
295			if mhost: # hostname
296			host_list.append(mhost.groups()[0])
297			hostname = mhost.groups()[0]
298			mstat = stat.match(det)
299			if mstat: # e.g. Production
300			if not ((mstat.groups()[0] == 'Production') and (status == 1)):
301			status = 0
302			mwait = wait.match(det)
303			if mwait: # Waiting jobs
304			if (mwait.groups()[0] > wait_jobs):
305			wait_jobs = mwait.groups()[0]
306			mname = name.match(det)
307			if mname: # jm name
308			jmname = mname.groups()[0]
309
310			jminfo["name"] = jmname
311			jminfo["status"] = status
312			jminfo["waiting_jobs"] = wait_jobs
313			jminfo["host"] = hostname
314
315			jminfo_list.append(copy.deepcopy(jminfo))
316
317			# Narrow the list of host to include only OSG sites if requested
318	ewv	1.13	osg_list = isOSGSite([x['host'] for x in jminfo_list])
319			if not onlyOSG:
320			CElist = [x['name'] for x in jminfo_list]
321	ewv	1.7	else:
322	ewv	1.13	CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]
323	ewv	1.7
324			# Narrow the OSG host list to include only those with the specified software and architecture
325	ewv	1.10	# softarch_list = getSoftwareAndArch(osg_list, software, arch)
326			softarch_list = getSoftwareAndArch2(CElist, software, arch)
327	ewv	1.7
328			# remove any non-OSG sites from the list
329			jminfo_newlist = []
330	ewv	1.10
331	ewv	1.7	for item in jminfo_list:
332			for narrowed_item in softarch_list:
333	ewv	1.10	if (item['name'] == narrowed_item):
334	ewv	1.8	if (jminfo_newlist.count(item) == 0):
335			jminfo_newlist.append(item)
336	ewv	1.7
337			return jminfo_newlist
338
339			# This function is used to sort lists of dictionaries
340			def compare_by (fieldname):
341			def compare_two_dicts (a, b):
342			return cmp(int(a[fieldname]), int(b[fieldname]))
343			return compare_two_dicts
344
345			def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
346			jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
347			# Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
348			jms.sort(compare_by('waiting_jobs'))
349			jmlist = []
350	ewv	1.8	r = re.compile('^(.:./jobmanager-.?)-(.)')
351	ewv	1.7	for jm in jms:
352	ewv	1.8	fullname = jm['name']
353			m = r.match(fullname)
354			if m:
355			name = m.groups()[0]
356			if (jmlist.count(name) == 0): jmlist.append(name)
357	ewv	1.7
358			return jmlist
359	ewv	1.16
360	ewv	1.17	def listAllCEs(software, arch, bdii='exp-bdii.cern.ch',onlyOSG=True):
361	ewv	1.16	''' List all GlueCEUniqueIDs that advertise support for CMS '''
362
363	ewv	1.18	# RE_cename = re.compile('^GlueCEUniqueID: (.:./jobmanager-.?)-(.)', re.IGNORECASE)
364	ewv	1.16	RE_cename = re.compile('^GlueCEUniqueID: (.*)', re.IGNORECASE)
365	ewv	1.17	hostSplit = re.compile(r'[^\w\.\-]')
366	ewv	1.16	filt = "'(&(GlueCEUniqueID=*)(GlueCEAccessControlBaseRule=VO:cms))'"
367			pout = runldapquery(filt, 'GlueCEUniqueID', bdii)
368	ewv	1.17	ceList = []
369			hostList = []
370	ewv	1.16	for l in pout:
371			m = RE_cename.match(l)
372			if m:
373			item = m.groups()[0]
374	ewv	1.17	hostname = hostSplit.split(item)[0]
375			if (ceList.count(item) == 0): ceList.append(item)
376			if (hostList.count(hostname) == 0): hostList.append(hostname)
377
378			if onlyOSG:
379			osgCEs = []
380			osgList = isOSGSite(hostList, bdii)
381			for ce in ceList:
382			hostname = hostSplit.split(ce)[0]
383			if hostname in osgList:
384			osgCEs.append(ce)
385			else:
386			osgCEs = ceList
387
388			softarch_list = getSoftwareAndArch2(osgCEs, software, arch)
389	ewv	1.16
390	ewv	1.18	shortCeList = [] # Convert to CE without queue
391			RE_short = re.compile('^(.:./jobmanager-.?)-(.)', re.IGNORECASE)
392			for ce in softarch_list:
393			m = RE_short.match(ce)
394			if m:
395			item = m.groups()[0]
396			if (shortCeList.count(item) == 0): shortCeList.append(item)
397
398			return shortCeList
399	ewv	1.16
400	ewv	1.15	def listAllSEs(bdii='exp-bdii.cern.ch'):
401	ewv	1.16	''' List all SEs that are bound to (CEs that advertise support for CMS) '''
402
403			RE_sename = re.compile('^GlueCESEBindGroupSEUniqueID: (.*)', re.IGNORECASE)
404	ewv	1.15	seList = []
405	ewv	1.16	filt = "'(\|"
406			ceList = listAllCEs(bdii)
407			for ce in ceList:
408			filtstring = '(GlueCESEBindGroupCEUniqueID=' + ce + ')'
409			filt += filtstring
410			filt += ")'"
411
412			pout = runldapquery(filt, 'GlueCESEBindGroupSEUniqueID', bdii)
413			for l in pout:
414			m = RE_sename.match(l)
415			if m:
416			item = m.groups()[0]
417			if (seList.count(item) == 0): seList.append(item)
418
419			return seList
420	ewv	1.15
421	ewv	1.7
422			if __name__ == '__main__':
423	ewv	1.15	import pprint
424	ewv	1.16	# pprint.pprint(listAllSEs('uscmsbd2.fnal.gov'))
425	ewv	1.18	pprint.pprint(listAllCEs( "CMSSW_1_6_11", "slc4_ia32_gcc345", onlyOSG=False))
426	ewv	1.17	# pprint.pprint(listAllCEs(onlyOSG=False))
427	ewv	1.16
428	ewv	1.17	seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
429	ewv	1.10	# seList = ['ccsrm.in2p3.fr', 'storm.ifca.es']
430	ewv	1.15	# jmlist = getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345", 'uscmsbd2.fnal.gov', True)
431			# for jm in jmlist:
432			# print jm
433	ewv	1.10	# print jm_from_se_bdii(sys.argv[1])