CRAB/python/osg_bdii.py

#!/usr/bin/env python

import re
import sys
import copy
import sets
import popen2

CR = '\r'
LF = '\n'
CRLF = CR+LF

def unwraplines(wrapped_list):
    r = re.compile('^ (.*)$')
    unwrapped_list = []
    for l in wrapped_list:
        m = r.match(l)
        if m:
            unwrapped_list[-1] += m.groups()[0]
        else:
            unwrapped_list.append(l.rstrip())

    return unwrapped_list


def runldapquery(filter, attribute, bdii):
    command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
    command += filter + ' ' + attribute
    #print "Command",command
    pout,pin,perr = popen2.popen3(command)

    pout = pout.readlines()
    p = perr.readlines()

    pout = unwraplines(pout)
    if (p):
        for l in p: print l
        raise RuntimeError('ldapsearch call failed')

    return pout

def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
    se = '\'' + se + '\''
    pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)

    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')
    jm = []
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jm.count(item) == 0):
                jm.append(item)

    return jm


def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
    status = []
    jmlist = jm_from_se_bdii(se)

    for jm in jmlist:
        jm += "-cms"

        pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)

        r = re.compile('^GlueCEStateStatus: (.*)')
        for l in pout:
            m = r.match(l)
            if m:
                status.append(m.groups()[0])

    return status

def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
    pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)

    status = ''
    r = re.compile('^GlueCEStateStatus: (.*)')
    for l in pout:
        m = r.match(l)
        if m:
            status = m.groups()[0]

    return status

def concatoutput(pout):
    output = ''
    for l in pout:
        if l == '':
            output = output + LF
        output = output + l + LF

    return output

def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
    # Get the Jobmanager list
    jmlist = []

    query = " '(|"
    for se in selist:
        query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
    query = query + ")' "

    pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)')

    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)


    query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(|"
    for l in jmlist:
        query += "(GlueCEInfoContactString=" + l + "-*)"

    query += "))' "

    pout = runldapquery(query, 'GlueCEInfoContactString', bdii)

    r = re.compile('^GlueCEInfoContactString: (.*:.*/jobmanager-.*)')
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(item)

    return jmlist

def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
    """
    Determine which hosts in the host_list are OSG sites.

    Take a list of CE host names, and find all the associated clusters.
    Take the resulting clusters and map them to sites.  Any such site whose
    description includes "OSG" is labeled an OSG site.
    
    @param host_list: A list of host names which are CEs we want to consider.
    @keyword bdii: The BDII instance to query
    @return: A list of host names filtered from host_list which are OSG sites. 
    """
    siteUniqueID_CE_map = {}
    results = sets.Set()
    description_re = re.compile('^GlueSiteDescription: (OSG.*)')
    siteid_re = re.compile('GlueForeignKey: GlueSiteUniqueID=(.*)')
    ce_re = re.compile('^GlueForeignKey: GlueCEUniqueID=(.*):')
    
    # Build the BDII query for all the hosts.
    # This asks for all GlueClusters which are associated with one of the
    # host names.
    query = " '(&(objectClass=GlueCluster)(|"
    for h in host_list:
        query += "(GlueForeignKey=GlueCEUniqueID=%s:*)" % h
    query += "))' "

    pout = runldapquery(query, 'GlueForeignKey', bdii)
    output = concatoutput(pout)

    # Now, we build a mapping from host to SiteUniqueID
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        details = stanza.split(LF)
        ces_tmp = sets.Set()
        siteUniqueID = None
        # Find out all the matching CEs and Sites with this cluster.
        for detail in details:
            m = ce_re.match(detail)
            if m:
                ces_tmp.add(m.groups()[0])
            m = siteid_re.match(detail)
            if m:
                siteUniqueID = m.groups()[0]
        if siteUniqueID:
            siteUniqueID_CE_map[siteUniqueID] = ces_tmp

    # Build a new query, this time for SiteUniqueIDs
    query = " '(|"
    for site in siteUniqueID_CE_map:
        query += "(GlueSiteUniqueID=%s)" % site
    query += ")' "
    pout = runldapquery(query, "GlueForeignKey GlueSiteDescription", bdii)
    output = concatoutput(pout)
   
    # See which resulting sites are OSG sites, and then add the
    # corresponding CEs into the results set.
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        isOsgSite = False
        siteUniqueID = None
        details = stanza.split(LF)
        # We need to find the stanza's siteUniqueID and if the description
        # is a "OSG Site".  If it is, afterward add it to the results.
        for detail in details:
            m = siteid_re.search(detail)
            if m:
                siteUniqueID = m.groups()[0]
            m = description_re.match(detail)
            if m:
                isOsgSite = True
        if siteUniqueID and isOsgSite:
            results.update(siteUniqueID_CE_map[siteUniqueID])
    return list(results)

def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = "'(|"

    for h in host_list:
        query += "(GlueCEInfoContactString=" + h + ")"
    query += ")'"

    pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
    r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
    s = re.compile('GlueCEInfoContactString: (.*)')

    ClusterMap =  {}
    ClusterUniqueID = None
    CEInfoContact = None

    for l in pout:
        m = r.match(l)
        if m:
            ClusterUniqueID = m.groups()[0]
        m = s.match(l)
        if m:
            CEInfoContact = m.groups()[0]

        if (ClusterUniqueID and CEInfoContact):
            ClusterMap[ClusterUniqueID] = CEInfoContact
            ClusterUniqueID = None
            CEInfoContact = None

    query = "'(|"
    for c in ClusterMap.keys():
        query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
    query += ")'"

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                ClusterUniqueID = m2.groups()[0]
                host = ClusterMap[ClusterUniqueID]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = " '(|"
    for h in host_list:
        query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
    query = query + ")' "

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                host = m2.groups()[0]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jminfo_list = []
    host_list = []

    stat = re.compile('^GlueCEStateStatus: (.*)')
    host = re.compile('^GlueCEInfoHostName: (.*)')
    wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
    name = re.compile('^GlueCEUniqueID: (.*)')

    jmlist = getJMListFromSEList(selist)

    query = " '(&(objectClass=GlueCEState)(|"
    for jm in jmlist:
        query = query + "(GlueCEUniqueID=" + jm + ")"
    query = query + "))' "

    pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
    output = concatoutput(pout)

    stanza_list = output.split(LF+LF)
    for stanza in stanza_list:
        if len(stanza) > 1:
            status = 1
            wait_jobs = 0
            jmname = ''
            hostname = 0
            jminfo = {}

            details = stanza.split(LF)
            for det in details:
                mhost = host.match(det)
                if mhost: # hostname
                    host_list.append(mhost.groups()[0])
                    hostname = mhost.groups()[0]
                mstat = stat.match(det)
                if mstat: # e.g. Production
                    if not ((mstat.groups()[0] == 'Production') and (status == 1)):
                        status = 0
                mwait = wait.match(det)
                if mwait: # Waiting jobs
                    if (mwait.groups()[0] > wait_jobs):
                        wait_jobs = mwait.groups()[0]
                mname = name.match(det)
                if mname: # jm name
                    jmname = mname.groups()[0]

            jminfo["name"] = jmname
            jminfo["status"] = status
            jminfo["waiting_jobs"] = wait_jobs
            jminfo["host"] = hostname

            jminfo_list.append(copy.deepcopy(jminfo))

    # Narrow the list of host to include only OSG sites if requested
    osg_list = isOSGSite([x['host'] for x in jminfo_list])
    if not onlyOSG:
        CElist = [x['name'] for x in jminfo_list]
    else:
        CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]

    # Narrow the OSG host list to include only those with the specified software and architecture
#    softarch_list = getSoftwareAndArch(osg_list, software, arch)
    softarch_list = getSoftwareAndArch2(CElist, software, arch)

    # remove any non-OSG sites from the list
    jminfo_newlist = []

    for item in jminfo_list:
        for narrowed_item in softarch_list:
            if (item['name'] == narrowed_item):
                if (jminfo_newlist.count(item) == 0):
                    jminfo_newlist.append(item)

    return jminfo_newlist

# This function is used to sort lists of dictionaries
def compare_by (fieldname):
    def compare_two_dicts (a, b):
        return cmp(int(a[fieldname]), int(b[fieldname]))
    return compare_two_dicts

def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
    jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
    # Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
    jms.sort(compare_by('waiting_jobs'))
    jmlist = []
    r = re.compile('^(.*:.*/jobmanager-.*?)-(.*)')
    for jm in jms:
        fullname = jm['name']
        m = r.match(fullname)
        if m:
            name = m.groups()[0]
            if (jmlist.count(name) == 0): jmlist.append(name)

    return jmlist

def listAllCEs(software, arch, bdii='exp-bdii.cern.ch',onlyOSG=True):
    ''' List all GlueCEUniqueIDs that advertise support for CMS '''

#    RE_cename = re.compile('^GlueCEUniqueID: (.*:.*/jobmanager-.*?)-(.*)', re.IGNORECASE)
    RE_cename = re.compile('^GlueCEUniqueID: (.*)', re.IGNORECASE)
    hostSplit  = re.compile(r'[^\w\.\-]')
    filt = "'(&(GlueCEUniqueID=*)(GlueCEAccessControlBaseRule=VO:cms))'"
    pout = runldapquery(filt, 'GlueCEUniqueID', bdii)
    ceList   = []
    hostList = []
    for l in pout:
        m = RE_cename.match(l)
        if m:
            item = m.groups()[0]
            hostname = hostSplit.split(item)[0]
            if (ceList.count(item) == 0):       ceList.append(item)
            if (hostList.count(hostname) == 0): hostList.append(hostname)

    if onlyOSG:
      osgCEs = []
      osgList =  isOSGSite(hostList, bdii)
      for ce in ceList:
        hostname = hostSplit.split(ce)[0]
        if hostname in osgList:
          osgCEs.append(ce)
    else:
      osgCEs = ceList

    softarch_list = getSoftwareAndArch2(osgCEs, software, arch)

    shortCeList   = [] # Convert to CE without queue
    RE_short = re.compile('^(.*:.*/jobmanager-.*?)-(.*)', re.IGNORECASE)
    for ce in softarch_list:
        m = RE_short.match(ce)
        if m:
            item = m.groups()[0]
            if (shortCeList.count(item) == 0):       shortCeList.append(item)

    return shortCeList

def listAllSEs(bdii='exp-bdii.cern.ch'):
    ''' List all SEs that are bound to (CEs that advertise support for CMS) '''

    RE_sename = re.compile('^GlueCESEBindGroupSEUniqueID: (.*)', re.IGNORECASE)
    seList = []
    filt = "'(|"
    ceList = listAllCEs(bdii)
    for ce in ceList:
        filtstring = '(GlueCESEBindGroupCEUniqueID=' + ce + ')'
        filt += filtstring
    filt += ")'"

    pout = runldapquery(filt, 'GlueCESEBindGroupSEUniqueID', bdii)
    for l in pout:
        m = RE_sename.match(l)
        if m:
            item = m.groups()[0]
            if (seList.count(item) == 0): seList.append(item)

    return seList


if __name__ == '__main__':
    import pprint
#    pprint.pprint(listAllSEs('uscmsbd2.fnal.gov'))
#    pprint.pprint(listAllCEs( "CMSSW_1_6_11", "slc4_ia32_gcc345", onlyOSG=False))
#    pprint.pprint(listAllCEs(onlyOSG=False))

#    seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
    seList = ['srm.unl.edu', 'dcache.rcac.purdue.edu']
    jmlist =  getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345")
    for jm in jmlist:
        print jm
#   print jm_from_se_bdii(sys.argv[1])
Revision:	1.19
Committed:	Fri Jun 27 14:56:50 2008 UTC (16 years, 10 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
Changes since 1.18:	+74 -24 lines
Log Message:	Fix for OSG 1.0 on HEAD
#	User	Rev	Content
1	ewv	1.7	#!/usr/bin/env python
2	ewv	1.19
3			import re
4			import sys
5			import copy
6			import sets
7			import popen2
8	ewv	1.7
9			CR = '\r'
10			LF = '\n'
11			CRLF = CR+LF
12
13			def unwraplines(wrapped_list):
14			r = re.compile('^ (.*)$')
15			unwrapped_list = []
16			for l in wrapped_list:
17			m = r.match(l)
18			if m:
19			unwrapped_list[-1] += m.groups()[0]
20			else:
21			unwrapped_list.append(l.rstrip())
22
23			return unwrapped_list
24
25
26			def runldapquery(filter, attribute, bdii):
27			command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
28			command += filter + ' ' + attribute
29	ewv	1.17	#print "Command",command
30	ewv	1.7	pout,pin,perr = popen2.popen3(command)
31
32			pout = pout.readlines()
33			p = perr.readlines()
34
35			pout = unwraplines(pout)
36			if (p):
37			for l in p: print l
38			raise RuntimeError('ldapsearch call failed')
39
40			return pout
41
42			def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
43			se = '\'' + se + '\''
44	ewv	1.13	pout = runldapquery(" '(GlueCESEBindGroupSEUniqueID=" + se + ")' ", 'GlueCESEBindGroupCEUniqueID', bdii)
45	ewv	1.7
46	ewv	1.8	r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
47	ewv	1.7	jm = []
48			for l in pout:
49			m = r.match(l)
50			if m:
51			item = m.groups()[0]
52			if (jm.count(item) == 0):
53	ewv	1.8	jm.append(item)
54	ewv	1.7
55			return jm
56
57
58			def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
59			status = []
60			jmlist = jm_from_se_bdii(se)
61
62			for jm in jmlist:
63			jm += "-cms"
64
65	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEUniqueID=" + jm + "))' ", 'GlueCEStateStatus', bdii)
66	ewv	1.7
67			r = re.compile('^GlueCEStateStatus: (.*)')
68			for l in pout:
69			m = r.match(l)
70			if m:
71			status.append(m.groups()[0])
72
73			return status
74
75			def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
76	ewv	1.13	pout = runldapquery(" '(&(objectClass=GlueCEState)(GlueCEInfoHostName=" + ce + ")(GlueCEAccessControlBaseRule=VO:cms))' ", 'GlueCEStateStatus', bdii)
77	ewv	1.7
78			status = ''
79			r = re.compile('^GlueCEStateStatus: (.*)')
80			for l in pout:
81			m = r.match(l)
82			if m:
83			status = m.groups()[0]
84
85			return status
86
87			def concatoutput(pout):
88			output = ''
89			for l in pout:
90			if l == '':
91			output = output + LF
92			output = output + l + LF
93
94			return output
95
96			def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
97			# Get the Jobmanager list
98			jmlist = []
99
100	ewv	1.13	query = " '(\|"
101	ewv	1.7	for se in selist:
102	ewv	1.13	query = query + "(GlueCESEBindGroupSEUniqueID=" + se + ")"
103			query = query + ")' "
104	ewv	1.7
105			pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
106	ewv	1.8	r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.?)-(.)')
107
108	ewv	1.7	for l in pout:
109			m = r.match(l)
110			if m:
111			item = m.groups()[0]
112			if (jmlist.count(item) == 0):
113	ewv	1.8	jmlist.append(item)
114
115	ewv	1.10
116	ewv	1.13	query = " '(&(GlueCEAccessControlBaseRule=VO:cms)(\|"
117	ewv	1.8	for l in jmlist:
118	ewv	1.13	query += "(GlueCEInfoContactString=" + l + "-*)"
119	ewv	1.8
120	ewv	1.13	query += "))' "
121	ewv	1.8
122			pout = runldapquery(query, 'GlueCEInfoContactString', bdii)
123
124			r = re.compile('^GlueCEInfoContactString: (.:./jobmanager-.*)')
125			for l in pout:
126			m = r.match(l)
127			if m:
128			item = m.groups()[0]
129			if (jmlist.count(item) == 0):
130			jmlist.append(item)
131	ewv	1.7
132			return jmlist
133
134			def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
135	ewv	1.19	"""
136			Determine which hosts in the host_list are OSG sites.
137	ewv	1.7
138	ewv	1.19	Take a list of CE host names, and find all the associated clusters.
139			Take the resulting clusters and map them to sites. Any such site whose
140			description includes "OSG" is labeled an OSG site.
141
142			@param host_list: A list of host names which are CEs we want to consider.
143			@keyword bdii: The BDII instance to query
144			@return: A list of host names filtered from host_list which are OSG sites.
145			"""
146			siteUniqueID_CE_map = {}
147			results = sets.Set()
148			description_re = re.compile('^GlueSiteDescription: (OSG.*)')
149			siteid_re = re.compile('GlueForeignKey: GlueSiteUniqueID=(.*)')
150			ce_re = re.compile('^GlueForeignKey: GlueCEUniqueID=(.*):')
151
152			# Build the BDII query for all the hosts.
153			# This asks for all GlueClusters which are associated with one of the
154			# host names.
155			query = " '(&(objectClass=GlueCluster)(\|"
156	ewv	1.7	for h in host_list:
157	ewv	1.19	query += "(GlueForeignKey=GlueCEUniqueID=%s:*)" % h
158			query += "))' "
159	ewv	1.7
160	ewv	1.19	pout = runldapquery(query, 'GlueForeignKey', bdii)
161	ewv	1.7	output = concatoutput(pout)
162
163	ewv	1.19	# Now, we build a mapping from host to SiteUniqueID
164			stanzas = output.split(LF + LF)
165			for stanza in stanzas:
166			details = stanza.split(LF)
167			ces_tmp = sets.Set()
168			siteUniqueID = None
169			# Find out all the matching CEs and Sites with this cluster.
170			for detail in details:
171			m = ce_re.match(detail)
172			if m:
173			ces_tmp.add(m.groups()[0])
174			m = siteid_re.match(detail)
175			if m:
176			siteUniqueID = m.groups()[0]
177			if siteUniqueID:
178			siteUniqueID_CE_map[siteUniqueID] = ces_tmp
179
180			# Build a new query, this time for SiteUniqueIDs
181			query = " '(\|"
182			for site in siteUniqueID_CE_map:
183			query += "(GlueSiteUniqueID=%s)" % site
184			query += ")' "
185			pout = runldapquery(query, "GlueForeignKey GlueSiteDescription", bdii)
186			output = concatoutput(pout)
187
188			# See which resulting sites are OSG sites, and then add the
189			# corresponding CEs into the results set.
190	ewv	1.7	stanzas = output.split(LF + LF)
191			for stanza in stanzas:
192	ewv	1.19	isOsgSite = False
193			siteUniqueID = None
194	ewv	1.7	details = stanza.split(LF)
195	ewv	1.19	# We need to find the stanza's siteUniqueID and if the description
196			# is a "OSG Site". If it is, afterward add it to the results.
197	ewv	1.7	for detail in details:
198	ewv	1.19	m = siteid_re.search(detail)
199			if m:
200			siteUniqueID = m.groups()[0]
201			m = description_re.match(detail)
202	ewv	1.7	if m:
203	ewv	1.19	isOsgSite = True
204			if siteUniqueID and isOsgSite:
205			results.update(siteUniqueID_CE_map[siteUniqueID])
206			return list(results)
207	ewv	1.7
208	ewv	1.10	def getSoftwareAndArch2(host_list, software, arch, bdii='exp-bdii.cern.ch'):
209			results_list = []
210
211			# Find installed CMSSW versions and Architecture
212			software = 'VO-cms-' + software
213			arch = 'VO-cms-' + arch
214
215			query = "'(\|"
216
217			for h in host_list:
218			query += "(GlueCEInfoContactString=" + h + ")"
219			query += ")'"
220
221			pout = runldapquery(query, 'GlueForeignKey GlueCEInfoContactString', bdii)
222			r = re.compile('GlueForeignKey: GlueClusterUniqueID=(.*)')
223			s = re.compile('GlueCEInfoContactString: (.*)')
224
225			ClusterMap = {}
226			ClusterUniqueID = None
227			CEInfoContact = None
228
229			for l in pout:
230			m = r.match(l)
231			if m:
232			ClusterUniqueID = m.groups()[0]
233			m = s.match(l)
234			if m:
235			CEInfoContact = m.groups()[0]
236
237			if (ClusterUniqueID and CEInfoContact):
238			ClusterMap[ClusterUniqueID] = CEInfoContact
239			ClusterUniqueID = None
240			CEInfoContact = None
241
242			query = "'(\|"
243			for c in ClusterMap.keys():
244			query += "(GlueChunkKey=GlueClusterUniqueID="+c+")"
245			query += ")'"
246
247			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueChunkKey', bdii)
248			output = concatoutput(pout)
249
250			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
251			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
252			stanzas = output.split(LF + LF)
253			for stanza in stanzas:
254			software_installed = 0
255			architecture = 0
256			host = ''
257			details = stanza.split(LF)
258			for detail in details:
259			m = r.match(detail)
260			if m:
261			if (m.groups()[0] == software):
262			software_installed = 1
263			elif (m.groups()[0] == arch):
264			architecture = 1
265			m2 = s.match(detail)
266			if m2:
267			ClusterUniqueID = m2.groups()[0]
268			host = ClusterMap[ClusterUniqueID]
269
270			if ((software_installed == 1) and (architecture == 1)):
271			results_list.append(host)
272
273			return results_list
274	ewv	1.7
275			def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
276			results_list = []
277
278			# Find installed CMSSW versions and Architecture
279			software = 'VO-cms-' + software
280			arch = 'VO-cms-' + arch
281
282	ewv	1.13	query = " '(\|"
283	ewv	1.7	for h in host_list:
284	ewv	1.13	query = query + "(GlueChunkKey='GlueClusterUniqueID=" + h + "\')"
285			query = query + ")' "
286	ewv	1.7
287			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
288			output = concatoutput(pout)
289
290			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
291			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
292			stanzas = output.split(LF + LF)
293			for stanza in stanzas:
294			software_installed = 0
295			architecture = 0
296			host = ''
297			details = stanza.split(LF)
298			for detail in details:
299			m = r.match(detail)
300			if m:
301			if (m.groups()[0] == software):
302			software_installed = 1
303			elif (m.groups()[0] == arch):
304			architecture = 1
305			m2 = s.match(detail)
306			if m2:
307			host = m2.groups()[0]
308
309			if ((software_installed == 1) and (architecture == 1)):
310			results_list.append(host)
311
312			return results_list
313
314			def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
315			jminfo_list = []
316			host_list = []
317
318			stat = re.compile('^GlueCEStateStatus: (.*)')
319			host = re.compile('^GlueCEInfoHostName: (.*)')
320			wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
321			name = re.compile('^GlueCEUniqueID: (.*)')
322
323			jmlist = getJMListFromSEList(selist)
324	ewv	1.8
325	ewv	1.13	query = " '(&(objectClass=GlueCEState)(\|"
326	ewv	1.7	for jm in jmlist:
327	ewv	1.13	query = query + "(GlueCEUniqueID=" + jm + ")"
328			query = query + "))' "
329	ewv	1.7
330			pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
331			output = concatoutput(pout)
332
333			stanza_list = output.split(LF+LF)
334			for stanza in stanza_list:
335			if len(stanza) > 1:
336			status = 1
337			wait_jobs = 0
338			jmname = ''
339			hostname = 0
340			jminfo = {}
341
342			details = stanza.split(LF)
343			for det in details:
344			mhost = host.match(det)
345			if mhost: # hostname
346			host_list.append(mhost.groups()[0])
347			hostname = mhost.groups()[0]
348			mstat = stat.match(det)
349			if mstat: # e.g. Production
350			if not ((mstat.groups()[0] == 'Production') and (status == 1)):
351			status = 0
352			mwait = wait.match(det)
353			if mwait: # Waiting jobs
354			if (mwait.groups()[0] > wait_jobs):
355			wait_jobs = mwait.groups()[0]
356			mname = name.match(det)
357			if mname: # jm name
358			jmname = mname.groups()[0]
359
360			jminfo["name"] = jmname
361			jminfo["status"] = status
362			jminfo["waiting_jobs"] = wait_jobs
363			jminfo["host"] = hostname
364
365			jminfo_list.append(copy.deepcopy(jminfo))
366
367			# Narrow the list of host to include only OSG sites if requested
368	ewv	1.13	osg_list = isOSGSite([x['host'] for x in jminfo_list])
369			if not onlyOSG:
370			CElist = [x['name'] for x in jminfo_list]
371	ewv	1.7	else:
372	ewv	1.13	CElist = [x['name'] for x in jminfo_list if osg_list.count(x['host'])]
373	ewv	1.7
374			# Narrow the OSG host list to include only those with the specified software and architecture
375	ewv	1.10	# softarch_list = getSoftwareAndArch(osg_list, software, arch)
376			softarch_list = getSoftwareAndArch2(CElist, software, arch)
377	ewv	1.7
378			# remove any non-OSG sites from the list
379			jminfo_newlist = []
380	ewv	1.10
381	ewv	1.7	for item in jminfo_list:
382			for narrowed_item in softarch_list:
383	ewv	1.10	if (item['name'] == narrowed_item):
384	ewv	1.8	if (jminfo_newlist.count(item) == 0):
385			jminfo_newlist.append(item)
386	ewv	1.7
387			return jminfo_newlist
388
389			# This function is used to sort lists of dictionaries
390			def compare_by (fieldname):
391			def compare_two_dicts (a, b):
392			return cmp(int(a[fieldname]), int(b[fieldname]))
393			return compare_two_dicts
394
395			def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True):
396			jms = getJMInfo(selist, software, arch, bdii, onlyOSG)
397			# Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
398			jms.sort(compare_by('waiting_jobs'))
399			jmlist = []
400	ewv	1.8	r = re.compile('^(.:./jobmanager-.?)-(.)')
401	ewv	1.7	for jm in jms:
402	ewv	1.8	fullname = jm['name']
403			m = r.match(fullname)
404			if m:
405			name = m.groups()[0]
406			if (jmlist.count(name) == 0): jmlist.append(name)
407	ewv	1.7
408			return jmlist
409	ewv	1.16
410	ewv	1.17	def listAllCEs(software, arch, bdii='exp-bdii.cern.ch',onlyOSG=True):
411	ewv	1.16	''' List all GlueCEUniqueIDs that advertise support for CMS '''
412
413	ewv	1.18	# RE_cename = re.compile('^GlueCEUniqueID: (.:./jobmanager-.?)-(.)', re.IGNORECASE)
414	ewv	1.16	RE_cename = re.compile('^GlueCEUniqueID: (.*)', re.IGNORECASE)
415	ewv	1.17	hostSplit = re.compile(r'[^\w\.\-]')
416	ewv	1.16	filt = "'(&(GlueCEUniqueID=*)(GlueCEAccessControlBaseRule=VO:cms))'"
417			pout = runldapquery(filt, 'GlueCEUniqueID', bdii)
418	ewv	1.17	ceList = []
419			hostList = []
420	ewv	1.16	for l in pout:
421			m = RE_cename.match(l)
422			if m:
423			item = m.groups()[0]
424	ewv	1.17	hostname = hostSplit.split(item)[0]
425			if (ceList.count(item) == 0): ceList.append(item)
426			if (hostList.count(hostname) == 0): hostList.append(hostname)
427
428			if onlyOSG:
429			osgCEs = []
430			osgList = isOSGSite(hostList, bdii)
431			for ce in ceList:
432			hostname = hostSplit.split(ce)[0]
433			if hostname in osgList:
434			osgCEs.append(ce)
435			else:
436			osgCEs = ceList
437
438			softarch_list = getSoftwareAndArch2(osgCEs, software, arch)
439	ewv	1.16
440	ewv	1.18	shortCeList = [] # Convert to CE without queue
441			RE_short = re.compile('^(.:./jobmanager-.?)-(.)', re.IGNORECASE)
442			for ce in softarch_list:
443			m = RE_short.match(ce)
444			if m:
445			item = m.groups()[0]
446			if (shortCeList.count(item) == 0): shortCeList.append(item)
447
448			return shortCeList
449	ewv	1.16
450	ewv	1.15	def listAllSEs(bdii='exp-bdii.cern.ch'):
451	ewv	1.16	''' List all SEs that are bound to (CEs that advertise support for CMS) '''
452
453			RE_sename = re.compile('^GlueCESEBindGroupSEUniqueID: (.*)', re.IGNORECASE)
454	ewv	1.15	seList = []
455	ewv	1.16	filt = "'(\|"
456			ceList = listAllCEs(bdii)
457			for ce in ceList:
458			filtstring = '(GlueCESEBindGroupCEUniqueID=' + ce + ')'
459			filt += filtstring
460			filt += ")'"
461
462			pout = runldapquery(filt, 'GlueCESEBindGroupSEUniqueID', bdii)
463			for l in pout:
464			m = RE_sename.match(l)
465			if m:
466			item = m.groups()[0]
467			if (seList.count(item) == 0): seList.append(item)
468
469			return seList
470	ewv	1.15
471	ewv	1.7
472			if __name__ == '__main__':
473	ewv	1.15	import pprint
474	ewv	1.16	# pprint.pprint(listAllSEs('uscmsbd2.fnal.gov'))
475	ewv	1.19	# pprint.pprint(listAllCEs( "CMSSW_1_6_11", "slc4_ia32_gcc345", onlyOSG=False))
476	ewv	1.17	# pprint.pprint(listAllCEs(onlyOSG=False))
477	ewv	1.16
478	ewv	1.19	# seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
479			seList = ['srm.unl.edu', 'dcache.rcac.purdue.edu']
480			jmlist = getJobManagerList(seList, "CMSSW_1_6_11", "slc4_ia32_gcc345")
481			for jm in jmlist:
482			print jm
483	ewv	1.10	# print jm_from_se_bdii(sys.argv[1])