CRAB/python/osg_bdii.py

#!/usr/bin/env python
import re, popen2, sys, copy

CR = '\r'
LF = '\n'
CRLF = CR+LF

def unwraplines(wrapped_list):
    r = re.compile('^ (.*)$')
    unwrapped_list = []
    for l in wrapped_list:
        m = r.match(l)
        if m:
            unwrapped_list[-1] += m.groups()[0]
        else:
            unwrapped_list.append(l.rstrip())

    return unwrapped_list


def runldapquery(filter, attribute, bdii):
    command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
    command += filter + ' ' + attribute
    pout,pin,perr = popen2.popen3(command)

    pout = pout.readlines()
    p = perr.readlines()

    pout = unwraplines(pout)
    if (p):
        for l in p: print l
        raise RuntimeError('ldapsearch call failed')

    return pout

def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
    se = '\'' + se + '\''
    pout = runldapquery(''' '(GlueCESEBindGroupSEUniqueID=''' + se + ''')' ''', 'GlueCESEBindGroupCEUniqueID', bdii)

    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*)-cms')
    jm = []
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jm.count(item) == 0):
                jm.append(m.groups()[0])

    return jm


def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
    status = []
    jmlist = jm_from_se_bdii(se)

    for jm in jmlist:
        jm += "-cms"

        pout = runldapquery(''' '(&(objectClass=GlueCEState)(GlueCEUniqueID=''' + jm + '''))' ''', 'GlueCEStateStatus', bdii)

        r = re.compile('^GlueCEStateStatus: (.*)')
        for l in pout:
            m = r.match(l)
            if m:
                status.append(m.groups()[0])

    return status

def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
    pout = runldapquery(''' '(&(objectClass=GlueCEState)(GlueCEInfoHostName=''' + ce + ''')(GlueCEAccessControlBaseRule=VO:cms))' ''', 'GlueCEStateStatus', bdii)

    status = ''
    r = re.compile('^GlueCEStateStatus: (.*)')
    for l in pout:
        m = r.match(l)
        if m:
            status = m.groups()[0]

    return status

def concatoutput(pout):
    output = ''
    for l in pout:
        if l == '':
            output = output + LF
        output = output + l + LF

    return output

def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
    # Get the Jobmanager list
    jmlist = []

    query = ''' '(|'''
    for se in selist:
        query = query + '''(GlueCESEBindGroupSEUniqueID=''' + se + ''')'''
    query = query + ''')' '''

    pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
    r = re.compile('^GlueCESEBindGroupCEUniqueID: (.*:.*/jobmanager-.*)-cms')
    for l in pout:
        m = r.match(l)
        if m:
            item = m.groups()[0]
            if (jmlist.count(item) == 0):
                jmlist.append(m.groups()[0])

    return jmlist

def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
    results_list = []
    r = re.compile('^GlueSiteDescription: (OSG.*)')
    s = re.compile('^GlueSiteUniqueID: (.*)')

    query = ''' '(|'''
    for h in host_list:
        query = query + '''(GlueSiteUniqueID=''' + h + ''')'''
    query = query + ''')' GlueSiteDescription'''

    pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
    output = concatoutput(pout)

    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        osg = 0
        host = ""
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            n = s.match(detail)
            if m:
                osg = 1
            if n:
                host = n.groups()[0]
        if (osg == 1):
            results_list.append(host)

    return results_list


def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
    results_list = []

    # Find installed CMSSW versions and Architecture
    software = 'VO-cms-' + software
    arch = 'VO-cms-' + arch

    query = ''' '(|'''
    for h in host_list:
        query = query + '''(GlueChunkKey='GlueClusterUniqueID=''' + h + '''\')'''
    query = query + ''')' '''

    pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
    output = concatoutput(pout)

    r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
    s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
    stanzas = output.split(LF + LF)
    for stanza in stanzas:
        software_installed = 0
        architecture = 0
        host = ''
        details = stanza.split(LF)
        for detail in details:
            m = r.match(detail)
            if m:
                if (m.groups()[0] == software):
                    software_installed = 1
                elif (m.groups()[0] == arch):
                    architecture = 1
            m2 = s.match(detail)
            if m2:
                host = m2.groups()[0]

        if ((software_installed == 1) and (architecture == 1)):
            results_list.append(host)

    return results_list

def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch'):
    jminfo_list = []
    host_list = []

    stat = re.compile('^GlueCEStateStatus: (.*)')
    host = re.compile('^GlueCEInfoHostName: (.*)')
    wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
    name = re.compile('^GlueCEUniqueID: (.*)')

    jmlist = getJMListFromSEList(selist)
    query = ''' '(&(objectClass=GlueCEState)(|'''
    for jm in jmlist:
        query = query + '''(GlueCEUniqueID=''' + jm + '''-cms)'''
    query = query + '''))' '''

    pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
    output = concatoutput(pout)

    stanza_list = output.split(LF+LF)
    for stanza in stanza_list:
        if len(stanza) > 1:
            status = 1
            wait_jobs = 0
            jmname = ''
            hostname = 0
            jminfo = {}

            details = stanza.split(LF)
            for det in details:
                mhost = host.match(det)
                if mhost: # hostname
                    host_list.append(mhost.groups()[0])
                    hostname = mhost.groups()[0]
                mstat = stat.match(det)
                if mstat: # e.g. Production
                    if not ((mstat.groups()[0] == 'Production') and (status == 1)):
                        status = 0
                mwait = wait.match(det)
                if mwait: # Waiting jobs
                    if (mwait.groups()[0] > wait_jobs):
                        wait_jobs = mwait.groups()[0]
                mname = name.match(det)
                if mname: # jm name
                    jmname = mname.groups()[0]

            jminfo["name"] = jmname
            jminfo["status"] = status
            jminfo["waiting_jobs"] = wait_jobs
            jminfo["host"] = hostname

            jminfo_list.append(copy.deepcopy(jminfo))

    # Narrow the list of host to include only OSG sites
    osg_list = isOSGSite(host_list)

    # Narrow the OSG host list to include only those with the specified software and architecture
    softarch_list = getSoftwareAndArch(osg_list, software, arch)

    # remove any non-OSG sites from the list
    jminfo_newlist = []
    for item in jminfo_list:
        for narrowed_item in softarch_list:
            if (item["host"] == narrowed_item):
                jminfo_newlist.append(item)

    return jminfo_newlist

# This function is used to sort lists of dictionaries
def compare_by (fieldname):
    def compare_two_dicts (a, b):
        return cmp(int(a[fieldname]), int(b[fieldname]))
    return compare_two_dicts

def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch'):
    jms = getJMInfo(selist, software, arch)
    # Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
    jms.sort(compare_by('waiting_jobs'))
    jmlist = []
    for jm in jms:
        jmlist.append(jm["name"][:-4])

    return jmlist

if __name__ == '__main__':
    seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
    jmlist =  getJobManagerList(seList, "CMSSW_2_0_0", "slc4_ia32_gcc345")
    for jm in jmlist:
        print jm
#   print jm_from_se_bdii(sys.argv[1])

Revision:	1.5
Committed:	Wed Apr 23 18:20:01 2008 UTC (17 years ago) by ewv
Content type:	text/x-python
Branch:	MAIN
Changes since 1.4:	+3 -7 lines
Log Message:	Fix iterator bug in osg_bdii, remove Condor specific WN script
#	User	Rev	Content
1	ewv	1.4	#!/usr/bin/env python
2			import re, popen2, sys, copy
3
4			CR = '\r'
5			LF = '\n'
6			CRLF = CR+LF
7
8			def unwraplines(wrapped_list):
9			r = re.compile('^ (.*)$')
10			unwrapped_list = []
11			for l in wrapped_list:
12			m = r.match(l)
13			if m:
14			unwrapped_list[-1] += m.groups()[0]
15			else:
16			unwrapped_list.append(l.rstrip())
17
18			return unwrapped_list
19
20
21			def runldapquery(filter, attribute, bdii):
22			command = 'ldapsearch -xLLL -p 2170 -h ' + bdii + ' -b o=grid '
23			command += filter + ' ' + attribute
24			pout,pin,perr = popen2.popen3(command)
25
26			pout = pout.readlines()
27			p = perr.readlines()
28
29			pout = unwraplines(pout)
30			if (p):
31			for l in p: print l
32			raise RuntimeError('ldapsearch call failed')
33
34			return pout
35
36			def jm_from_se_bdii(se, bdii='exp-bdii.cern.ch'):
37			se = '\'' + se + '\''
38			pout = runldapquery(''' '(GlueCESEBindGroupSEUniqueID=''' + se + ''')' ''', 'GlueCESEBindGroupCEUniqueID', bdii)
39
40			r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.*)-cms')
41			jm = []
42			for l in pout:
43			m = r.match(l)
44			if m:
45			item = m.groups()[0]
46			if (jm.count(item) == 0):
47			jm.append(m.groups()[0])
48
49			return jm
50
51
52			def cestate_from_se_bdii(se, bdii='exp-bdii.cern.ch' ):
53			status = []
54			jmlist = jm_from_se_bdii(se)
55
56			for jm in jmlist:
57			jm += "-cms"
58
59			pout = runldapquery(''' '(&(objectClass=GlueCEState)(GlueCEUniqueID=''' + jm + '''))' ''', 'GlueCEStateStatus', bdii)
60
61			r = re.compile('^GlueCEStateStatus: (.*)')
62			for l in pout:
63			m = r.match(l)
64			if m:
65			status.append(m.groups()[0])
66
67			return status
68
69			def cestate_from_ce_bdii(ce, bdii='exp-bdii.cern.ch'):
70			pout = runldapquery(''' '(&(objectClass=GlueCEState)(GlueCEInfoHostName=''' + ce + ''')(GlueCEAccessControlBaseRule=VO:cms))' ''', 'GlueCEStateStatus', bdii)
71
72			status = ''
73			r = re.compile('^GlueCEStateStatus: (.*)')
74			for l in pout:
75			m = r.match(l)
76			if m:
77			status = m.groups()[0]
78
79			return status
80
81			def concatoutput(pout):
82			output = ''
83			for l in pout:
84			if l == '':
85			output = output + LF
86			output = output + l + LF
87
88			return output
89
90			def getJMListFromSEList(selist, bdii='exp-bdii.cern.ch'):
91			# Get the Jobmanager list
92			jmlist = []
93
94			query = ''' '(\|'''
95			for se in selist:
96			query = query + '''(GlueCESEBindGroupSEUniqueID=''' + se + ''')'''
97			query = query + ''')' '''
98
99			pout = runldapquery(query, 'GlueCESEBindGroupCEUniqueID', bdii)
100			r = re.compile('^GlueCESEBindGroupCEUniqueID: (.:./jobmanager-.*)-cms')
101			for l in pout:
102			m = r.match(l)
103			if m:
104			item = m.groups()[0]
105			if (jmlist.count(item) == 0):
106			jmlist.append(m.groups()[0])
107
108			return jmlist
109
110			def isOSGSite(host_list, bdii='exp-bdii.cern.ch'):
111			results_list = []
112			r = re.compile('^GlueSiteDescription: (OSG.*)')
113			s = re.compile('^GlueSiteUniqueID: (.*)')
114
115			query = ''' '(\|'''
116			for h in host_list:
117			query = query + '''(GlueSiteUniqueID=''' + h + ''')'''
118			query = query + ''')' GlueSiteDescription'''
119
120			pout = runldapquery(query, 'GlueSubClusterUniqueID GlueSiteUniqueID', bdii)
121			output = concatoutput(pout)
122
123			stanzas = output.split(LF + LF)
124			for stanza in stanzas:
125			osg = 0
126			host = ""
127			details = stanza.split(LF)
128			for detail in details:
129			m = r.match(detail)
130			n = s.match(detail)
131			if m:
132			osg = 1
133			if n:
134			host = n.groups()[0]
135			if (osg == 1):
136			results_list.append(host)
137
138			return results_list
139
140
141			def getSoftwareAndArch(host_list, software, arch, bdii='exp-bdii.cern.ch'):
142			results_list = []
143
144			# Find installed CMSSW versions and Architecture
145			software = 'VO-cms-' + software
146			arch = 'VO-cms-' + arch
147
148			query = ''' '(\|'''
149			for h in host_list:
150			query = query + '''(GlueChunkKey='GlueClusterUniqueID=''' + h + '''\')'''
151			query = query + ''')' '''
152
153			pout = runldapquery(query, 'GlueHostApplicationSoftwareRunTimeEnvironment GlueSubClusterUniqueID GlueChunkKey', bdii)
154			output = concatoutput(pout)
155
156			r = re.compile('^GlueHostApplicationSoftwareRunTimeEnvironment: (.*)')
157			s = re.compile('^GlueChunkKey: GlueClusterUniqueID=(.*)')
158			stanzas = output.split(LF + LF)
159			for stanza in stanzas:
160			software_installed = 0
161			architecture = 0
162			host = ''
163			details = stanza.split(LF)
164			for detail in details:
165			m = r.match(detail)
166			if m:
167			if (m.groups()[0] == software):
168			software_installed = 1
169			elif (m.groups()[0] == arch):
170			architecture = 1
171			m2 = s.match(detail)
172			if m2:
173			host = m2.groups()[0]
174
175			if ((software_installed == 1) and (architecture == 1)):
176			results_list.append(host)
177
178			return results_list
179
180			def getJMInfo(selist, software, arch, bdii='exp-bdii.cern.ch'):
181			jminfo_list = []
182			host_list = []
183
184			stat = re.compile('^GlueCEStateStatus: (.*)')
185			host = re.compile('^GlueCEInfoHostName: (.*)')
186			wait = re.compile('^GlueCEStateWaitingJobs: (.*)')
187			name = re.compile('^GlueCEUniqueID: (.*)')
188
189			jmlist = getJMListFromSEList(selist)
190			query = ''' '(&(objectClass=GlueCEState)(\|'''
191			for jm in jmlist:
192			query = query + '''(GlueCEUniqueID=''' + jm + '''-cms)'''
193			query = query + '''))' '''
194
195			pout = runldapquery(query, 'GlueCEUniqueID GlueCEStateStatus GlueCEInfoHostName GlueCEStateWaitingJobs GlueCEStateFreeJobSlots', bdii)
196			output = concatoutput(pout)
197
198			stanza_list = output.split(LF+LF)
199			for stanza in stanza_list:
200			if len(stanza) > 1:
201			status = 1
202			wait_jobs = 0
203			jmname = ''
204			hostname = 0
205			jminfo = {}
206
207			details = stanza.split(LF)
208			for det in details:
209			mhost = host.match(det)
210			if mhost: # hostname
211			host_list.append(mhost.groups()[0])
212			hostname = mhost.groups()[0]
213			mstat = stat.match(det)
214			if mstat: # e.g. Production
215			if not ((mstat.groups()[0] == 'Production') and (status == 1)):
216			status = 0
217			mwait = wait.match(det)
218			if mwait: # Waiting jobs
219			if (mwait.groups()[0] > wait_jobs):
220			wait_jobs = mwait.groups()[0]
221			mname = name.match(det)
222			if mname: # jm name
223			jmname = mname.groups()[0]
224
225			jminfo["name"] = jmname
226			jminfo["status"] = status
227			jminfo["waiting_jobs"] = wait_jobs
228			jminfo["host"] = hostname
229
230			jminfo_list.append(copy.deepcopy(jminfo))
231
232			# Narrow the list of host to include only OSG sites
233			osg_list = isOSGSite(host_list)
234
235			# Narrow the OSG host list to include only those with the specified software and architecture
236			softarch_list = getSoftwareAndArch(osg_list, software, arch)
237
238			# remove any non-OSG sites from the list
239	ewv	1.5	jminfo_newlist = []
240	ewv	1.4	for item in jminfo_list:
241			for narrowed_item in softarch_list:
242			if (item["host"] == narrowed_item):
243	ewv	1.5	jminfo_newlist.append(item)
244	ewv	1.4
245	ewv	1.5	return jminfo_newlist
246	ewv	1.4
247			# This function is used to sort lists of dictionaries
248			def compare_by (fieldname):
249			def compare_two_dicts (a, b):
250			return cmp(int(a[fieldname]), int(b[fieldname]))
251			return compare_two_dicts
252
253			def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch'):
254			jms = getJMInfo(selist, software, arch)
255			# Sort by waiting_jobs field and return the jobmanager with the least waiting jobs
256			jms.sort(compare_by('waiting_jobs'))
257			jmlist = []
258			for jm in jms:
259			jmlist.append(jm["name"][:-4])
260
261			return jmlist
262
263			if __name__ == '__main__':
264			seList = ['ccsrm.in2p3.fr', 'cmssrm.hep.wisc.edu', 'pccms2.cmsfarm1.ba.infn.it', 'polgrid4.in2p3.fr', 'srm-disk.pic.es', 'srm.ciemat.es', 'srm.ihepa.ufl.edu', 't2data2.t2.ucsd.edu']
265			jmlist = getJobManagerList(seList, "CMSSW_2_0_0", "slc4_ia32_gcc345")
266			for jm in jmlist:
267			print jm
268			# print jm_from_se_bdii(sys.argv[1])
269