| 1 |
|
#!/usr/bin/env python |
| 2 |
< |
import re, sys |
| 2 |
> |
import re |
| 3 |
> |
import sys |
| 4 |
|
import ldap |
| 5 |
|
|
| 6 |
|
DEBUG = 0 |
| 7 |
+ |
map_source = {'ceList': [], 'bdii': ''} |
| 8 |
|
ce_to_cluster_map = {} |
| 9 |
|
cluster_to_site_map = {} |
| 10 |
|
|
| 17 |
|
|
| 18 |
|
bdiiuri = 'ldap://' + bdii + ':2170' |
| 19 |
|
l = ldap.initialize(bdiiuri) |
| 20 |
< |
|
| 20 |
> |
|
| 21 |
|
l.simple_bind_s('', '') |
| 22 |
|
|
| 23 |
|
base = "o=grid" |
| 25 |
|
timeout = 0 |
| 26 |
|
result_set = [] |
| 27 |
|
filter = filter.strip("'") |
| 28 |
< |
|
| 28 |
> |
|
| 29 |
|
try: |
| 30 |
|
result_id = l.search(base, scope, filter, attribute) |
| 31 |
|
while 1: |
| 45 |
|
""" |
| 46 |
|
Given a list of SE FQDNs, return list of CEUniqueIDs that advertise CMS |
| 47 |
|
support and are in Production, |
| 48 |
< |
sorted by number of waiting jobs in descending order |
| 48 |
> |
sorted by number of waiting jobs in descending order |
| 49 |
|
""" |
| 50 |
|
jmlist = [] |
| 51 |
|
|
| 75 |
|
|
| 76 |
|
jminfo_list.sort(compare_by('waiting_jobs')) |
| 77 |
|
jmlist = [x['ce'] for x in jminfo_list] |
| 76 |
– |
|
| 78 |
|
return jmlist |
| 79 |
|
|
| 80 |
< |
def generateMaps(ce_list, bdii='exp-bdii.cern.ch'): |
| 80 |
> |
def generateMaps(ceList, bdii='exp-bdii.cern.ch'): |
| 81 |
|
""" |
| 82 |
|
Generate maps of CE to Cluster and Cluster to Site as the globals |
| 83 |
|
ce_to_cluster_map, cluster_to_site_map |
| 84 |
< |
|
| 85 |
< |
ce_list: list of GlueCEUniqueIDs |
| 84 |
> |
|
| 85 |
> |
ceList: list of GlueCEUniqueIDs |
| 86 |
|
bdii: BDII instance to query |
| 87 |
|
""" |
| 88 |
< |
# if ce_to_cluster_map: return |
| 89 |
< |
query = buildOrQuery('GlueCEUniqueID', ce_list) |
| 88 |
> |
if (ceList == map_source['ceList'] |
| 89 |
> |
and bdii == map_source['bdii']): return |
| 90 |
> |
|
| 91 |
> |
query = buildOrQuery('GlueCEUniqueID', ceList) |
| 92 |
|
|
| 93 |
|
pout = runldapquery(query, 'GlueCEUniqueID GlueForeignKey', bdii) |
| 94 |
|
|
| 98 |
|
clusterid = x[0][1]['GlueForeignKey'][0] |
| 99 |
|
m = r.match(clusterid) |
| 100 |
|
if m: ce_to_cluster_map[host] = m.groups()[0] |
| 101 |
< |
|
| 101 |
> |
|
| 102 |
|
query = "(&(objectClass=GlueCluster)" |
| 103 |
|
query += buildOrQuery('GlueClusterUniqueID', ce_to_cluster_map.values()) |
| 104 |
|
query += ")" |
| 105 |
< |
|
| 105 |
> |
|
| 106 |
|
pout = runldapquery(query, 'GlueClusterUniqueID GlueForeignKey', bdii) |
| 107 |
|
r = re.compile('^GlueSiteUniqueID=(.*)') |
| 108 |
|
for x in pout: |
| 114 |
|
site = m.groups()[0] |
| 115 |
|
cluster_to_site_map[cluster] = site |
| 116 |
|
|
| 117 |
+ |
# cache the list sources |
| 118 |
+ |
map_source['ceList'] = ceList |
| 119 |
+ |
map_source['bdii'] = bdii |
| 120 |
+ |
|
| 121 |
+ |
if (DEBUG): print 40*'*', 'exit generateMaps', 40*'*' |
| 122 |
|
def buildOrQuery(gluekey, list): |
| 123 |
|
""" |
| 124 |
|
Returns a nugget of LDAP requesting the OR of all items |
| 128 |
|
query = "(|" |
| 129 |
|
for x in list: |
| 130 |
|
query += "(%s=%s)" % (gluekey, x) |
| 131 |
< |
query += ")" |
| 131 |
> |
query += ")" |
| 132 |
|
return query |
| 133 |
|
|
| 134 |
|
def isOSGSite(host_list, bdii='exp-bdii.cern.ch'): |
| 135 |
|
""" |
| 136 |
|
Given a list of CEs, return only the ones which belong to OSG sites |
| 137 |
|
""" |
| 138 |
< |
if (not ce_to_cluster_map): generateMaps(host_list, bdii) |
| 138 |
> |
generateMaps(host_list, bdii) |
| 139 |
|
|
| 140 |
|
query = buildOrQuery('GlueSiteUniqueID', cluster_to_site_map.values()) |
| 141 |
|
pout = runldapquery(query, 'GlueSiteUniqueID GlueSiteDescription', bdii) |
| 162 |
|
Given a list of CEs, return only those that match a given software |
| 163 |
|
and architecture tag |
| 164 |
|
""" |
| 165 |
< |
if (not ce_to_cluster_map): generateMaps(host_list, bdii) |
| 165 |
> |
generateMaps(host_list, bdii) |
| 166 |
|
|
| 167 |
|
results_list = [] |
| 168 |
|
software = 'VO-cms-' + software |
| 184 |
|
|
| 185 |
|
return results_list |
| 186 |
|
|
| 187 |
< |
|
| 187 |
> |
|
| 188 |
|
def getJobManagerList(selist, software, arch, bdii='exp-bdii.cern.ch', onlyOSG=True): |
| 189 |
|
""" |
| 190 |
|
Given a list of SE FQDNs, return list of CEUniqueIDs that advertise CMS |
| 193 |
|
|
| 194 |
|
If OnlyOSG is True, return only OSG Sites. |
| 195 |
|
""" |
| 196 |
< |
|
| 196 |
> |
|
| 197 |
|
jmlist = getJMListFromSEList(selist, bdii) |
| 198 |
< |
generateMaps(jmlist, bdii) |
| 199 |
< |
if (onlyOSG): jmlist = isOSGSite(jmlist, bdii) |
| 200 |
< |
jmlist = getSoftwareAndArch(jmlist, software, arch, bdii) |
| 201 |
< |
res = removeQueues(jmlist) |
| 198 |
> |
jmlist = filterCE(jmlist, software, arch, bdii, onlyOSG) |
| 199 |
> |
|
| 200 |
> |
def filterCE(ceList, software, arch, bdii, onlyOSG): |
| 201 |
> |
""" |
| 202 |
> |
Given a list of CEUniqueIDs, filter out only the ones with given |
| 203 |
> |
software, arch, and if it belongs to an OSG site. |
| 204 |
> |
""" |
| 205 |
> |
generateMaps(ceList, bdii) |
| 206 |
> |
if (onlyOSG): ceList = isOSGSite(ceList, bdii) |
| 207 |
> |
ceList = getSoftwareAndArch(ceList, software, arch, bdii) |
| 208 |
> |
res = removeQueues(ceList) |
| 209 |
|
return res |
| 210 |
< |
|
| 210 |
> |
|
| 211 |
|
def removeQueues(celist): |
| 212 |
|
""" |
| 213 |
|
Given a list of CEUniqueIDs, return a list of jobmanager contact |
| 222 |
|
if (jmlist.count(item) == 0): |
| 223 |
|
jmlist.append(item) |
| 224 |
|
return jmlist |
| 225 |
< |
|
| 226 |
< |
def listAllCEs(bdii='exp-bdii.cern.ch'): |
| 225 |
> |
|
| 226 |
> |
def listAllCEs(software='', arch='', onlyOSG=False, bdii='exp-bdii.cern.ch'): |
| 227 |
|
''' List all GlueCEUniqueIDs that advertise support for CMS ''' |
| 228 |
< |
|
| 228 |
> |
|
| 229 |
|
RE_cename = re.compile('^GlueCEUniqueID: (.*)', re.IGNORECASE) |
| 230 |
|
filt = "'(&(GlueCEUniqueID=*)(GlueCEAccessControlBaseRule=VO:cms))'" |
| 231 |
|
res = runldapquery(filt, 'GlueCEUniqueID', bdii) |
| 232 |
|
ceList = [x[0][1]['GlueCEUniqueID'][0] for x in res] |
| 233 |
+ |
|
| 234 |
+ |
if (software or arch or onlyOSG): |
| 235 |
+ |
ceList = filterCE(ceList, software, arch, bdii, onlyOSG) |
| 236 |
+ |
|
| 237 |
|
return ceList |
| 238 |
< |
|
| 238 |
> |
|
| 239 |
|
def listAllSEs(bdii='exp-bdii.cern.ch'): |
| 240 |
|
''' List all SEs that are bound to CEs that advertise support for CMS ''' |
| 241 |
< |
|
| 241 |
> |
|
| 242 |
|
RE_sename = re.compile('^GlueCESEBindGroupSEUniqueID: (.*)', re.IGNORECASE) |
| 243 |
|
seList = [] |
| 244 |
< |
ceList = listAllCEs(bdii) |
| 244 |
> |
ceList = listAllCEs(bdii=bdii) |
| 245 |
|
|
| 246 |
|
query = buildOrQuery('GlueCESEBindGroupCEUniqueID', ceList) |
| 247 |
|
res = runldapquery(query, 'GlueCESEBindGroupSEUniqueID', bdii) |
| 254 |
|
pass |
| 255 |
|
|
| 256 |
|
if (seList.count(item) == 0): seList.append(item) |
| 257 |
< |
return seList |
| 257 |
> |
return seList |
