| 1 |
#---------------------------------------------------------------------------------------------------
|
| 2 |
# Python Module File to translate storage/computing elements into meta site names
|
| 3 |
#
|
| 4 |
# Author: C.Paus (Jun 16, 2010)
|
| 5 |
#---------------------------------------------------------------------------------------------------
|
| 6 |
import os,sys,re,string
|
| 7 |
|
| 8 |
#---------------------------------------------------------------------------------------------------
|
| 9 |
"""
|
| 10 |
Class: Translator(table)
|
| 11 |
Each SubTask in CRAB can be described through this class
|
| 12 |
"""
|
| 13 |
#---------------------------------------------------------------------------------------------------
|
| 14 |
class Translator:
|
| 15 |
"Translator for the storage and computing elements to the "
|
| 16 |
# variable to be determined
|
| 17 |
ceTable = 'undefined' # compute elements translation table
|
| 18 |
seTable = 'undefined' # storage elements translation table
|
| 19 |
allSites = 'undefined' # comma separated list of all sites
|
| 20 |
prefSitesTable = 'undefined' # comma separated list of preferred sites
|
| 21 |
ces = {}
|
| 22 |
ses = {}
|
| 23 |
preferredSitesList = []
|
| 24 |
preferredSites = []
|
| 25 |
#-----------------------------------------------------------------------------------------------
|
| 26 |
# constructor to connect with existing setup
|
| 27 |
#-----------------------------------------------------------------------------------------------
|
| 28 |
def __init__(self,ceTable,seTable,prefSitesTable):
|
| 29 |
self.ceTable = ceTable
|
| 30 |
self.seTable = seTable
|
| 31 |
self.prefSitesTable = prefSitesTable
|
| 32 |
if os.path.exists(self.seTable):
|
| 33 |
self.ses = self.readTable(self.seTable)
|
| 34 |
else:
|
| 35 |
print ' WARNING -- SE table file not found.'
|
| 36 |
if os.path.exists(self.ceTable):
|
| 37 |
self.ces = self.readTable(self.ceTable)
|
| 38 |
else:
|
| 39 |
print ' WARNING -- CE table file not found.'
|
| 40 |
if os.path.exists(self.prefSitesTable):
|
| 41 |
self.preferredSitesList = self.readList(self.prefSitesTable)
|
| 42 |
else:
|
| 43 |
print ' WARNING -- prefSites list file not found.'
|
| 44 |
|
| 45 |
#self.show()
|
| 46 |
|
| 47 |
def readList(self,file):
|
| 48 |
list = []
|
| 49 |
cmd = 'grep -v ^# ' + file + ' | tr -d \' \''
|
| 50 |
print ' Loading list: ' + cmd
|
| 51 |
for line in os.popen(cmd).readlines():
|
| 52 |
line = line[:-1]
|
| 53 |
# decode the storage directory name
|
| 54 |
names = line.split(" ")
|
| 55 |
list.append(names[0])
|
| 56 |
|
| 57 |
return list
|
| 58 |
|
| 59 |
def readTable(self,file):
|
| 60 |
table = {}
|
| 61 |
cmd = 'grep -v ^# ' + file + ' | tr -d \' \''
|
| 62 |
print ' Loading table: ' + cmd
|
| 63 |
for line in os.popen(cmd).readlines():
|
| 64 |
line = line[:-1]
|
| 65 |
# decode the storage directory name
|
| 66 |
names = line.split(":")
|
| 67 |
if names[1] in table:
|
| 68 |
print ' Site already in tables %s (%s)'%(names[1],names[0])
|
| 69 |
table[names[1]] = names[0]
|
| 70 |
|
| 71 |
return table
|
| 72 |
|
| 73 |
def translateSes(self,seString):
|
| 74 |
print ' translating SE sites: ' + seString
|
| 75 |
sites = seString.split(",")
|
| 76 |
newString = ''
|
| 77 |
#print ' Looping through sites %s'%(seString)
|
| 78 |
for site in sites:
|
| 79 |
if site in self.ses:
|
| 80 |
#print ' -> %s == %s'%(site,self.ses[site])
|
| 81 |
if newString == '':
|
| 82 |
newString = self.ses[site]
|
| 83 |
else:
|
| 84 |
newString = "%s,%s"%(newString,self.ses[site])
|
| 85 |
else:
|
| 86 |
print ' dropping %s because it is not in our table'%(site)
|
| 87 |
|
| 88 |
#print 'Translated: %s'%(newString)
|
| 89 |
self.allSites = newString
|
| 90 |
return newString
|
| 91 |
|
| 92 |
def translateCes(self,ceString):
|
| 93 |
print ' translating CE sites: ' + ceString
|
| 94 |
sites = ceString.split(",")
|
| 95 |
newString = ''
|
| 96 |
#print ' Looping through sites %s'%(ceString)
|
| 97 |
for site in sites:
|
| 98 |
if site in self.ces:
|
| 99 |
#print ' -> %s == %s'%(site,self.ces[site])
|
| 100 |
if newString == '':
|
| 101 |
newString = self.ces[site]
|
| 102 |
else:
|
| 103 |
newString = "%s,%s"%(newString,self.ces[site])
|
| 104 |
else:
|
| 105 |
print ' dropping %s because it is not in our table'%(site)
|
| 106 |
|
| 107 |
#print 'Translated: %s'%(newString)
|
| 108 |
self.allSites = newString
|
| 109 |
return newString
|
| 110 |
|
| 111 |
def selectPreferred(self):
|
| 112 |
#preferredSitesList = ['T2_US_MIT','T2_US_Wisconsin']
|
| 113 |
self.preferredSites = ''
|
| 114 |
for site in self.preferredSitesList:
|
| 115 |
if re.search(site,self.allSites):
|
| 116 |
if self.preferredSites != '':
|
| 117 |
self.preferredSites = self.preferredSites + ','
|
| 118 |
self.preferredSites = self.preferredSites + site
|
| 119 |
|
| 120 |
if self.preferredSites != '':
|
| 121 |
print ' '
|
| 122 |
print ' ' + self.allSites
|
| 123 |
print ' INFO -- ' + self.preferredSites + ' found a preferred site (reducing).'
|
| 124 |
print ' '
|
| 125 |
else:
|
| 126 |
#print ' '
|
| 127 |
#print ' INFO -- no preferred site in the list. Keeping all sites.'
|
| 128 |
#print ' '
|
| 129 |
self.preferredSites = self.allSites
|
| 130 |
return self.preferredSites
|
| 131 |
|
| 132 |
#-----------------------------------------------------------------------------------------------
|
| 133 |
# present the current crab subtask
|
| 134 |
#-----------------------------------------------------------------------------------------------
|
| 135 |
def show(self):
|
| 136 |
print ' Translator (CE Table: %s, SE Table: %s) ===='%(self.ceTable,self.seTable)
|
| 137 |
print ' ==== Compute Elements'
|
| 138 |
for name,hostname in self.ces.iteritems():
|
| 139 |
print ' Site: %s --> CE: %s'%(name,hostname)
|
| 140 |
print ' ==== Storage Elements'
|
| 141 |
for name,hostname in self.ses.iteritems():
|
| 142 |
print ' Site: %s --> SE: %s'%(name,hostname)
|
