Processing/python/translator.py

#---------------------------------------------------------------------------------------------------
# Python Module File to translate storage/computing elements into meta site names
#
# Author: C.Paus                                                                      (Jun 16, 2010)
#---------------------------------------------------------------------------------------------------
import os,sys,re,string

#---------------------------------------------------------------------------------------------------
"""
Class:  Translator(table)
Each SubTask in CRAB can be described through this class
"""
#---------------------------------------------------------------------------------------------------
class Translator:
    "Translator for the storage and computing elements to the "
    # variable to be determined
    ceTable            = 'undefined' # compute elements translation table
    seTable            = 'undefined' # storage elements translation table
    allSites           = 'undefined' # comma separated list of all sites
    prefSitesTable     = 'undefined' # comma separated list of preferred sites
    ces                = {}
    ses                = {}
    preferredSitesList = []
    preferredSites     = []
    #-----------------------------------------------------------------------------------------------
    # constructor to connect with existing setup
    #-----------------------------------------------------------------------------------------------
    def __init__(self,ceTable,seTable,prefSitesTable):
        self.ceTable        = ceTable
        self.seTable        = seTable
        self.prefSitesTable = prefSitesTable
        if os.path.exists(self.seTable):
            self.ses = self.readTable(self.seTable)
        else:
            print ' WARNING -- SE table file not found.'
        if os.path.exists(self.ceTable):
            self.ces = self.readTable(self.ceTable)
        else:
            print ' WARNING -- CE table file not found.'
        if os.path.exists(self.prefSitesTable):
            self.preferredSitesList = self.readList(self.prefSitesTable)
        else:
            print ' WARNING -- prefSites list file not found.'

        #self.show()

    def readList(self,file):
        list = []
        cmd = 'grep -v ^# ' + file + ' | tr -d \' \''
        print ' Loading list: ' + cmd
        for line in os.popen(cmd).readlines():
            line  = line[:-1]
            # decode the storage directory name
            names = line.split(" ")
            list.append(names[0])

        return list

    def readTable(self,file):
        table = {}
        cmd = 'grep -v ^# ' + file + ' | tr -d \' \''
        print ' Loading table: ' + cmd
        for line in os.popen(cmd).readlines():
            line         = line[:-1]
            # decode the storage directory name
            names        = line.split(":")
            if names[1] in table:
                print ' Site already in tables %s (%s)'%(names[1],names[0])
            table[names[1]] = names[0]

        return table

    def translateSes(self,seString):
        print ' translating SE sites: ' + seString
        sites = seString.split(",")
        newString = ''
        #print ' Looping through sites %s'%(seString)
        for site in sites:
            if site in self.ses:
                #print '   ->  %s  ==  %s'%(site,self.ses[site])
                if newString == '':
                    newString = self.ses[site]
                else:
                    newString = "%s,%s"%(newString,self.ses[site])
            else:
                print '  dropping  %s  because it is not in our table'%(site)

        #print 'Translated: %s'%(newString)
        self.allSites = newString
        return newString

    def translateCes(self,ceString):
        print ' translating CE sites: ' + ceString
        sites = ceString.split(",")
        newString = ''
        #print ' Looping through sites %s'%(ceString)
        for site in sites:
            if site in self.ces:
                #print '   ->  %s  ==  %s'%(site,self.ces[site])
                if newString == '':
                    newString = self.ces[site]
                else:
                    newString = "%s,%s"%(newString,self.ces[site])
            else:
                print '  dropping  %s  because it is not in our table'%(site)

        #print 'Translated: %s'%(newString)
        self.allSites = newString
        return newString

    def selectPreferred(self):
        #preferredSitesList = ['T2_US_MIT','T2_US_Wisconsin']
        self.preferredSites = ''
        for site in self.preferredSitesList:
            if re.search(site,self.allSites):
                if self.preferredSites != '':
                    self.preferredSites = self.preferredSites + ','
                self.preferredSites = self.preferredSites + site
     
        if self.preferredSites != '':
            print ' '
            print '         ' + self.allSites
            print ' INFO -- ' + self.preferredSites + ' found a preferred site (reducing).'
            print ' '
        else:
            #print ' ' 
            #print ' INFO -- no preferred site in the list. Keeping all sites.'
            #print ' '
            self.preferredSites = self.allSites
        return self.preferredSites    
            
    #-----------------------------------------------------------------------------------------------
    # present the current crab subtask
    #-----------------------------------------------------------------------------------------------
    def show(self):
        print ' Translator (CE Table: %s,  SE Table: %s) ===='%(self.ceTable,self.seTable)
        print ' ==== Compute Elements'
        for name,hostname in self.ces.iteritems():
            print '   Site: %s  -->  CE: %s'%(name,hostname)
        print ' ==== Storage Elements'
        for name,hostname in self.ses.iteritems():
            print '   Site: %s  -->  SE: %s'%(name,hostname)
Revision:	1.6
Committed:	Mon Sep 19 21:45:41 2011 UTC (13 years, 7 months ago) by paus
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	Mit_032, Mit_031, Mit_025c_branch2, Mit_025c_branch1, Mit_030, Mit_029c, Mit_029b, Mit_030_pre1, Mit_029a, Mit_029, Mit_029_pre1, Mit_028a, Mit_025c_branch0, Mit_028, Mit_027a, Mit_027, Mit_026, Mit_025e, Mit_025d, Mit_025c, Mit_025b, Mit_025a, Mit_025, Mit_025pre2, HEAD
Branch point for:	Mit_025c_branch
Changes since 1.5:	+30 -11 lines
Log Message:	Reinstate the bin and python areas.
#	User	Rev	Content
1	paus	1.1	#---------------------------------------------------------------------------------------------------
2			# Python Module File to translate storage/computing elements into meta site names
3			#
4			# Author: C.Paus (Jun 16, 2010)
5			#---------------------------------------------------------------------------------------------------
6			import os,sys,re,string
7
8			#---------------------------------------------------------------------------------------------------
9			"""
10			Class: Translator(table)
11			Each SubTask in CRAB can be described through this class
12			"""
13			#---------------------------------------------------------------------------------------------------
14			class Translator:
15			"Translator for the storage and computing elements to the "
16			# variable to be determined
17	paus	1.6	ceTable = 'undefined' # compute elements translation table
18			seTable = 'undefined' # storage elements translation table
19			allSites = 'undefined' # comma separated list of all sites
20			prefSitesTable = 'undefined' # comma separated list of preferred sites
21			ces = {}
22			ses = {}
23			preferredSitesList = []
24			preferredSites = []
25	paus	1.1	#-----------------------------------------------------------------------------------------------
26			# constructor to connect with existing setup
27			#-----------------------------------------------------------------------------------------------
28	paus	1.6	def __init__(self,ceTable,seTable,prefSitesTable):
29			self.ceTable = ceTable
30			self.seTable = seTable
31			self.prefSitesTable = prefSitesTable
32	paus	1.1	if os.path.exists(self.seTable):
33			self.ses = self.readTable(self.seTable)
34			else:
35			print ' WARNING -- SE table file not found.'
36			if os.path.exists(self.ceTable):
37			self.ces = self.readTable(self.ceTable)
38			else:
39			print ' WARNING -- CE table file not found.'
40	paus	1.6	if os.path.exists(self.prefSitesTable):
41			self.preferredSitesList = self.readList(self.prefSitesTable)
42			else:
43			print ' WARNING -- prefSites list file not found.'
44	paus	1.1
45			#self.show()
46
47	paus	1.6	def readList(self,file):
48			list = []
49			cmd = 'grep -v ^# ' + file + ' \| tr -d \' \''
50			print ' Loading list: ' + cmd
51			for line in os.popen(cmd).readlines():
52			line = line[:-1]
53			# decode the storage directory name
54			names = line.split(" ")
55			list.append(names[0])
56
57			return list
58
59	paus	1.1	def readTable(self,file):
60			table = {}
61			cmd = 'grep -v ^# ' + file + ' \| tr -d \' \''
62			print ' Loading table: ' + cmd
63			for line in os.popen(cmd).readlines():
64			line = line[:-1]
65			# decode the storage directory name
66			names = line.split(":")
67			if names[1] in table:
68			print ' Site already in tables %s (%s)'%(names[1],names[0])
69			table[names[1]] = names[0]
70
71			return table
72
73			def translateSes(self,seString):
74	paus	1.2	print ' translating SE sites: ' + seString
75	paus	1.1	sites = seString.split(",")
76			newString = ''
77			#print ' Looping through sites %s'%(seString)
78			for site in sites:
79			if site in self.ses:
80			#print ' -> %s == %s'%(site,self.ses[site])
81			if newString == '':
82			newString = self.ses[site]
83			else:
84			newString = "%s,%s"%(newString,self.ses[site])
85			else:
86			print ' dropping %s because it is not in our table'%(site)
87
88			#print 'Translated: %s'%(newString)
89	paus	1.2	self.allSites = newString
90	paus	1.1	return newString
91
92			def translateCes(self,ceString):
93	paus	1.2	print ' translating CE sites: ' + ceString
94	paus	1.1	sites = ceString.split(",")
95			newString = ''
96			#print ' Looping through sites %s'%(ceString)
97			for site in sites:
98			if site in self.ces:
99			#print ' -> %s == %s'%(site,self.ces[site])
100			if newString == '':
101			newString = self.ces[site]
102			else:
103			newString = "%s,%s"%(newString,self.ces[site])
104			else:
105			print ' dropping %s because it is not in our table'%(site)
106
107			#print 'Translated: %s'%(newString)
108	paus	1.2	self.allSites = newString
109	paus	1.1	return newString
110
111	paus	1.2	def selectPreferred(self):
112	paus	1.6	#preferredSitesList = ['T2_US_MIT','T2_US_Wisconsin']
113	paus	1.3	self.preferredSites = ''
114	paus	1.6	for site in self.preferredSitesList:
115	paus	1.3	if re.search(site,self.allSites):
116			if self.preferredSites != '':
117			self.preferredSites = self.preferredSites + ','
118			self.preferredSites = self.preferredSites + site
119
120			if self.preferredSites != '':
121	paus	1.2	print ' '
122			print ' ' + self.allSites
123	paus	1.3	print ' INFO -- ' + self.preferredSites + ' found a preferred site (reducing).'
124	paus	1.2	print ' '
125			else:
126			#print ' '
127			#print ' INFO -- no preferred site in the list. Keeping all sites.'
128			#print ' '
129			self.preferredSites = self.allSites
130			return self.preferredSites
131
132	paus	1.1	#-----------------------------------------------------------------------------------------------
133			# present the current crab subtask
134			#-----------------------------------------------------------------------------------------------
135			def show(self):
136			print ' Translator (CE Table: %s, SE Table: %s) ===='%(self.ceTable,self.seTable)
137			print ' ==== Compute Elements'
138			for name,hostname in self.ces.iteritems():
139			print ' Site: %s --> CE: %s'%(name,hostname)
140			print ' ==== Storage Elements'
141			for name,hostname in self.ses.iteritems():
142			print ' Site: %s --> SE: %s'%(name,hostname)