CRAB/python/SchedulerGlite.py

from SchedulerGrid import SchedulerGrid
from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
from GliteConfig import *
import EdgLoggingInfo
import common

import os, sys, time

class SchedulerGlite(SchedulerGrid):
    def __init__(self, name="GLITE"):
        SchedulerGrid.__init__(self,name)

    def configure(self,cfg_params):
        SchedulerGrid.configure(self, cfg_params)
        self.checkProxy()
        self.environment_unique_identifier = 'GLITE_WMS_JOBID'

    def rb_configure(self, RB):
        if not RB: return None
        glite_config = None
        rb_param_file = None

        gliteConfig = GliteConfig(RB)
        glite_config = gliteConfig.config()

        if (glite_config ):
            rb_param_file = glite_config
        return rb_param_file

    def ce_list(self):
        """
        Returns string with requirement CE related
        """
        req = ''
        if self.EDG_ce_white_list:
            ce_white_list = self.EDG_ce_white_list
            tmpCe=[]
            concString = '&&'
            for ce in ce_white_list:
                tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
            if len(tmpCe) == 1:
                req +=  " && (" + concString.join(tmpCe) + ") "
            elif len(tmpCe) > 1:
                firstCE = 0
                for reqTemp in tmpCe:
                    if firstCE == 0:
                        req += " && ( (" + reqTemp + ") "
                        firstCE = 1
                    elif firstCE > 0:
                        req += " || (" + reqTemp + ") "
                if firstCE > 0:
                    req += ") "

        if self.EDG_ce_black_list:
            ce_black_list = self.EDG_ce_black_list
            tmpCe=[]
            concString = '&&'
            for ce in ce_black_list:
                tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
            if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "

        # requirement added to skip gliteCE
        req += '&& (!RegExp("blah", other.GlueCEUniqueId))'

        return req,self.EDG_ce_white_list,self.EDG_ce_black_list

    def se_list(self, id, dest):
        """
        Returns string with requirement SE related
        """
        hostList=self.findSites_(id,dest)
        req=''
        reqtmp=[]
        concString = '||'

        for arg in hostList:
            reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')

        if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "

        return req

    def jdlParam(self):
        """
        Returns
        """
        req=''
        if self.EDG_addJdlParam:
            if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
            for p in self.EDG_addJdlParam:
             #   param_file.write(string.strip(p)+';\n')
                req+=string.strip(p)+';\n' ## BL--DS
        return req

    def specific_req(self):
        """
        Returns string with specific requirements
        """
        req=''
        if self.EDG_clock_time:
            if (not req == ' '): req = req + ' && '
            req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time

        if self.EDG_cpu_time:
            if (not req == ' '): req = req + ' && '
            req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time

        return req

    def sched_fix_parameter(self):
        """
        Returns string with requirements and scheduler-specific parameters
        """
        index = int(common._db.nJobs())
        job = common.job_list[index-1]
        jbt = job.type()
        req = ''
        req = req + jbt.getRequirements()

        if self.EDG_requirements:
            if (not req == ' '): req = req +  ' && '
            req = req + self.EDG_requirements

        Task_Req={'jobType':req}## DS--BL
        common._db.updateTask_(Task_Req)

    def sched_parameter(self,i,task):
        """
        Returns string with requirements and scheduler-specific parameters
        """
        dest=  task.jobs[i-1]['dlsDestination'] ## DS--BL

        req=''
        req +=task['jobType']

        sched_param=''
        sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(i,dest) +\
                                        self.ce_list()[0] +';\n' ## BL--DS
        if self.EDG_addJdlParam: sched_param+=self.jdlParam() ## BL--DS
        sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
        sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
        sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
        sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'

        return sched_param

    def decodeLogInfo(self, file):
        """
        Parse logging info file and return main info
        """
        loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
        reason = loggingInfo.decodeReason(file)
        return reason

    def findSites_(self, n, sites):
        itr4 =[]
        if len(sites)>0 and sites[0]=="":
            return itr4
        if sites != [""]:
            ##Addedd Daniele
            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
            if len(replicas)!=0:
                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)

            itr4 = replicas
            #####
        return itr4


    def tOut(self, list):
        return 180
Revision:	1.49
Committed:	Thu Apr 24 15:10:42 2008 UTC (17 years ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_2_0_pre12
Changes since 1.48:	+0 -24 lines
Log Message:	No need for separate gLite/Condor wsSetupEnvironment
#	User	Rev	Content
1	slacapra	1.31	from SchedulerGrid import SchedulerGrid
2	spiga	1.2	from crab_logger import Logger
3			from crab_exceptions import *
4			from crab_util import *
5	fanzago	1.4	from GliteConfig import *
6	spiga	1.47	import EdgLoggingInfo
7	spiga	1.2	import common
8
9			import os, sys, time
10
11	slacapra	1.31	class SchedulerGlite(SchedulerGrid):
12	slacapra	1.33	def __init__(self, name="GLITE"):
13			SchedulerGrid.__init__(self,name)
14	slacapra	1.31
15			def configure(self,cfg_params):
16			SchedulerGrid.configure(self, cfg_params)
17	ewv	1.42	self.checkProxy()
18	slacapra	1.31	self.environment_unique_identifier = 'GLITE_WMS_JOBID'
19	slacapra	1.3
20	fanzago	1.4	def rb_configure(self, RB):
21	slacapra	1.31	if not RB: return None
22			glite_config = None
23			rb_param_file = None
24	fanzago	1.4
25			gliteConfig = GliteConfig(RB)
26	slacapra	1.31	glite_config = gliteConfig.config()
27	fanzago	1.4
28	slacapra	1.31	if (glite_config ):
29	spiga	1.40	rb_param_file = glite_config
30	slacapra	1.31	return rb_param_file
31	fanzago	1.4
32	spiga	1.36	def ce_list(self):
33	spiga	1.2	"""
34	ewv	1.42	Returns string with requirement CE related
35	spiga	1.2	"""
36	ewv	1.42	req = ''
37	spiga	1.2	if self.EDG_ce_white_list:
38	mcinquil	1.32	ce_white_list = self.EDG_ce_white_list
39	slacapra	1.13	tmpCe=[]
40			concString = '&&'
41			for ce in ce_white_list:
42			tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
43	mcinquil	1.20	if len(tmpCe) == 1:
44			req += " && (" + concString.join(tmpCe) + ") "
45			elif len(tmpCe) > 1:
46			firstCE = 0
47			for reqTemp in tmpCe:
48			if firstCE == 0:
49			req += " && ( (" + reqTemp + ") "
50			firstCE = 1
51			elif firstCE > 0:
52			req += " \|\| (" + reqTemp + ") "
53			if firstCE > 0:
54			req += ") "
55	ewv	1.28
56	spiga	1.2	if self.EDG_ce_black_list:
57	mcinquil	1.32	ce_black_list = self.EDG_ce_black_list
58	slacapra	1.13	tmpCe=[]
59			concString = '&&'
60	spiga	1.2	for ce in ce_black_list:
61	slacapra	1.13	tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
62	spiga	1.36	if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "
63
64			# requirement added to skip gliteCE
65			req += '&& (!RegExp("blah", other.GlueCEUniqueId))'
66
67	spiga	1.41	return req,self.EDG_ce_white_list,self.EDG_ce_black_list
68	spiga	1.36
69	spiga	1.39	def se_list(self, id, dest):
70	spiga	1.36	"""
71	ewv	1.42	Returns string with requirement SE related
72			"""
73	spiga	1.39	hostList=self.findSites_(id,dest)
74	spiga	1.36	req=''
75			reqtmp=[]
76			concString = '\|\|'
77
78			for arg in hostList:
79			reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')
80
81			if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "
82	spiga	1.11
83	spiga	1.36	return req
84
85			def jdlParam(self):
86			"""
87	ewv	1.42	Returns
88			"""
89	spiga	1.36	req=''
90			if self.EDG_addJdlParam:
91	ewv	1.42	if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
92	spiga	1.36	for p in self.EDG_addJdlParam:
93			# param_file.write(string.strip(p)+';\n')
94			req+=string.strip(p)+';\n' ## BL--DS
95			return req
96
97			def specific_req(self):
98			"""
99			Returns string with specific requirements
100	ewv	1.42	"""
101	spiga	1.36	req=''
102	spiga	1.2	if self.EDG_clock_time:
103	slacapra	1.13	if (not req == ' '): req = req + ' && '
104			req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
105	spiga	1.2
106			if self.EDG_cpu_time:
107	slacapra	1.13	if (not req == ' '): req = req + ' && '
108			req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
109	ewv	1.42
110	spiga	1.36	return req
111	ewv	1.28
112	spiga	1.39	def sched_fix_parameter(self):
113	spiga	1.36	"""
114			Returns string with requirements and scheduler-specific parameters
115			"""
116	ewv	1.42	index = int(common._db.nJobs())
117	spiga	1.39	job = common.job_list[index-1]
118	spiga	1.36	jbt = job.type()
119			req = ''
120			req = req + jbt.getRequirements()
121	slacapra	1.13
122	spiga	1.36	if self.EDG_requirements:
123			if (not req == ' '): req = req + ' && '
124			req = req + self.EDG_requirements
125	spiga	1.2
126	spiga	1.39	Task_Req={'jobType':req}## DS--BL
127	ewv	1.42	common._db.updateTask_(Task_Req)
128	spiga	1.39
129			def sched_parameter(self,i,task):
130			"""
131			Returns string with requirements and scheduler-specific parameters
132			"""
133	spiga	1.48	dest= task.jobs[i-1]['dlsDestination'] ## DS--BL
134	spiga	1.39
135			req=''
136			req +=task['jobType']
137
138			sched_param=''
139			sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(i,dest) +\
140	spiga	1.41	self.ce_list()[0] +';\n' ## BL--DS
141	spiga	1.39	if self.EDG_addJdlParam: sched_param+=self.jdlParam() ## BL--DS
142			sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
143			sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
144			sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
145			sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
146
147			return sched_param
148	ewv	1.42
149	spiga	1.47	def decodeLogInfo(self, file):
150			"""
151			Parse logging info file and return main info
152			"""
153			loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
154			reason = loggingInfo.decodeReason(file)
155			return reason
156
157	spiga	1.39	def findSites_(self, n, sites):
158			itr4 =[]
159			if len(sites)>0 and sites[0]=="":
160			return itr4
161			if sites != [""]:
162			##Addedd Daniele
163			replicas = self.blackWhiteListParser.checkBlackList(sites,n)
164			if len(replicas)!=0:
165			replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
166	ewv	1.42
167	spiga	1.39	itr4 = replicas
168			#####
169			return itr4
170
171
172
173	spiga	1.14
174	mcinquil	1.27	def tOut(self, list):
175	mcinquil	1.18	return 180