CRAB/python/SchedulerGlite.py

from SchedulerGrid import SchedulerGrid
from crab_logger import Logger
from crab_exceptions import *
from crab_util import *
from GliteConfig import *
import EdgLoggingInfo
import common

import os, sys, time

class SchedulerGlite(SchedulerGrid):
    def __init__(self, name="GLITE"):
        SchedulerGrid.__init__(self,name)

        self.OSBsize = 55000000

    def configure(self,cfg_params):
        SchedulerGrid.configure(self, cfg_params)
        self.environment_unique_identifier = 'GLITE_WMS_JOBID'

    def realSchedParams(self,cfg_params):
        """
        Return dictionary with specific parameters, to use 
        with real scheduler  
        """
        self.rb_param_file=''
        # if (cfg_params.has_key('EDG.rb')):
        #     self.rb_param_file=common.scheduler.rb_configure(cfg_params.get("EDG.rb"))
        self.wms_service=cfg_params.get("EDG.wms_service",'')
        self.skipWMSAuth=cfg_params.get("EDG.skipwmsauth",0)
        params = { 'service' : self.wms_service, \
                   'config' : self.rb_param_file, \
                   'skipWMSAuth' : self.skipWMSAuth 
                 }
        return  params
      

    def rb_configure(self, RB):
        if not RB: return None
        glite_config = None
        rb_param_file = None

        gliteConfig = GliteConfig(RB)
        glite_config = gliteConfig.config()

        if (glite_config ):
            rb_param_file = glite_config
        return rb_param_file

    def ce_list(self):
        """
        Returns string with requirement CE related
        """
        req = ''
        if self.EDG_ce_white_list:
            ce_white_list = self.EDG_ce_white_list
            tmpCe=[]
            concString = '&&'
            for ce in ce_white_list:
                tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
            if len(tmpCe) == 1:
                req +=  " && (" + concString.join(tmpCe) + ") "
            elif len(tmpCe) > 1:
                firstCE = 0
                for reqTemp in tmpCe:
                    if firstCE == 0:
                        req += " && ( (" + reqTemp + ") "
                        firstCE = 1
                    elif firstCE > 0:
                        req += " || (" + reqTemp + ") "
                if firstCE > 0:
                    req += ") "

        if self.EDG_ce_black_list:
            ce_black_list = self.EDG_ce_black_list
            tmpCe=[]
            concString = '&&'
            for ce in ce_black_list:
                tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
            if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "

        # requirement added to skip gliteCE
        req += '&& (!RegExp("blah", other.GlueCEUniqueId))'

        return req,self.EDG_ce_white_list,self.EDG_ce_black_list

    def se_list(self, id, dest):
        """
        Returns string with requirement SE related
        """
        hostList=self.findSites_(id,dest)
        req=''
        reqtmp=[]
        concString = '||'

        for arg in hostList:
            reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')

        if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "

        return req

    def jdlParam(self):
        """
        Returns
        """
        req=''
        if self.EDG_addJdlParam:
            if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
            for p in self.EDG_addJdlParam:
                req+=string.strip(p)+';\n' 
        return req

    def specific_req(self):
        """
        Returns string with specific requirements
        """
        req=''
        if self.EDG_clock_time:
            if (not req == ' '): req = req + ' && '
            req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time

        if self.EDG_cpu_time:
            if (not req == ' '): req = req + ' && '
            req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time

        return req

    def sched_fix_parameter(self):
        """
        Returns string with requirements and scheduler-specific parameters
        """
        index = int(common._db.nJobs())
        job = common.job_list[index-1]
        jbt = job.type()
        req = ''
        req = req + jbt.getRequirements()

        if self.EDG_requirements:
            if (not req == ' '): req = req +  ' && '
            req = req + self.EDG_requirements

        Task_Req={'jobType':req}
        common._db.updateTask_(Task_Req)

    def sched_parameter(self,i,task):
        """
        Returns string with requirements and scheduler-specific parameters
        """
        dest=  task.jobs[i-1]['dlsDestination'] 

        req=''
        req +=task['jobType']

        sched_param=''
        sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(i,dest) +\
                                        self.ce_list()[0] +';\n' 
        if self.EDG_addJdlParam: sched_param+=self.jdlParam() 
        sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
        sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
        sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
        sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'

        return sched_param

    def decodeLogInfo(self, file):
        """
        Parse logging info file and return main info
        """
        loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
        reason = loggingInfo.decodeReason(file)
        return reason

    def findSites_(self, n, sites):
        itr4 =[]
        if len(sites)>0 and sites[0]=="":
            return itr4
        if sites != [""]:
            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
            if len(replicas)!=0:
                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)

            itr4 = replicas
        return itr4

    
    def wsExitFunc(self):
        """
        """
        txt = '\n'

        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()
        ### specific Glite check for OSB
        txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    tmp_size=`ls -gGrta ${out_files}.tgz | awk \'{ print $3 }\'`\n'
        txt += '    rm ${out_files}.tgz\n'  
        txt += '    size=`expr $tmp_size`\n'
        txt += '    echo "Total Output dimension: $size"\n'
        txt += '    limit='+str(self.OSBsize) +' \n'  
        txt += '    echo "WARNING: output files size limit is set to: $limit"\n'
        txt += '    if [ "$limit" -lt "$size" ]; then\n'
        txt += '        exceed=1\n'
        txt += '        job_exit_code=70000\n'
        txt += '        echo "Output Sanbox too big. Produced output is lost "\n'
        txt += '    else\n'
        txt += '        exceed=0\n'
        txt += '        echo "Total Output dimension $size is fine."\n'
        txt += '    fi\n'

        txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
        txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
        txt += '    if [ $exceed -ne 1 ]; then\n'
        txt += '        tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    else\n'
        txt += '        tar zcvf ${out_files}.tgz CMSSW_${NJob}.stdout CMSSW_${NJob}.stderr\n'
        txt += '    fi\n'
        txt += '    exit $job_exit_code\n'

        txt += '}\n'
        return txt
Revision:	1.59
Committed:	Thu Jul 17 10:17:18 2008 UTC (16 years, 9 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
Changes since 1.58:	+2 -2 lines
Log Message:	default RB is CERN if none is set
#	User	Rev	Content
1	slacapra	1.31	from SchedulerGrid import SchedulerGrid
2	spiga	1.2	from crab_logger import Logger
3			from crab_exceptions import *
4			from crab_util import *
5	fanzago	1.4	from GliteConfig import *
6	spiga	1.47	import EdgLoggingInfo
7	spiga	1.2	import common
8
9			import os, sys, time
10
11	slacapra	1.31	class SchedulerGlite(SchedulerGrid):
12	slacapra	1.33	def __init__(self, name="GLITE"):
13			SchedulerGrid.__init__(self,name)
14	slacapra	1.31
15	fanzago	1.54	self.OSBsize = 55000000
16	spiga	1.51
17	slacapra	1.31	def configure(self,cfg_params):
18			SchedulerGrid.configure(self, cfg_params)
19			self.environment_unique_identifier = 'GLITE_WMS_JOBID'
20	slacapra	1.3
21	spiga	1.51	def realSchedParams(self,cfg_params):
22			"""
23			Return dictionary with specific parameters, to use
24			with real scheduler
25			"""
26			self.rb_param_file=''
27	slacapra	1.59	# if (cfg_params.has_key('EDG.rb')):
28			# self.rb_param_file=common.scheduler.rb_configure(cfg_params.get("EDG.rb"))
29	spiga	1.51	self.wms_service=cfg_params.get("EDG.wms_service",'')
30	afanfani	1.55	self.skipWMSAuth=cfg_params.get("EDG.skipwmsauth",0)
31	spiga	1.51	params = { 'service' : self.wms_service, \
32	afanfani	1.55	'config' : self.rb_param_file, \
33			'skipWMSAuth' : self.skipWMSAuth
34	spiga	1.51	}
35			return params
36
37
38	fanzago	1.4	def rb_configure(self, RB):
39	slacapra	1.31	if not RB: return None
40			glite_config = None
41			rb_param_file = None
42	fanzago	1.4
43			gliteConfig = GliteConfig(RB)
44	slacapra	1.31	glite_config = gliteConfig.config()
45	fanzago	1.4
46	slacapra	1.31	if (glite_config ):
47	spiga	1.40	rb_param_file = glite_config
48	slacapra	1.31	return rb_param_file
49	fanzago	1.4
50	spiga	1.36	def ce_list(self):
51	spiga	1.2	"""
52	ewv	1.42	Returns string with requirement CE related
53	spiga	1.2	"""
54	ewv	1.42	req = ''
55	spiga	1.2	if self.EDG_ce_white_list:
56	mcinquil	1.32	ce_white_list = self.EDG_ce_white_list
57	slacapra	1.13	tmpCe=[]
58			concString = '&&'
59			for ce in ce_white_list:
60			tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
61	mcinquil	1.20	if len(tmpCe) == 1:
62			req += " && (" + concString.join(tmpCe) + ") "
63			elif len(tmpCe) > 1:
64			firstCE = 0
65			for reqTemp in tmpCe:
66			if firstCE == 0:
67			req += " && ( (" + reqTemp + ") "
68			firstCE = 1
69			elif firstCE > 0:
70			req += " \|\| (" + reqTemp + ") "
71			if firstCE > 0:
72			req += ") "
73	ewv	1.28
74	spiga	1.2	if self.EDG_ce_black_list:
75	mcinquil	1.32	ce_black_list = self.EDG_ce_black_list
76	slacapra	1.13	tmpCe=[]
77			concString = '&&'
78	spiga	1.2	for ce in ce_black_list:
79	slacapra	1.13	tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
80	spiga	1.36	if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "
81
82			# requirement added to skip gliteCE
83			req += '&& (!RegExp("blah", other.GlueCEUniqueId))'
84
85	spiga	1.41	return req,self.EDG_ce_white_list,self.EDG_ce_black_list
86	spiga	1.36
87	spiga	1.39	def se_list(self, id, dest):
88	spiga	1.36	"""
89	ewv	1.42	Returns string with requirement SE related
90			"""
91	spiga	1.39	hostList=self.findSites_(id,dest)
92	spiga	1.36	req=''
93			reqtmp=[]
94			concString = '\|\|'
95
96			for arg in hostList:
97			reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')
98
99			if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "
100	spiga	1.11
101	spiga	1.36	return req
102
103			def jdlParam(self):
104			"""
105	ewv	1.42	Returns
106			"""
107	spiga	1.36	req=''
108			if self.EDG_addJdlParam:
109	ewv	1.42	if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
110	spiga	1.36	for p in self.EDG_addJdlParam:
111	spiga	1.56	req+=string.strip(p)+';\n'
112	spiga	1.36	return req
113
114			def specific_req(self):
115			"""
116			Returns string with specific requirements
117	ewv	1.42	"""
118	spiga	1.36	req=''
119	spiga	1.2	if self.EDG_clock_time:
120	slacapra	1.13	if (not req == ' '): req = req + ' && '
121			req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
122	spiga	1.2
123			if self.EDG_cpu_time:
124	slacapra	1.13	if (not req == ' '): req = req + ' && '
125			req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
126	ewv	1.42
127	spiga	1.36	return req
128	ewv	1.28
129	spiga	1.39	def sched_fix_parameter(self):
130	spiga	1.36	"""
131			Returns string with requirements and scheduler-specific parameters
132			"""
133	ewv	1.42	index = int(common._db.nJobs())
134	spiga	1.39	job = common.job_list[index-1]
135	spiga	1.36	jbt = job.type()
136			req = ''
137			req = req + jbt.getRequirements()
138	slacapra	1.13
139	spiga	1.36	if self.EDG_requirements:
140			if (not req == ' '): req = req + ' && '
141			req = req + self.EDG_requirements
142	spiga	1.2
143	spiga	1.56	Task_Req={'jobType':req}
144	ewv	1.42	common._db.updateTask_(Task_Req)
145	spiga	1.39
146			def sched_parameter(self,i,task):
147			"""
148			Returns string with requirements and scheduler-specific parameters
149			"""
150	spiga	1.56	dest= task.jobs[i-1]['dlsDestination']
151	spiga	1.39
152			req=''
153			req +=task['jobType']
154
155			sched_param=''
156			sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(i,dest) +\
157	spiga	1.56	self.ce_list()[0] +';\n'
158			if self.EDG_addJdlParam: sched_param+=self.jdlParam()
159	spiga	1.39	sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
160			sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
161			sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
162			sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
163
164			return sched_param
165	ewv	1.42
166	spiga	1.47	def decodeLogInfo(self, file):
167			"""
168			Parse logging info file and return main info
169			"""
170			loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
171			reason = loggingInfo.decodeReason(file)
172			return reason
173
174	spiga	1.39	def findSites_(self, n, sites):
175			itr4 =[]
176			if len(sites)>0 and sites[0]=="":
177			return itr4
178			if sites != [""]:
179			replicas = self.blackWhiteListParser.checkBlackList(sites,n)
180			if len(replicas)!=0:
181			replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
182	ewv	1.42
183	spiga	1.39	itr4 = replicas
184			return itr4
185
186	spiga	1.51
187			def wsExitFunc(self):
188			"""
189			"""
190			txt = '\n'
191
192			txt += '#\n'
193			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
194			txt += '#\n\n'
195
196			txt += 'func_exit() { \n'
197			txt += self.wsExitFunc_common()
198			### specific Glite check for OSB
199			txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
200			txt += ' tmp_size=`ls -gGrta ${out_files}.tgz \| awk \'{ print $3 }\'`\n'
201			txt += ' rm ${out_files}.tgz\n'
202			txt += ' size=`expr $tmp_size`\n'
203			txt += ' echo "Total Output dimension: $size"\n'
204			txt += ' limit='+str(self.OSBsize) +' \n'
205			txt += ' echo "WARNING: output files size limit is set to: $limit"\n'
206	spiga	1.53	txt += ' if [ "$limit" -lt "$size" ]; then\n'
207	spiga	1.51	txt += ' exceed=1\n'
208			txt += ' job_exit_code=70000\n'
209			txt += ' echo "Output Sanbox too big. Produced output is lost "\n'
210			txt += ' else\n'
211			txt += ' exceed=0\n'
212	spiga	1.53	txt += ' echo "Total Output dimension $size is fine."\n'
213	spiga	1.51	txt += ' fi\n'
214
215			txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
216			txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
217			txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
218			txt += ' if [ $exceed -ne 1 ]; then\n'
219			txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
220			txt += ' else\n'
221			txt += ' tar zcvf ${out_files}.tgz CMSSW_${NJob}.stdout CMSSW_${NJob}.stderr\n'
222			txt += ' fi\n'
223			txt += ' exit $job_exit_code\n'
224
225			txt += '}\n'
226			return txt