CRAB/python/SchedulerGlite.py

"""
CRAB interface to BossLite gLite Scheduler
"""

__revision__ = "$Id: SchedulerGlite.py,v 1.81 2010/05/19 17:30:39 spiga Exp $"
__version__ = "$Revision: 1.81 $"

from SchedulerGrid import SchedulerGrid
from crab_exceptions import *
from crab_util import *
import EdgLoggingInfo
import common
from WMCore.SiteScreening.BlackWhiteListParser import CEBlackWhiteListParser

import os, sys, time

class SchedulerGlite(SchedulerGrid):
    def __init__(self, name="GLITE"):
        SchedulerGrid.__init__(self,name)

        self.EDG_retry_count        = 0
        self.EDG_shallow_retry_count= -1
        self.OSBsize = 55000000

    def configure(self,cfg_params):
        SchedulerGrid.configure(self, cfg_params)
        self.environment_unique_identifier = '$GLITE_WMS_JOBID'

    def realSchedParams(self,cfg_params):
        """
        Return dictionary with specific parameters, to use
        with real scheduler
        """
        self.rb_param_file=''
        if (not cfg_params.has_key('GRID.rb')):
            cfg_params['GRID.rb']='CERN'
        self.rb_param_file=common.scheduler.rb_configure(cfg_params.get("GRID.rb"))
        self.wms_service=cfg_params.get("GRID.wms_service",'')
        self.skipWMSAuth=cfg_params.get("GRID.skipwmsauth",1)
        params = { 'service' : self.wms_service, \
                   'config' : self.rb_param_file, \
                   'skipWMSAuth' : self.skipWMSAuth
                 }
        return  params


    def rb_configure(self, RB):
        url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
        from Downloader import Downloader
        import httplib
        common.logger.debug('Downloading config files for WMS: '+url)
        ## 25-Jun-2009 SL: patch to use Cream enabled WMS
        if ( self.cfg_params.get('GRID.use_cream',None) ):
            RB='CREAM'
        if not RB: return None
        rb_param_file = None
        configFileName = 'glite_wms_'+str(RB)+'.conf'

        results = Downloader(url)
        try:
            gliteConfig  = results.filePath(configFileName)
        except httplib.HTTPException, ex: 
            raise CrabException( "Problem getting RB config file: %s, reason:"%(configFileName, ex) )

        if (gliteConfig ):
            rb_param_file = gliteConfig
        return rb_param_file

    def ce_list(self):
        """
        Returns string with requirement CE related
        """
        ceParser = CEBlackWhiteListParser(self.EDG_ce_white_list,
                                          self.EDG_ce_black_list, common.logger())
        req = ''
        ce_white_list = []
        ce_black_list = []
        if self.EDG_ce_white_list:
            ce_white_list = ceParser.whiteList()
            tmpCe=[]
            concString = '&&'
            for ce in ce_white_list:
                tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
            if len(tmpCe) == 1:
                req +=  " && (" + concString.join(tmpCe) + ") "
            elif len(tmpCe) > 1:
                firstCE = 0
                for reqTemp in tmpCe:
                    if firstCE == 0:
                        req += " && ( (" + reqTemp + ") "
                        firstCE = 1
                    elif firstCE > 0:
                        req += " || (" + reqTemp + ") "
                if firstCE > 0:
                    req += ") "

        if self.EDG_ce_black_list:
            ce_black_list = ceParser.blackList()
            tmpCe=[]
            concString = '&&'
            for ce in ce_black_list:
                tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
            if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "

        # requirement added to skip gliteCE
        # not more needed
 #       req += '&& (!RegExp("blah", other.GlueCEUniqueId))'
        retWL = ','.join(ce_white_list)
        retBL = ','.join(ce_black_list)
        if not retWL:
            retWL = None
        if not retBL:
            retBL = None

        return req, retWL, retBL

    def se_list(self, dest):
        """
        Returns string with requirement SE related
        """
        hostList=self.findSites_(dest)
        req=''
        reqtmp=[]
        concString = '||'

        for arg in hostList:
            reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')

        if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "

        return req

    def jdlParam(self):
        """
        Returns
        """
        req=''
        if self.EDG_addJdlParam:
            if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
            for p in self.EDG_addJdlParam:
                req+=string.strip(p)+';\n'
        return req

    def specific_req(self):
        """
        Returns string with specific requirements
        """
        req=''
        if self.EDG_clock_time:
            if (not req == ' '): req = req + ' && '
            req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time

        if self.EDG_cpu_time:
            if (not req == ' '): req = req + ' && '
            req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time

        return req

    def sched_parameter(self,i,task):
        """
        Returns string with requirements and scheduler-specific parameters
        """
        dest=  task.jobs[i-1]['dlsDestination']

        req=''
        req +=task['jobType']

        sched_param=''
        sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(dest) +\
                                        self.ce_list()[0] +';\n'
        if self.EDG_addJdlParam: sched_param+=self.jdlParam()
        sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
        sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
        sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
        sched_param+='DefaultNodeRetryCount = '+str(self.EDG_retry_count)+';\n'
        sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
        sched_param+='DefaultNodeShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'

        return sched_param

    def decodeLogInfo(self, file):
        """
        Parse logging info file and return main info
        """
        loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
        reason = loggingInfo.decodeReason(file)
        return reason

    def findSites_(self, sites):
        itr4 =[]
        if len(sites)>0 and sites[0]=="":
            return itr4
        if sites != [""]:
            replicas = self.blackWhiteListParser.checkBlackList(sites)
            if len(replicas)!=0:
                replicas = self.blackWhiteListParser.checkWhiteList(replicas)

            itr4 = replicas
        return itr4

    def delegateProxy(self):
        self.boss().delegateProxy()   
        return

    def wsExitFunc(self):
        """
        """
        txt = '\n'

        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()
        ### specific Glite check for OSB
        txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    tmp_size=`ls -gGrta ${out_files}.tgz | awk \'{ print $3 }\'`\n'
        txt += '    rm ${out_files}.tgz\n'
        txt += '    size=`expr $tmp_size`\n'
        txt += '    echo "Total Output dimension: $size"\n'
        txt += '    limit='+str(self.OSBsize) +' \n'
        txt += '    echo "WARNING: output files size limit is set to: $limit"\n'
        txt += '    if [ "$limit" -lt "$size" ]; then\n'
        txt += '        exceed=1\n'
        txt += '        job_exit_code=70000\n'
        txt += '        echo "Output Sanbox too big. Produced output is lost "\n'
        txt += '    else\n'
        txt += '        exceed=0\n'
        txt += '        echo "Total Output dimension $size is fine."\n'
        txt += '    fi\n'

        txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
        txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
        txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
        txt += '    if [ $exceed -ne 1 ]; then\n'
        txt += '        tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    else\n'
        txt += '        tar zcvf ${out_files}.tgz CMSSW_${NJob}.stdout CMSSW_${NJob}.stderr\n'
        txt += '    fi\n'
        txt += '    python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --errorcode $job_exit_code \n'
        txt += '    exit $job_exit_code\n'

        txt += '}\n'
        return txt

    def listMatch(self, dest, full):
        matching='fast'
        
        if self.boss().schedulerConfig['name'] == 'SchedulerGLite' :
            taskId=common._db.getTask()
            req=str(self.sched_parameter(1,taskId))
            sites = self.boss().schedSession().matchResources(taskId, requirements=req)
        else :
            sites = SchedulerGrid.listMatch(self, dest, full)
            
        if full == True: matching='full'
        common.logger.debug("list of available site ( "+str(matching) +" matching ) : "+str(sites))
        
        return sites
Revision:	1.82
Committed:	Wed Jun 9 10:28:56 2010 UTC (14 years, 10 months ago) by farinafa
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta
Changes since 1.81:	+3 -3 lines
Log Message:	Refinements for bug #68550: shallow_retry set to -1, retry_count to 0
#	User	Rev	Content
1	ewv	1.62	"""
2			CRAB interface to BossLite gLite Scheduler
3			"""
4
5	farinafa	1.82	__revision__ = "$Id: SchedulerGlite.py,v 1.81 2010/05/19 17:30:39 spiga Exp $"
6			__version__ = "$Revision: 1.81 $"
7	ewv	1.62
8	slacapra	1.31	from SchedulerGrid import SchedulerGrid
9	spiga	1.2	from crab_exceptions import *
10			from crab_util import *
11	spiga	1.47	import EdgLoggingInfo
12	spiga	1.2	import common
13	ewv	1.65	from WMCore.SiteScreening.BlackWhiteListParser import CEBlackWhiteListParser
14	spiga	1.2
15			import os, sys, time
16
17	slacapra	1.31	class SchedulerGlite(SchedulerGrid):
18	slacapra	1.33	def __init__(self, name="GLITE"):
19			SchedulerGrid.__init__(self,name)
20	slacapra	1.31
21	spiga	1.81	self.EDG_retry_count = 0
22	farinafa	1.82	self.EDG_shallow_retry_count= -1
23	fanzago	1.54	self.OSBsize = 55000000
24	spiga	1.51
25	slacapra	1.31	def configure(self,cfg_params):
26			SchedulerGrid.configure(self, cfg_params)
27	ewv	1.61	self.environment_unique_identifier = '$GLITE_WMS_JOBID'
28	slacapra	1.3
29	spiga	1.51	def realSchedParams(self,cfg_params):
30			"""
31	ewv	1.61	Return dictionary with specific parameters, to use
32			with real scheduler
33	spiga	1.51	"""
34			self.rb_param_file=''
35	spiga	1.68	if (not cfg_params.has_key('GRID.rb')):
36			cfg_params['GRID.rb']='CERN'
37			self.rb_param_file=common.scheduler.rb_configure(cfg_params.get("GRID.rb"))
38			self.wms_service=cfg_params.get("GRID.wms_service",'')
39			self.skipWMSAuth=cfg_params.get("GRID.skipwmsauth",1)
40	spiga	1.51	params = { 'service' : self.wms_service, \
41	afanfani	1.55	'config' : self.rb_param_file, \
42	ewv	1.61	'skipWMSAuth' : self.skipWMSAuth
43	spiga	1.51	}
44			return params
45	ewv	1.61
46	spiga	1.51
47	fanzago	1.4	def rb_configure(self, RB):
48	spiga	1.75	url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
49	spiga	1.74	from Downloader import Downloader
50	farinafa	1.79	import httplib
51	spiga	1.75	common.logger.debug('Downloading config files for WMS: '+url)
52	slacapra	1.71	## 25-Jun-2009 SL: patch to use Cream enabled WMS
53			if ( self.cfg_params.get('GRID.use_cream',None) ):
54			RB='CREAM'
55	slacapra	1.31	if not RB: return None
56			rb_param_file = None
57	spiga	1.74	configFileName = 'glite_wms_'+str(RB)+'.conf'
58	fanzago	1.4
59	spiga	1.76	results = Downloader(url)
60	farinafa	1.79	try:
61			gliteConfig = results.filePath(configFileName)
62			except httplib.HTTPException, ex:
63			raise CrabException( "Problem getting RB config file: %s, reason:"%(configFileName, ex) )
64	fanzago	1.4
65	spiga	1.77	if (gliteConfig ):
66			rb_param_file = gliteConfig
67	slacapra	1.31	return rb_param_file
68	fanzago	1.4
69	spiga	1.36	def ce_list(self):
70	spiga	1.2	"""
71	ewv	1.42	Returns string with requirement CE related
72	spiga	1.2	"""
73	ewv	1.65	ceParser = CEBlackWhiteListParser(self.EDG_ce_white_list,
74	slacapra	1.69	self.EDG_ce_black_list, common.logger())
75	ewv	1.42	req = ''
76	ewv	1.65	ce_white_list = []
77			ce_black_list = []
78	spiga	1.2	if self.EDG_ce_white_list:
79	ewv	1.65	ce_white_list = ceParser.whiteList()
80	slacapra	1.13	tmpCe=[]
81			concString = '&&'
82			for ce in ce_white_list:
83			tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
84	mcinquil	1.20	if len(tmpCe) == 1:
85			req += " && (" + concString.join(tmpCe) + ") "
86			elif len(tmpCe) > 1:
87			firstCE = 0
88			for reqTemp in tmpCe:
89			if firstCE == 0:
90			req += " && ( (" + reqTemp + ") "
91			firstCE = 1
92			elif firstCE > 0:
93			req += " \|\| (" + reqTemp + ") "
94			if firstCE > 0:
95			req += ") "
96	ewv	1.28
97	spiga	1.2	if self.EDG_ce_black_list:
98	ewv	1.65	ce_black_list = ceParser.blackList()
99	slacapra	1.13	tmpCe=[]
100			concString = '&&'
101	spiga	1.2	for ce in ce_black_list:
102	slacapra	1.13	tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
103	spiga	1.36	if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "
104
105			# requirement added to skip gliteCE
106	spiga	1.72	# not more needed
107			# req += '&& (!RegExp("blah", other.GlueCEUniqueId))'
108	ewv	1.66	retWL = ','.join(ce_white_list)
109			retBL = ','.join(ce_black_list)
110			if not retWL:
111			retWL = None
112			if not retBL:
113			retBL = None
114	spiga	1.36
115	ewv	1.66	return req, retWL, retBL
116	spiga	1.36
117	slacapra	1.60	def se_list(self, dest):
118	spiga	1.36	"""
119	ewv	1.42	Returns string with requirement SE related
120			"""
121	slacapra	1.60	hostList=self.findSites_(dest)
122	spiga	1.36	req=''
123			reqtmp=[]
124			concString = '\|\|'
125
126			for arg in hostList:
127			reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')
128
129			if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "
130	spiga	1.11
131	spiga	1.36	return req
132
133			def jdlParam(self):
134			"""
135	ewv	1.42	Returns
136			"""
137	spiga	1.36	req=''
138			if self.EDG_addJdlParam:
139	ewv	1.42	if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
140	spiga	1.36	for p in self.EDG_addJdlParam:
141	ewv	1.61	req+=string.strip(p)+';\n'
142	spiga	1.36	return req
143
144			def specific_req(self):
145			"""
146			Returns string with specific requirements
147	ewv	1.42	"""
148	spiga	1.36	req=''
149	spiga	1.2	if self.EDG_clock_time:
150	slacapra	1.13	if (not req == ' '): req = req + ' && '
151			req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
152	spiga	1.2
153			if self.EDG_cpu_time:
154	slacapra	1.13	if (not req == ' '): req = req + ' && '
155			req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
156	ewv	1.42
157	spiga	1.36	return req
158	ewv	1.28
159	spiga	1.39	def sched_parameter(self,i,task):
160			"""
161			Returns string with requirements and scheduler-specific parameters
162			"""
163	ewv	1.61	dest= task.jobs[i-1]['dlsDestination']
164	spiga	1.39
165			req=''
166			req +=task['jobType']
167
168			sched_param=''
169	slacapra	1.60	sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(dest) +\
170	ewv	1.61	self.ce_list()[0] +';\n'
171			if self.EDG_addJdlParam: sched_param+=self.jdlParam()
172	spiga	1.39	sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
173			sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
174			sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
175	farinafa	1.80	sched_param+='DefaultNodeRetryCount = '+str(self.EDG_retry_count)+';\n'
176	spiga	1.39	sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
177	farinafa	1.80	sched_param+='DefaultNodeShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
178	spiga	1.39
179			return sched_param
180	ewv	1.42
181	spiga	1.47	def decodeLogInfo(self, file):
182			"""
183			Parse logging info file and return main info
184			"""
185			loggingInfo = EdgLoggingInfo.EdgLoggingInfo()
186			reason = loggingInfo.decodeReason(file)
187			return reason
188
189	slacapra	1.60	def findSites_(self, sites):
190	spiga	1.39	itr4 =[]
191			if len(sites)>0 and sites[0]=="":
192			return itr4
193			if sites != [""]:
194	slacapra	1.60	replicas = self.blackWhiteListParser.checkBlackList(sites)
195	spiga	1.39	if len(replicas)!=0:
196	slacapra	1.60	replicas = self.blackWhiteListParser.checkWhiteList(replicas)
197	ewv	1.42
198	spiga	1.39	itr4 = replicas
199			return itr4
200
201	spiga	1.70	def delegateProxy(self):
202			self.boss().delegateProxy()
203			return
204	ewv	1.61
205	spiga	1.51	def wsExitFunc(self):
206			"""
207			"""
208			txt = '\n'
209
210			txt += '#\n'
211			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
212			txt += '#\n\n'
213
214			txt += 'func_exit() { \n'
215			txt += self.wsExitFunc_common()
216			### specific Glite check for OSB
217			txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
218			txt += ' tmp_size=`ls -gGrta ${out_files}.tgz \| awk \'{ print $3 }\'`\n'
219	ewv	1.61	txt += ' rm ${out_files}.tgz\n'
220	spiga	1.51	txt += ' size=`expr $tmp_size`\n'
221			txt += ' echo "Total Output dimension: $size"\n'
222	ewv	1.61	txt += ' limit='+str(self.OSBsize) +' \n'
223	spiga	1.51	txt += ' echo "WARNING: output files size limit is set to: $limit"\n'
224	spiga	1.53	txt += ' if [ "$limit" -lt "$size" ]; then\n'
225	spiga	1.51	txt += ' exceed=1\n'
226			txt += ' job_exit_code=70000\n'
227			txt += ' echo "Output Sanbox too big. Produced output is lost "\n'
228			txt += ' else\n'
229			txt += ' exceed=0\n'
230	spiga	1.53	txt += ' echo "Total Output dimension $size is fine."\n'
231	spiga	1.51	txt += ' fi\n'
232
233			txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
234			txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
235			txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
236			txt += ' if [ $exceed -ne 1 ]; then\n'
237			txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
238			txt += ' else\n'
239			txt += ' tar zcvf ${out_files}.tgz CMSSW_${NJob}.stdout CMSSW_${NJob}.stderr\n'
240			txt += ' fi\n'
241	spiga	1.73	txt += ' python $RUNTIME_AREA/fillCrabFjr.py $RUNTIME_AREA/crab_fjr_$NJob.xml --errorcode $job_exit_code \n'
242	spiga	1.51	txt += ' exit $job_exit_code\n'
243
244			txt += '}\n'
245			return txt
246	spigafi	1.78
247			def listMatch(self, dest, full):
248			matching='fast'
249
250			if self.boss().schedulerConfig['name'] == 'SchedulerGLite' :
251			taskId=common._db.getTask()
252			req=str(self.sched_parameter(1,taskId))
253			sites = self.boss().schedSession().matchResources(taskId, requirements=req)
254			else :
255			sites = SchedulerGrid.listMatch(self, dest, full)
256
257			if full == True: matching='full'
258			common.logger.debug("list of available site ( "+str(matching) +" matching ) : "+str(sites))
259
260			return sites