CRAB/python/SchedulerSge.py

from Scheduler import Scheduler
from SchedulerLocal import SchedulerLocal
from crab_exceptions import *
from crab_util import *
import common

import os,string

#
#  Naming convention:
#  methods starting with 'ws' are responsible to provide
#  corresponding part of the job script ('ws' stands for 'write script').
#
# Author: Hartmut Stadie <stadie@mail.desy.de> Inst. f. Experimentalphysik; Universitaet Hamburg
#

class SchedulerSge(SchedulerLocal) :

    def __init__(self):
        Scheduler.__init__(self,"SGE")
        self.datasetPath   = None
        self.selectNoInput = None
        return

    def configure(self, cfg_params):
        SchedulerLocal.configure(self, cfg_params)

        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        self.pool = cfg_params.get('USER.storage_pool',None)
        return

    def envUniqueID(self):
        id = "https://"+common.scheduler.name()+":/${JOB_ID}-"+ \
            string.replace(common._db.queryTask('name'),"_","-")
        return id

    def realSchedParams(self,cfg_params):
        """
        Return dictionary with specific parameters, to use
        with real scheduler
        """
        params = {}
        return  params

    def sched_parameter(self,i,task):
        """
        Returns parameter scheduler-specific, to use with BOSS .
        """
        index = int(common._db.nJobs()) - 1
        sched_param= ''

        for i in range(index): # Add loop DS
            sched_param= ''
            if (self.queue):
                sched_param += '-q '+self.queue +' '
            if (self.res): sched_param += '-l '+self.res +' '

        pass

        #default is request 2G memory and 48 hours CPU time
        #sched_param += ' -V -l h_vmem=2G -l h_cpu=172800 '
#        sched_param += ' -V -l h_vmem='
#        sched_param += self.vmem.__str__()
#        sched_param += 'G -l h_cpu='
#        sched_param += self.cpu.__str__()
#        sched_param += ' '

        return sched_param

    def loggingInfo(self, id):
        """ return logging info about job nj """
        print "Warning: SchedulerSge::loggingInfo not implemented!"
        return ""

    def wsExitFunc(self):
        """
        """
        txt = '\n'

        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()

        txt += '    cp ${SGE_STDOUT_PATH} CMSSW_${NJob}.stdout \n'
        txt += '    cp ${SGE_STDERR_PATH} CMSSW_${NJob}.stderr \n'
        txt += '    tar zcvf ${out_files}.tgz  ${filesToCheck}\n'
        txt += '    exit $job_exit_code\n'
        txt += '}\n'

        return txt

    def listMatch(self, dest, full):
        """
        """
        sites = [self.blackWhiteListParser.cleanForBlackWhiteList(dest,'list')]
        return sites

    def wsCopyOutput(self):
        txt=self.wsCopyOutput_comm(self.pool)
        return txt

    def userName(self):
        """ return the user name """

        ## hack for german naf
        import pwd,getpass
        tmp=pwd.getpwnam(getpass.getuser())[4]
        tmp=tmp.rstrip(',')
        tmp=tmp.rstrip(',')
        tmp=tmp.rstrip(',')


        return "/CN="+tmp.strip()
Revision:	1.12
Committed:	Tue May 11 13:39:50 2010 UTC (14 years, 11 months ago) by farinafa
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta
Changes since 1.11:	+0 -44 lines
Log Message:	Fix for bug #67005: myproxy_server is now got from cmsdoc, proxy_server configurable deprecated. Plus a number of clean-up for code referring to pre-Credential API epoch
#	User	Rev	Content
1	spiga	1.7	from Scheduler import Scheduler
2			from SchedulerLocal import SchedulerLocal
3			from crab_exceptions import *
4			from crab_util import *
5			import common
6
7			import os,string
8
9			#
10			# Naming convention:
11			# methods starting with 'ws' are responsible to provide
12			# corresponding part of the job script ('ws' stands for 'write script').
13			#
14			# Author: Hartmut Stadie <stadie@mail.desy.de> Inst. f. Experimentalphysik; Universitaet Hamburg
15			#
16
17			class SchedulerSge(SchedulerLocal) :
18
19			def __init__(self):
20			Scheduler.__init__(self,"SGE")
21			self.datasetPath = None
22			self.selectNoInput = None
23	spiga	1.6	return
24	slacapra	1.1
25	spiga	1.7	def configure(self, cfg_params):
26			SchedulerLocal.configure(self, cfg_params)
27
28			try:
29			tmp = cfg_params['CMSSW.datasetpath']
30			if tmp.lower() == 'none':
31			self.datasetPath = None
32			self.selectNoInput = 1
33			else:
34			self.datasetPath = tmp
35			self.selectNoInput = 0
36			except KeyError:
37			msg = "Error: datasetpath not defined "
38			raise CrabException(msg)
39
40			self.pool = cfg_params.get('USER.storage_pool',None)
41			return
42
43			def envUniqueID(self):
44			id = "https://"+common.scheduler.name()+":/${JOB_ID}-"+ \
45			string.replace(common._db.queryTask('name'),"_","-")
46			return id
47
48			def realSchedParams(self,cfg_params):
49	spiga	1.6	"""
50	spiga	1.7	Return dictionary with specific parameters, to use
51			with real scheduler
52	spiga	1.6	"""
53	spiga	1.7	params = {}
54			return params
55	slacapra	1.1
56	spiga	1.7	def sched_parameter(self,i,task):
57	spiga	1.6	"""
58	spiga	1.7	Returns parameter scheduler-specific, to use with BOSS .
59	spiga	1.6	"""
60	spiga	1.7	index = int(common._db.nJobs()) - 1
61			sched_param= ''
62	ewv	1.5
63	spiga	1.7	for i in range(index): # Add loop DS
64			sched_param= ''
65			if (self.queue):
66			sched_param += '-q '+self.queue +' '
67	spiga	1.11	if (self.res): sched_param += '-l '+self.res +' '
68	mcinquil	1.10
69			pass
70	slacapra	1.1
71	spiga	1.7	#default is request 2G memory and 48 hours CPU time
72			#sched_param += ' -V -l h_vmem=2G -l h_cpu=172800 '
73			# sched_param += ' -V -l h_vmem='
74			# sched_param += self.vmem.__str__()
75			# sched_param += 'G -l h_cpu='
76			# sched_param += self.cpu.__str__()
77			# sched_param += ' '
78	spiga	1.4
79	spiga	1.7	return sched_param
80	spiga	1.6
81	spiga	1.7	def loggingInfo(self, id):
82			""" return logging info about job nj """
83			print "Warning: SchedulerSge::loggingInfo not implemented!"
84			return ""
85	spiga	1.6
86	spiga	1.7	def wsExitFunc(self):
87	slacapra	1.1	"""
88			"""
89	spiga	1.7	txt = '\n'
90	slacapra	1.1
91	spiga	1.7	txt += '#\n'
92			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
93			txt += '#\n\n'
94	slacapra	1.1
95	spiga	1.7	txt += 'func_exit() { \n'
96			txt += self.wsExitFunc_common()
97	spiga	1.3
98	spiga	1.7	txt += ' cp ${SGE_STDOUT_PATH} CMSSW_${NJob}.stdout \n'
99			txt += ' cp ${SGE_STDERR_PATH} CMSSW_${NJob}.stderr \n'
100			txt += ' tar zcvf ${out_files}.tgz ${filesToCheck}\n'
101			txt += ' exit $job_exit_code\n'
102			txt += '}\n'
103	slacapra	1.1
104	spiga	1.7	return txt
105	slacapra	1.1
106	spiga	1.7	def listMatch(self, dest, full):
107	slacapra	1.1	"""
108			"""
109	spiga	1.7	sites = [self.blackWhiteListParser.cleanForBlackWhiteList(dest,'list')]
110			return sites
111	slacapra	1.1
112	spiga	1.7	def wsCopyOutput(self):
113			txt=self.wsCopyOutput_comm(self.pool)
114			return txt
115	spiga	1.2
116	spiga	1.7	def userName(self):
117			""" return the user name """
118	slacapra	1.1
119	spiga	1.7	## hack for german naf
120			import pwd,getpass
121			tmp=pwd.getpwnam(getpass.getuser())[4]
122			tmp=tmp.rstrip(',')
123			tmp=tmp.rstrip(',')
124			tmp=tmp.rstrip(',')
125	slacapra	1.1
126	ewv	1.5
127	spiga	1.7	return "/CN="+tmp.strip()