CRAB/python/SchedulerCondor.py

"""
Implements the vanilla (local) Condor scheduler
"""

__revision__ = "$Id: SchedulerCondor.py,v 1.15 2008/11/25 22:24:23 ewv Exp $"
__version__ = "$Revision: 1.15 $"

from SchedulerLocal  import SchedulerLocal
from crab_exceptions import CrabException

import common
import os


class SchedulerCondor(SchedulerLocal) :
    """
    Class to implement the vanilla (local) Condor scheduler
     Naming convention:  Methods starting with 'ws' provide
     the corresponding part of the job script
     ('ws' stands for 'write script').
    """

    def __init__(self):
        SchedulerLocal.__init__(self,"CONDOR")
        self.datasetPath   = None
        self.selectNoInput = None
        self.environment_unique_identifier = None
        return


    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        SchedulerLocal.configure(self, cfg_params)
        self.environment_unique_identifier = '${HOSTNAME}_${CONDOR_ID}_' \
                                             + common._db.queryTask('name')

        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        self.return_data = cfg_params.get('USER.return_data', 0)
        self.copy_data   = cfg_params.get("USER.copy_data", 0)

        if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ):
            msg = 'Error: return_data and copy_data cannot be set both to 0\n'
            msg = msg + 'Please modify your crab.cfg file\n'
            raise CrabException(msg)

        if ( int(self.return_data) == 1 and int(self.copy_data) == 1 ):
            msg = 'Error: return_data and copy_data cannot be set both to 1\n'
            msg = msg + 'Please modify your crab.cfg file\n'
            raise CrabException(msg)

        if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
            msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
            msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
            common.logger.message(msg)
            raise CrabException(msg)

        if int(self.copy_data) == 1:
            self.SE = cfg_params.get('USER.storage_element', None)
            if not self.SE:
                msg = "Error. The [USER] section has no 'storage_element'"
                common.logger.message(msg)
                raise CrabException(msg)

            self.proxyValid = 0
            self.dontCheckProxy = int(cfg_params.get("EDG.dont_check_proxy",0))
            self.proxyServer = cfg_params.get("EDG.proxy_server",'myproxy.cern.ch')
            common.logger.debug(5,'Setting myproxy server to ' + self.proxyServer)

            self.group = cfg_params.get("EDG.group", None)
            self.role  = cfg_params.get("EDG.role", None)
            self.VO    = cfg_params.get('EDG.virtual_organization', 'cms')

            self.checkProxy()
        self.role  = None

        return


    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters
        """

        index = int(common._db.nJobs()) - 1
        schedParam = ''

        for i in range(index):
            pass

        return schedParam


    def realSchedParams(self, cfg_params):
        """
        Return dictionary with specific parameters, to use with real scheduler
        """

        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        jobDir = common.work_space.jobDir()
        params = {'tmpDir':tmpDir,
                  'jobDir':jobDir}
        return params


    def listMatch(self, seList, full):
        """
        Check the compatibility of available resources
        """

        return [True]


    def decodeLogInfo(self, fileName):
        """
        Parse logging info file and return main info
        """

        import CondorGLoggingInfo
        loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
        reason = loggingInfo.decodeReason(fileName)
        return reason


    def wsCopyOutput(self):
        """
        Write a CopyResults part of a job script, e.g.
        to copy produced output into a storage element.
        """
        txt = self.wsCopyOutput_comm()
        return txt


    def wsExitFunc(self):
        """
        Returns the part of the job script which runs prior to exit
        """

        txt = '\n'
        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()

        txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
        txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'

        txt += '    exit $job_exit_code\n'
        txt += '}\n'

        return txt

    def wsInitialEnvironment(self):
        """
        Returns part of a job script which does scheduler-specific work.
        """

        txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
        txt += 'echo "Beginning environment"\n'
        txt += 'printenv | sort\n'

        txt += 'middleware='+self.name()+' \n'
        txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
        txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
        txt += 'fi\n'

        txt += """
if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
    echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
    if [ -e ../default.tgz ] ;then
      echo "Found ISB in parent directory (Local Condor)"
      cp ../default.tgz $_CONDOR_SCRATCH_DIR
    fi
    cd $_CONDOR_SCRATCH_DIR
fi
"""

        return txt
Revision:	1.16
Committed:	Mon Jan 26 19:43:48 2009 UTC (16 years, 3 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_5_0_pre2, CRAB_2_5_0_pre1
Changes since 1.15:	+4 -6 lines
Log Message:	Dummy listmatch for Condor
#	User	Rev	Content
1	ewv	1.10	"""
2			Implements the vanilla (local) Condor scheduler
3			"""
4
5	ewv	1.16	__revision__ = "$Id: SchedulerCondor.py,v 1.15 2008/11/25 22:24:23 ewv Exp $"
6			__version__ = "$Revision: 1.15 $"
7	ewv	1.10
8			from SchedulerLocal import SchedulerLocal
9			from crab_exceptions import CrabException
10	ewv	1.1
11			import common
12			import os
13
14
15			class SchedulerCondor(SchedulerLocal) :
16	ewv	1.10	"""
17			Class to implement the vanilla (local) Condor scheduler
18	ewv	1.14	Naming convention: Methods starting with 'ws' provide
19			the corresponding part of the job script
20			('ws' stands for 'write script').
21	ewv	1.10	"""
22
23			def __init__(self):
24			SchedulerLocal.__init__(self,"CONDOR")
25			self.datasetPath = None
26			self.selectNoInput = None
27			self.environment_unique_identifier = None
28			return
29
30
31			def configure(self, cfg_params):
32			"""
33			Configure the scheduler with the config settings from the user
34			"""
35
36			SchedulerLocal.configure(self, cfg_params)
37	ewv	1.14	self.environment_unique_identifier = '${HOSTNAME}_${CONDOR_ID}_' \
38			+ common._db.queryTask('name')
39	ewv	1.10
40			try:
41			tmp = cfg_params['CMSSW.datasetpath']
42			if tmp.lower() == 'none':
43			self.datasetPath = None
44			self.selectNoInput = 1
45			else:
46			self.datasetPath = tmp
47			self.selectNoInput = 0
48			except KeyError:
49			msg = "Error: datasetpath not defined "
50			raise CrabException(msg)
51
52	ewv	1.15	self.return_data = cfg_params.get('USER.return_data', 0)
53			self.copy_data = cfg_params.get("USER.copy_data", 0)
54
55			if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ):
56			msg = 'Error: return_data and copy_data cannot be set both to 0\n'
57			msg = msg + 'Please modify your crab.cfg file\n'
58			raise CrabException(msg)
59
60			if ( int(self.return_data) == 1 and int(self.copy_data) == 1 ):
61			msg = 'Error: return_data and copy_data cannot be set both to 1\n'
62			msg = msg + 'Please modify your crab.cfg file\n'
63			raise CrabException(msg)
64
65			if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
66			msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
67			msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
68			common.logger.message(msg)
69			raise CrabException(msg)
70
71			if int(self.copy_data) == 1:
72			self.SE = cfg_params.get('USER.storage_element', None)
73			if not self.SE:
74			msg = "Error. The [USER] section has no 'storage_element'"
75			common.logger.message(msg)
76			raise CrabException(msg)
77
78			self.proxyValid = 0
79			self.dontCheckProxy = int(cfg_params.get("EDG.dont_check_proxy",0))
80			self.proxyServer = cfg_params.get("EDG.proxy_server",'myproxy.cern.ch')
81			common.logger.debug(5,'Setting myproxy server to ' + self.proxyServer)
82
83			self.group = cfg_params.get("EDG.group", None)
84			self.role = cfg_params.get("EDG.role", None)
85			self.VO = cfg_params.get('EDG.virtual_organization', 'cms')
86
87			self.checkProxy()
88	ewv	1.16	self.role = None
89	ewv	1.15
90	ewv	1.10	return
91	ewv	1.1
92	ewv	1.7
93	ewv	1.10	def sched_parameter(self, i, task):
94			"""
95			Return scheduler-specific parameters
96			"""
97	ewv	1.1
98	ewv	1.10	index = int(common._db.nJobs()) - 1
99			schedParam = ''
100	ewv	1.3
101	ewv	1.10	for i in range(index):
102			pass
103
104			return schedParam
105	ewv	1.1
106
107	ewv	1.10	def realSchedParams(self, cfg_params):
108			"""
109			Return dictionary with specific parameters, to use with real scheduler
110			"""
111	ewv	1.1
112	ewv	1.10	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
113	ewv	1.13	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
114			jobDir = common.work_space.jobDir()
115			params = {'tmpDir':tmpDir,
116			'jobDir':jobDir}
117	ewv	1.10	return params
118	ewv	1.3
119	ewv	1.2
120	ewv	1.10	def listMatch(self, seList, full):
121			"""
122			Check the compatibility of available resources
123			"""
124	ewv	1.2
125	ewv	1.16	return [True]
126	ewv	1.3
127	ewv	1.7
128	ewv	1.10	def decodeLogInfo(self, fileName):
129			"""
130			Parse logging info file and return main info
131			"""
132	ewv	1.7
133	ewv	1.10	import CondorGLoggingInfo
134			loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
135			reason = loggingInfo.decodeReason(fileName)
136			return reason
137	ewv	1.7
138	ewv	1.3
139	ewv	1.14	def wsCopyOutput(self):
140			"""
141			Write a CopyResults part of a job script, e.g.
142			to copy produced output into a storage element.
143			"""
144			txt = self.wsCopyOutput_comm()
145			return txt
146
147
148	ewv	1.10	def wsExitFunc(self):
149			"""
150			Returns the part of the job script which runs prior to exit
151			"""
152	ewv	1.3
153	ewv	1.10	txt = '\n'
154			txt += '#\n'
155			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
156			txt += '#\n\n'
157	ewv	1.3
158	ewv	1.10	txt += 'func_exit() { \n'
159			txt += self.wsExitFunc_common()
160	spiga	1.8
161	ewv	1.10	txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
162	ewv	1.12	txt += ' cp ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
163			txt += ' cp crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
164	ewv	1.3
165	ewv	1.10	txt += ' exit $job_exit_code\n'
166			txt += '}\n'
167	ewv	1.3
168	ewv	1.10	return txt
169	ewv	1.5
170	ewv	1.10	def wsInitialEnvironment(self):
171			"""
172			Returns part of a job script which does scheduler-specific work.
173			"""
174	ewv	1.5
175	ewv	1.10	txt = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
176			txt += 'echo "Beginning environment"\n'
177			txt += 'printenv \| sort\n'
178	ewv	1.5
179	ewv	1.10	txt += 'middleware='+self.name()+' \n'
180	ewv	1.14	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
181			txt += ' export PATH=${PATH}:/opt/d-cache/srm/bin\n'
182			txt += 'fi\n'
183
184	ewv	1.10	txt += """
185	ewv	1.6	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
186	ewv	1.12	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
187	ewv	1.6	if [ -e ../default.tgz ] ;then
188			echo "Found ISB in parent directory (Local Condor)"
189			cp ../default.tgz $_CONDOR_SCRATCH_DIR
190			fi
191			cd $_CONDOR_SCRATCH_DIR
192			fi
193			"""
194	ewv	1.5
195	ewv	1.10	return txt