CRAB/python/SchedulerCondor.py

"""
Implements the vanilla (local) Condor scheduler
"""

__revision__ = "$Id: SchedulerCondor.py,v 1.13 2008/09/30 19:35:34 ewv Exp $"
__version__ = "$Revision: 1.13 $"

from SchedulerLocal  import SchedulerLocal
from crab_exceptions import CrabException

import common
import os


class SchedulerCondor(SchedulerLocal) :
    """
    Class to implement the vanilla (local) Condor scheduler
     Naming convention:  Methods starting with 'ws' provide
     the corresponding part of the job script
     ('ws' stands for 'write script').
    """

    def __init__(self):
        SchedulerLocal.__init__(self,"CONDOR")
        self.datasetPath   = None
        self.selectNoInput = None
        self.environment_unique_identifier = None
        return


    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        SchedulerLocal.configure(self, cfg_params)
        self.environment_unique_identifier = '${HOSTNAME}_${CONDOR_ID}_' \
                                             + common._db.queryTask('name')

        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        return


    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters
        """

        index = int(common._db.nJobs()) - 1
        schedParam = ''

        for i in range(index):
            pass

        return schedParam


    def realSchedParams(self, cfg_params):
        """
        Return dictionary with specific parameters, to use with real scheduler
        """

        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        jobDir = common.work_space.jobDir()
        params = {'tmpDir':tmpDir,
                  'jobDir':jobDir}
        return params


    def listMatch(self, seList, full):
        """
        Check the compatibility of available resources
        """

        if self.selectNoInput:
            return [True]
        else:
            return SchedulerLocal.listMatch(self, seList, full)


    def decodeLogInfo(self, fileName):
        """
        Parse logging info file and return main info
        """

        import CondorGLoggingInfo
        loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
        reason = loggingInfo.decodeReason(fileName)
        return reason


    def wsCopyOutput(self):
        """
        Write a CopyResults part of a job script, e.g.
        to copy produced output into a storage element.
        """
        txt = self.wsCopyOutput_comm()
        return txt


    def wsExitFunc(self):
        """
        Returns the part of the job script which runs prior to exit
        """

        txt = '\n'
        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()

        txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
        txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'

        txt += '    exit $job_exit_code\n'
        txt += '}\n'

        return txt

    def wsInitialEnvironment(self):
        """
        Returns part of a job script which does scheduler-specific work.
        """

        txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
        txt += 'echo "Beginning environment"\n'
        txt += 'printenv | sort\n'

        txt += 'middleware='+self.name()+' \n'
        txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
        txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
        txt += 'fi\n'

        txt += """
if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
    echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
    if [ -e ../default.tgz ] ;then
      echo "Found ISB in parent directory (Local Condor)"
      cp ../default.tgz $_CONDOR_SCRATCH_DIR
    fi
    cd $_CONDOR_SCRATCH_DIR
fi
"""

        return txt
Revision:	1.14
Committed:	Fri Oct 10 19:24:44 2008 UTC (16 years, 6 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_4_3_pre3, CRAB_2_4_3_pre2, CRAB_2_4_3_pre1, CRAB_2_4_2, CRAB_2_4_2_pre3, CRAB_2_4_2_pre2, CRAB_2_4_2_pre1, CRAB_2_4_1, CRAB_2_4_1_pre4, CRAB_2_4_1_pre3, CRAB_2_4_1_pre2, CRAB_2_4_1_pre1, CRAB_2_4_0_Tutorial, CRAB_2_4_0_Tutorial_pre1, CRAB_2_4_0, CRAB_2_4_0_pre9
Changes since 1.13:	+20 -5 lines
Log Message:	Support for cmscp in SchedulerCondor
#	User	Rev	Content
1	ewv	1.10	"""
2			Implements the vanilla (local) Condor scheduler
3			"""
4
5	ewv	1.14	__revision__ = "$Id: SchedulerCondor.py,v 1.13 2008/09/30 19:35:34 ewv Exp $"
6			__version__ = "$Revision: 1.13 $"
7	ewv	1.10
8			from SchedulerLocal import SchedulerLocal
9			from crab_exceptions import CrabException
10	ewv	1.1
11			import common
12			import os
13
14
15			class SchedulerCondor(SchedulerLocal) :
16	ewv	1.10	"""
17			Class to implement the vanilla (local) Condor scheduler
18	ewv	1.14	Naming convention: Methods starting with 'ws' provide
19			the corresponding part of the job script
20			('ws' stands for 'write script').
21	ewv	1.10	"""
22
23			def __init__(self):
24			SchedulerLocal.__init__(self,"CONDOR")
25			self.datasetPath = None
26			self.selectNoInput = None
27			self.environment_unique_identifier = None
28			return
29
30
31			def configure(self, cfg_params):
32			"""
33			Configure the scheduler with the config settings from the user
34			"""
35
36			SchedulerLocal.configure(self, cfg_params)
37	ewv	1.14	self.environment_unique_identifier = '${HOSTNAME}_${CONDOR_ID}_' \
38			+ common._db.queryTask('name')
39	ewv	1.10
40			try:
41			tmp = cfg_params['CMSSW.datasetpath']
42			if tmp.lower() == 'none':
43			self.datasetPath = None
44			self.selectNoInput = 1
45			else:
46			self.datasetPath = tmp
47			self.selectNoInput = 0
48			except KeyError:
49			msg = "Error: datasetpath not defined "
50			raise CrabException(msg)
51
52			return
53	ewv	1.1
54	ewv	1.7
55	ewv	1.10	def sched_parameter(self, i, task):
56			"""
57			Return scheduler-specific parameters
58			"""
59	ewv	1.1
60	ewv	1.10	index = int(common._db.nJobs()) - 1
61			schedParam = ''
62	ewv	1.3
63	ewv	1.10	for i in range(index):
64			pass
65
66			return schedParam
67	ewv	1.1
68
69	ewv	1.10	def realSchedParams(self, cfg_params):
70			"""
71			Return dictionary with specific parameters, to use with real scheduler
72			"""
73	ewv	1.1
74	ewv	1.10	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
75	ewv	1.13	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
76			jobDir = common.work_space.jobDir()
77			params = {'tmpDir':tmpDir,
78			'jobDir':jobDir}
79	ewv	1.10	return params
80	ewv	1.3
81	ewv	1.2
82	ewv	1.10	def listMatch(self, seList, full):
83			"""
84			Check the compatibility of available resources
85			"""
86	ewv	1.2
87	ewv	1.10	if self.selectNoInput:
88			return [True]
89			else:
90			return SchedulerLocal.listMatch(self, seList, full)
91	ewv	1.3
92	ewv	1.7
93	ewv	1.10	def decodeLogInfo(self, fileName):
94			"""
95			Parse logging info file and return main info
96			"""
97	ewv	1.7
98	ewv	1.10	import CondorGLoggingInfo
99			loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
100			reason = loggingInfo.decodeReason(fileName)
101			return reason
102	ewv	1.7
103	ewv	1.3
104	ewv	1.14	def wsCopyOutput(self):
105			"""
106			Write a CopyResults part of a job script, e.g.
107			to copy produced output into a storage element.
108			"""
109			txt = self.wsCopyOutput_comm()
110			return txt
111
112
113	ewv	1.10	def wsExitFunc(self):
114			"""
115			Returns the part of the job script which runs prior to exit
116			"""
117	ewv	1.3
118	ewv	1.10	txt = '\n'
119			txt += '#\n'
120			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
121			txt += '#\n\n'
122	ewv	1.3
123	ewv	1.10	txt += 'func_exit() { \n'
124			txt += self.wsExitFunc_common()
125	spiga	1.8
126	ewv	1.10	txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
127	ewv	1.12	txt += ' cp ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
128			txt += ' cp crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
129	ewv	1.3
130	ewv	1.10	txt += ' exit $job_exit_code\n'
131			txt += '}\n'
132	ewv	1.3
133	ewv	1.10	return txt
134	ewv	1.5
135	ewv	1.10	def wsInitialEnvironment(self):
136			"""
137			Returns part of a job script which does scheduler-specific work.
138			"""
139	ewv	1.5
140	ewv	1.10	txt = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
141			txt += 'echo "Beginning environment"\n'
142			txt += 'printenv \| sort\n'
143	ewv	1.5
144	ewv	1.10	txt += 'middleware='+self.name()+' \n'
145	ewv	1.14	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
146			txt += ' export PATH=${PATH}:/opt/d-cache/srm/bin\n'
147			txt += 'fi\n'
148
149	ewv	1.10	txt += """
150	ewv	1.6	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
151	ewv	1.12	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
152	ewv	1.6	if [ -e ../default.tgz ] ;then
153			echo "Found ISB in parent directory (Local Condor)"
154			cp ../default.tgz $_CONDOR_SCRATCH_DIR
155			fi
156			cd $_CONDOR_SCRATCH_DIR
157			fi
158			"""
159	ewv	1.5
160	ewv	1.10	return txt