CRAB/python/SchedulerCondor.py

"""
Implements the vanilla (local) Condor scheduler
"""

__revision__ = "$Id: SchedulerCondor.py,v 1.25 2009/05/26 10:23:01 spiga Exp $"
__version__ = "$Revision: 1.25 $"

from SchedulerLocal  import SchedulerLocal
from crab_exceptions import CrabException

import common
import os
import socket
import sha

class SchedulerCondor(SchedulerLocal) :
    """
    Class to implement the vanilla (local) Condor scheduler
     Naming convention:  Methods starting with 'ws' provide
     the corresponding part of the job script
     ('ws' stands for 'write script').
    """

    def __init__(self):
        SchedulerLocal.__init__(self,"CONDOR")
        self.datasetPath   = None
        self.selectNoInput = None
        self.return_data   = 0
        self.copy_data     = 0

        self.environment_unique_identifier = None
        return


    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        SchedulerLocal.configure(self, cfg_params)

        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        self.return_data = cfg_params.get('USER.return_data', 0)
        self.copy_data   = cfg_params.get("USER.copy_data", 0)

        self.proxyValid = 0
        self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
        self.proxyServer = cfg_params.get("GRID.proxy_server", 'myproxy.cern.ch')
        common.logger.debug('Setting myproxy server to ' + self.proxyServer)

        self.group = cfg_params.get("GRID.group", None)
        self.role  = cfg_params.get("GRID.role", None)
        self.VO    = cfg_params.get('GRID.virtual_organization', 'cms')

        self.checkProxy()

        return

    def envUniqueID(self):
        taskHash = sha.new(common._db.queryTask('name')).hexdigest()
        id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
        return id

    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters
        """

        index = int(common._db.nJobs()) - 1
        schedParam = ''

        for i in range(index):
            pass

        return schedParam


    def realSchedParams(self, cfg_params):
        """
        Return dictionary with specific parameters, to use with real scheduler
        """

        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
        jobDir = common.work_space.jobDir()
        params = {'tmpDir':tmpDir,
                  'jobDir':jobDir}
        return params


    def listMatch(self, seList, full):
        """
        Check the compatibility of available resources
        """

        return [True]


    def decodeLogInfo(self, fileName):
        """
        Parse logging info file and return main info
        """

        import CondorGLoggingInfo
        loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
        reason = loggingInfo.decodeReason(fileName)
        return reason


    def wsCopyOutput(self):
        """
        Write a CopyResults part of a job script, e.g.
        to copy produced output into a storage element.
        """
        txt = self.wsCopyOutput_comm()
        return txt


    def wsExitFunc(self):
        """
        Returns the part of the job script which runs prior to exit
        """

        txt = '\n'
        txt += '#\n'
        txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
        txt += '#\n\n'

        txt += 'func_exit() { \n'
        txt += self.wsExitFunc_common()

        txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
        txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
        txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'

        txt += '    exit $job_exit_code\n'
        txt += '}\n'

        return txt

    def wsInitialEnvironment(self):
        """
        Returns part of a job script which does scheduler-specific work.
        """

        txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
        txt += 'echo "Beginning environment"\n'
        txt += 'printenv | sort\n'

        txt += 'middleware='+self.name()+' \n'
        txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
        txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
        txt += 'fi\n'

        txt += """
if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
    echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
    if [ -e ../default.tgz ] ;then
      echo "Found ISB in parent directory (Local Condor)"
      cp ../default.tgz $_CONDOR_SCRATCH_DIR
    fi
    cd $_CONDOR_SCRATCH_DIR
fi
"""

        return txt
Revision:	1.26
Committed:	Tue May 26 16:53:23 2009 UTC (15 years, 11 months ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, CRAB_2_6_0_pre7, CRAB_2_6_0_pre6, CRAB_2_6_0_pre5, CRAB_2_6_0_pre4, CRAB_2_6_0_pre3
Changes since 1.25:	+7 -7 lines
Log Message:	adapting code to EDG --> GRID migration
#	User	Rev	Content
1	ewv	1.10	"""
2			Implements the vanilla (local) Condor scheduler
3			"""
4
5	spiga	1.26	__revision__ = "$Id: SchedulerCondor.py,v 1.25 2009/05/26 10:23:01 spiga Exp $"
6			__version__ = "$Revision: 1.25 $"
7	ewv	1.10
8			from SchedulerLocal import SchedulerLocal
9			from crab_exceptions import CrabException
10	ewv	1.1
11			import common
12			import os
13	ewv	1.17	import socket
14			import sha
15	ewv	1.1
16			class SchedulerCondor(SchedulerLocal) :
17	ewv	1.10	"""
18			Class to implement the vanilla (local) Condor scheduler
19	ewv	1.14	Naming convention: Methods starting with 'ws' provide
20			the corresponding part of the job script
21			('ws' stands for 'write script').
22	ewv	1.10	"""
23
24			def __init__(self):
25			SchedulerLocal.__init__(self,"CONDOR")
26			self.datasetPath = None
27			self.selectNoInput = None
28	ewv	1.19	self.return_data = 0
29			self.copy_data = 0
30
31	ewv	1.10	self.environment_unique_identifier = None
32			return
33
34
35			def configure(self, cfg_params):
36			"""
37			Configure the scheduler with the config settings from the user
38			"""
39
40			SchedulerLocal.configure(self, cfg_params)
41
42			try:
43			tmp = cfg_params['CMSSW.datasetpath']
44			if tmp.lower() == 'none':
45			self.datasetPath = None
46			self.selectNoInput = 1
47			else:
48			self.datasetPath = tmp
49			self.selectNoInput = 0
50			except KeyError:
51			msg = "Error: datasetpath not defined "
52			raise CrabException(msg)
53
54	ewv	1.15	self.return_data = cfg_params.get('USER.return_data', 0)
55			self.copy_data = cfg_params.get("USER.copy_data", 0)
56
57	ewv	1.24	self.proxyValid = 0
58	spiga	1.26	self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
59			self.proxyServer = cfg_params.get("GRID.proxy_server", 'myproxy.cern.ch')
60	spiga	1.25	common.logger.debug('Setting myproxy server to ' + self.proxyServer)
61	ewv	1.24
62	spiga	1.26	self.group = cfg_params.get("GRID.group", None)
63			self.role = cfg_params.get("GRID.role", None)
64			self.VO = cfg_params.get('GRID.virtual_organization', 'cms')
65	ewv	1.15
66	ewv	1.24	self.checkProxy()
67	ewv	1.15
68	ewv	1.10	return
69	ewv	1.1
70	ewv	1.23	def envUniqueID(self):
71	spiga	1.22	taskHash = sha.new(common._db.queryTask('name')).hexdigest()
72			id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
73			return id
74	ewv	1.7
75	ewv	1.10	def sched_parameter(self, i, task):
76			"""
77			Return scheduler-specific parameters
78			"""
79	ewv	1.1
80	ewv	1.10	index = int(common._db.nJobs()) - 1
81			schedParam = ''
82	ewv	1.3
83	ewv	1.10	for i in range(index):
84			pass
85
86			return schedParam
87	ewv	1.1
88
89	ewv	1.10	def realSchedParams(self, cfg_params):
90			"""
91			Return dictionary with specific parameters, to use with real scheduler
92			"""
93	ewv	1.1
94	ewv	1.10	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
95	ewv	1.13	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
96			jobDir = common.work_space.jobDir()
97			params = {'tmpDir':tmpDir,
98			'jobDir':jobDir}
99	ewv	1.10	return params
100	ewv	1.3
101	ewv	1.2
102	ewv	1.10	def listMatch(self, seList, full):
103			"""
104			Check the compatibility of available resources
105			"""
106	ewv	1.2
107	ewv	1.16	return [True]
108	ewv	1.3
109	ewv	1.7
110	ewv	1.10	def decodeLogInfo(self, fileName):
111			"""
112			Parse logging info file and return main info
113			"""
114	ewv	1.7
115	ewv	1.10	import CondorGLoggingInfo
116			loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
117			reason = loggingInfo.decodeReason(fileName)
118			return reason
119	ewv	1.7
120	ewv	1.3
121	ewv	1.14	def wsCopyOutput(self):
122			"""
123			Write a CopyResults part of a job script, e.g.
124			to copy produced output into a storage element.
125			"""
126			txt = self.wsCopyOutput_comm()
127			return txt
128
129
130	ewv	1.10	def wsExitFunc(self):
131			"""
132			Returns the part of the job script which runs prior to exit
133			"""
134	ewv	1.3
135	ewv	1.10	txt = '\n'
136			txt += '#\n'
137			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
138			txt += '#\n\n'
139	ewv	1.3
140	ewv	1.10	txt += 'func_exit() { \n'
141			txt += self.wsExitFunc_common()
142	spiga	1.8
143	ewv	1.10	txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
144	ewv	1.12	txt += ' cp ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
145			txt += ' cp crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
146	ewv	1.3
147	ewv	1.10	txt += ' exit $job_exit_code\n'
148			txt += '}\n'
149	ewv	1.3
150	ewv	1.10	return txt
151	ewv	1.5
152	ewv	1.10	def wsInitialEnvironment(self):
153			"""
154			Returns part of a job script which does scheduler-specific work.
155			"""
156	ewv	1.5
157	ewv	1.10	txt = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
158			txt += 'echo "Beginning environment"\n'
159			txt += 'printenv \| sort\n'
160	ewv	1.5
161	ewv	1.10	txt += 'middleware='+self.name()+' \n'
162	ewv	1.14	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
163			txt += ' export PATH=${PATH}:/opt/d-cache/srm/bin\n'
164			txt += 'fi\n'
165
166	ewv	1.10	txt += """
167	ewv	1.6	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
168	ewv	1.12	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
169	ewv	1.6	if [ -e ../default.tgz ] ;then
170			echo "Found ISB in parent directory (Local Condor)"
171			cp ../default.tgz $_CONDOR_SCRATCH_DIR
172			fi
173			cd $_CONDOR_SCRATCH_DIR
174			fi
175			"""
176	ewv	1.5
177	ewv	1.10	return txt