CRAB/python/SchedulerCondor.py

__revision__ = "$Id: SchedulerCondor.py,v 1.8 2008/06/11 10:03:07 spiga Exp $"
__version__ = "$Revision: 1.8 $"

from Scheduler import Scheduler
from SchedulerLocal import SchedulerLocal
from crab_exceptions import *
from crab_util import *
from crab_logger import Logger
import common

import os

#  Naming convention:
#  methods starting with 'ws' are responsible to provide
#  corresponding part of the job script ('ws' stands for 'write script').

class SchedulerCondor(SchedulerLocal) :

  def __init__(self):
    Scheduler.__init__(self,"CONDOR")
    return


  def configure(self, cfg_params):
    SchedulerLocal.configure(self, cfg_params)
    self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')

    try:
      tmp =  cfg_params['CMSSW.datasetpath']
      if string.lower(tmp)=='none':
        self.datasetPath = None
        self.selectNoInput = 1
      else:
        self.datasetPath = tmp
        self.selectNoInput = 0
    except KeyError:
      msg = "Error: datasetpath not defined "
      raise CrabException(msg)

    return


  def sched_parameter(self,i,task):
    """
    Return scheduler-specific parameters
    """
    index = int(common._db.nJobs()) - 1
    sched_param= ''

    for i in range(index):
      pass

    return sched_param


  def realSchedParams(self,cfg_params):
    """
    Return dictionary with specific parameters, to use
    with real scheduler
    """

    tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
    params = {'tmpDir':tmpDir}
    return  params


  def listMatch(self, seList, full, onlyOSG=True):
    """
    Check the compatibility of available resources
    """

    # May have problems with onlyOSG being false, probably due to lengths of lists and command line.
    # Either re-write osg_bdii.py with a proper ldap library or break the queries apart

    if self.selectNoInput:
      return [True]
    else:
      return SchedulerLocal.listMatch(self, seList, full)

  def decodeLogInfo(self, file):
    """
    Parse logging info file and return main info
    """
    import CondorGLoggingInfo
    loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
    reason = loggingInfo.decodeReason(file)
    return reason


  def wsExitFunc(self):
    """
    """
    txt = '\n'
    txt += '#\n'
    txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
    txt += '#\n\n'

    txt += 'func_exit() { \n'
    txt += self.wsExitFunc_common()

    txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
    txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
    txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'

    txt += '    exit $job_exit_code\n'
    txt += '}\n'

    return txt

  def wsInitialEnvironment(self):
    """
    Returns part of a job script which does scheduler-specific work.
    """

    txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
    txt += 'echo "Beginning environment"\n'
    txt += 'printenv | sort\n'

    txt += 'middleware='+self.name()+' \n'
    txt += """
if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
    ORIG_WD=`pwd`
    echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
    if [ -e ../default.tgz ] ;then
      echo "Found ISB in parent directory (Local Condor)"
      cp ../default.tgz $_CONDOR_SCRATCH_DIR
    fi
    cd $_CONDOR_SCRATCH_DIR
fi
"""

    return txt
Revision:	1.9
Committed:	Tue Jun 17 21:54:14 2008 UTC (16 years, 10 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_3_2_pre1, CRAB_2_4_0_test
Branch point for:	AnaDataSet
Changes since 1.8:	+4 -7 lines
Log Message:	Restore super-class listMatch for case with dataset
#	User	Rev	Content
1	ewv	1.9	__revision__ = "$Id: SchedulerCondor.py,v 1.8 2008/06/11 10:03:07 spiga Exp $"
2			__version__ = "$Revision: 1.8 $"
3	ewv	1.1
4			from Scheduler import Scheduler
5			from SchedulerLocal import SchedulerLocal
6			from crab_exceptions import *
7			from crab_util import *
8			from crab_logger import Logger
9			import common
10
11			import os
12
13			# Naming convention:
14			# methods starting with 'ws' are responsible to provide
15			# corresponding part of the job script ('ws' stands for 'write script').
16
17			class SchedulerCondor(SchedulerLocal) :
18
19			def __init__(self):
20			Scheduler.__init__(self,"CONDOR")
21			return
22
23	ewv	1.3
24	ewv	1.1	def configure(self, cfg_params):
25			SchedulerLocal.configure(self, cfg_params)
26			self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	ewv	1.7
28			try:
29			tmp = cfg_params['CMSSW.datasetpath']
30			if string.lower(tmp)=='none':
31			self.datasetPath = None
32			self.selectNoInput = 1
33			else:
34			self.datasetPath = tmp
35			self.selectNoInput = 0
36			except KeyError:
37			msg = "Error: datasetpath not defined "
38			raise CrabException(msg)
39
40	ewv	1.1	return
41
42	ewv	1.3
43	ewv	1.1	def sched_parameter(self,i,task):
44			"""
45			Return scheduler-specific parameters
46			"""
47			index = int(common._db.nJobs()) - 1
48			sched_param= ''
49
50			for i in range(index):
51			pass
52
53			return sched_param
54
55	ewv	1.3
56	ewv	1.2	def realSchedParams(self,cfg_params):
57			"""
58			Return dictionary with specific parameters, to use
59			with real scheduler
60			"""
61
62			tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
63			params = {'tmpDir':tmpDir}
64			return params
65
66	ewv	1.3
67	ewv	1.7	def listMatch(self, seList, full, onlyOSG=True):
68			"""
69			Check the compatibility of available resources
70			"""
71
72			# May have problems with onlyOSG being false, probably due to lengths of lists and command line.
73			# Either re-write osg_bdii.py with a proper ldap library or break the queries apart
74
75			if self.selectNoInput:
76			return [True]
77	ewv	1.9	else:
78			return SchedulerLocal.listMatch(self, seList, full)
79	ewv	1.7
80	ewv	1.4	def decodeLogInfo(self, file):
81			"""
82			Parse logging info file and return main info
83			"""
84			import CondorGLoggingInfo
85			loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
86			reason = loggingInfo.decodeReason(file)
87			return reason
88	ewv	1.3
89
90			def wsExitFunc(self):
91			"""
92			"""
93			txt = '\n'
94			txt += '#\n'
95			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
96			txt += '#\n\n'
97
98			txt += 'func_exit() { \n'
99	spiga	1.8	txt += self.wsExitFunc_common()
100
101	ewv	1.3	txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
102			txt += ' cp ${out_files}.tgz $ORIG_WD/\n'
103			txt += ' cp crab_fjr_$NJob.xml $ORIG_WD/\n'
104
105			txt += ' exit $job_exit_code\n'
106			txt += '}\n'
107
108			return txt
109	ewv	1.5
110			def wsInitialEnvironment(self):
111			"""
112			Returns part of a job script which does scheduler-specific work.
113			"""
114
115			txt = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
116			txt += 'echo "Beginning environment"\n'
117			txt += 'printenv \| sort\n'
118
119			txt += 'middleware='+self.name()+' \n'
120			txt += """
121	ewv	1.6	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
122			ORIG_WD=`pwd`
123			echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
124			if [ -e ../default.tgz ] ;then
125			echo "Found ISB in parent directory (Local Condor)"
126			cp ../default.tgz $_CONDOR_SCRATCH_DIR
127			fi
128			cd $_CONDOR_SCRATCH_DIR
129			fi
130			"""
131	ewv	1.5
132			return txt