CRAB/python/SchedulerCondor.py

__revision__ = "$Id: SchedulerCondor.py,v 1.6 2008/05/29 19:20:40 ewv Exp $"
__version__ = "$Revision: 1.6 $"

from Scheduler import Scheduler
from SchedulerLocal import SchedulerLocal
from crab_exceptions import *
from crab_util import *
from crab_logger import Logger
import common

import os

#  Naming convention:
#  methods starting with 'ws' are responsible to provide
#  corresponding part of the job script ('ws' stands for 'write script').

class SchedulerCondor(SchedulerLocal) :

  def __init__(self):
    Scheduler.__init__(self,"CONDOR")
    return


  def configure(self, cfg_params):
    SchedulerLocal.configure(self, cfg_params)
    self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')

    try:
      tmp =  cfg_params['CMSSW.datasetpath']
      if string.lower(tmp)=='none':
        self.datasetPath = None
        self.selectNoInput = 1
      else:
        self.datasetPath = tmp
        self.selectNoInput = 0
    except KeyError:
      msg = "Error: datasetpath not defined "
      raise CrabException(msg)

    return


  def sched_parameter(self,i,task):
    """
    Return scheduler-specific parameters
    """
    index = int(common._db.nJobs()) - 1
    sched_param= ''

    for i in range(index):
      pass

    return sched_param


  def realSchedParams(self,cfg_params):
    """
    Return dictionary with specific parameters, to use
    with real scheduler
    """

    tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
    params = {'tmpDir':tmpDir}
    return  params


  def listMatch(self, seList, full, onlyOSG=True):
    """
    Check the compatibility of available resources
    """

    # May have problems with onlyOSG being false, probably due to lengths of lists and command line.
    # Either re-write osg_bdii.py with a proper ldap library or break the queries apart

    #scram = Scram.Scram(None)
    #versionCMSSW = scram.getSWVersion()
    #arch = scram.getArch()

    if self.selectNoInput:
      return [True]


  def decodeLogInfo(self, file):
    """
    Parse logging info file and return main info
    """
    import CondorGLoggingInfo
    loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
    reason = loggingInfo.decodeReason(file)
    return reason


  def wsExitFunc(self):
    """
    """
    txt = '\n'
    txt += '#\n'
    txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
    txt += '#\n\n'

    txt += 'func_exit() { \n'
    txt += '    if [ $PYTHONPATH ]; then \n'
    txt += '        update_fjr\n'
    txt += '    fi\n'
    txt += '    for file in $filesToCheck ; do\n'
    txt += '        if [ -e $file ]; then\n'
    txt += '            echo "tarring file $file in  $out_files"\n'
    txt += '        else\n'
    txt += '            echo "WARNING: output file $file not found!"\n'
    txt += '        fi\n'
    txt += '    done\n'
    txt += '    final_list=$filesToCheck\n'
    txt += '    echo "JOB_EXIT_STATUS = $job_exit_code"\n'
    txt += '    echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
    txt += '    dumpStatus $RUNTIME_AREA/$repo\n'
    txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
    txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
    txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'

    txt += '    exit $job_exit_code\n'
    txt += '}\n'

    return txt

  def wsInitialEnvironment(self):
    """
    Returns part of a job script which does scheduler-specific work.
    """

    txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
    txt += 'echo "Beginning environment"\n'
    txt += 'printenv | sort\n'

    txt += 'middleware='+self.name()+' \n'
    txt += """
if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
    ORIG_WD=`pwd`
    echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
    if [ -e ../default.tgz ] ;then
      echo "Found ISB in parent directory (Local Condor)"
      cp ../default.tgz $_CONDOR_SCRATCH_DIR
    fi
    cd $_CONDOR_SCRATCH_DIR
fi
"""

    return txt
Revision:	1.7
Committed:	Tue Jun 10 17:46:00 2008 UTC (16 years, 10 months ago) by ewv
Content type:	text/x-python
Branch:	MAIN
Changes since 1.6:	+31 -2 lines
Log Message:	Enable Pythia+Condor, fix case in SchedulerLocal
#	User	Rev	Content
1	ewv	1.7	__revision__ = "$Id: SchedulerCondor.py,v 1.6 2008/05/29 19:20:40 ewv Exp $"
2			__version__ = "$Revision: 1.6 $"
3	ewv	1.1
4			from Scheduler import Scheduler
5			from SchedulerLocal import SchedulerLocal
6			from crab_exceptions import *
7			from crab_util import *
8			from crab_logger import Logger
9			import common
10
11			import os
12
13			# Naming convention:
14			# methods starting with 'ws' are responsible to provide
15			# corresponding part of the job script ('ws' stands for 'write script').
16
17			class SchedulerCondor(SchedulerLocal) :
18
19			def __init__(self):
20			Scheduler.__init__(self,"CONDOR")
21			return
22
23	ewv	1.3
24	ewv	1.1	def configure(self, cfg_params):
25			SchedulerLocal.configure(self, cfg_params)
26			self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	ewv	1.7
28			try:
29			tmp = cfg_params['CMSSW.datasetpath']
30			if string.lower(tmp)=='none':
31			self.datasetPath = None
32			self.selectNoInput = 1
33			else:
34			self.datasetPath = tmp
35			self.selectNoInput = 0
36			except KeyError:
37			msg = "Error: datasetpath not defined "
38			raise CrabException(msg)
39
40	ewv	1.1	return
41
42	ewv	1.3
43	ewv	1.1	def sched_parameter(self,i,task):
44			"""
45			Return scheduler-specific parameters
46			"""
47			index = int(common._db.nJobs()) - 1
48			sched_param= ''
49
50			for i in range(index):
51			pass
52
53			return sched_param
54
55	ewv	1.3
56	ewv	1.2	def realSchedParams(self,cfg_params):
57			"""
58			Return dictionary with specific parameters, to use
59			with real scheduler
60			"""
61
62			tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
63			params = {'tmpDir':tmpDir}
64			return params
65
66	ewv	1.3
67	ewv	1.7	def listMatch(self, seList, full, onlyOSG=True):
68			"""
69			Check the compatibility of available resources
70			"""
71
72			# May have problems with onlyOSG being false, probably due to lengths of lists and command line.
73			# Either re-write osg_bdii.py with a proper ldap library or break the queries apart
74
75			#scram = Scram.Scram(None)
76			#versionCMSSW = scram.getSWVersion()
77			#arch = scram.getArch()
78
79			if self.selectNoInput:
80			return [True]
81
82
83	ewv	1.4	def decodeLogInfo(self, file):
84			"""
85			Parse logging info file and return main info
86			"""
87			import CondorGLoggingInfo
88			loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
89			reason = loggingInfo.decodeReason(file)
90			return reason
91	ewv	1.3
92
93			def wsExitFunc(self):
94			"""
95			"""
96			txt = '\n'
97			txt += '#\n'
98			txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
99			txt += '#\n\n'
100
101			txt += 'func_exit() { \n'
102			txt += ' if [ $PYTHONPATH ]; then \n'
103			txt += ' update_fjr\n'
104			txt += ' fi\n'
105			txt += ' for file in $filesToCheck ; do\n'
106			txt += ' if [ -e $file ]; then\n'
107			txt += ' echo "tarring file $file in $out_files"\n'
108			txt += ' else\n'
109			txt += ' echo "WARNING: output file $file not found!"\n'
110			txt += ' fi\n'
111			txt += ' done\n'
112			txt += ' final_list=$filesToCheck\n'
113			txt += ' echo "JOB_EXIT_STATUS = $job_exit_code"\n'
114			txt += ' echo "JobExitCode=$job_exit_code" >> $RUNTIME_AREA/$repo\n'
115			txt += ' dumpStatus $RUNTIME_AREA/$repo\n'
116			txt += ' tar zcvf ${out_files}.tgz ${final_list}\n'
117			txt += ' cp ${out_files}.tgz $ORIG_WD/\n'
118			txt += ' cp crab_fjr_$NJob.xml $ORIG_WD/\n'
119
120			txt += ' exit $job_exit_code\n'
121			txt += '}\n'
122
123			return txt
124	ewv	1.5
125			def wsInitialEnvironment(self):
126			"""
127			Returns part of a job script which does scheduler-specific work.
128			"""
129
130			txt = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
131			txt += 'echo "Beginning environment"\n'
132			txt += 'printenv \| sort\n'
133
134			txt += 'middleware='+self.name()+' \n'
135			txt += """
136	ewv	1.6	if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
137			ORIG_WD=`pwd`
138			echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
139			if [ -e ../default.tgz ] ;then
140			echo "Found ISB in parent directory (Local Condor)"
141			cp ../default.tgz $_CONDOR_SCRATCH_DIR
142			fi
143			cd $_CONDOR_SCRATCH_DIR
144			fi
145			"""
146	ewv	1.5
147			return txt