| 1 |
"""
|
| 2 |
Scheduler implementation used by CondorG and Glidein (Common parts)
|
| 3 |
This class was originally SchedulerCondor_g.
|
| 4 |
For a history of this code, see that file.
|
| 5 |
"""
|
| 6 |
|
| 7 |
from SchedulerGrid import SchedulerGrid
|
| 8 |
from crab_exceptions import CrabException
|
| 9 |
from crab_util import runCommand
|
| 10 |
|
| 11 |
from ProdCommon.BDII.Bdii import getJobManagerList, listAllCEs
|
| 12 |
from WMCore.SiteScreening.BlackWhiteListParser import CEBlackWhiteListParser
|
| 13 |
|
| 14 |
import Scram
|
| 15 |
import CondorGLoggingInfo
|
| 16 |
import common
|
| 17 |
|
| 18 |
#import popen2
|
| 19 |
import os
|
| 20 |
|
| 21 |
# FUTURE: for python 2.4 & 2.6
|
| 22 |
try:
|
| 23 |
from hashlib import sha1
|
| 24 |
except:
|
| 25 |
from sha import sha as sha1
|
| 26 |
|
| 27 |
__revision__ = "$Id: SchedulerCondorCommon.py,v 1.44 2010/08/19 15:03:57 ewv Exp $"
|
| 28 |
__version__ = "$Revision: 1.44 $"
|
| 29 |
|
| 30 |
class SchedulerCondorCommon(SchedulerGrid):
|
| 31 |
"""
|
| 32 |
Scheduler implementation used by CondorG and Glidein (Common parts)
|
| 33 |
"""
|
| 34 |
|
| 35 |
def __init__(self, name):
|
| 36 |
SchedulerGrid.__init__(self, name)
|
| 37 |
self.environment_unique_identifier = None
|
| 38 |
self.msgPre = '[Condor-G Scheduler]: '
|
| 39 |
|
| 40 |
return
|
| 41 |
|
| 42 |
|
| 43 |
def configure(self, cfgParams):
|
| 44 |
"""
|
| 45 |
Configure the scheduler with the config settings from the user
|
| 46 |
"""
|
| 47 |
# FIXME: Get rid of try/except and use get() instead
|
| 48 |
|
| 49 |
if not os.environ.has_key('EDG_WL_LOCATION'):
|
| 50 |
# This is an ugly hack needed for SchedulerGrid.configure() to
|
| 51 |
# work!
|
| 52 |
os.environ['EDG_WL_LOCATION'] = ''
|
| 53 |
|
| 54 |
if not os.environ.has_key('X509_USER_PROXY'):
|
| 55 |
# Set X509_USER_PROXY to the default location. We'll do this
|
| 56 |
# because in functions called by Scheduler.checkProxy()
|
| 57 |
# voms-proxy-info will be called with '-file $X509_USER_PROXY',
|
| 58 |
# so if X509_USER_PROXY isn't set, it won't work.
|
| 59 |
os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())
|
| 60 |
|
| 61 |
SchedulerGrid.configure(self, cfgParams)
|
| 62 |
if cfgParams.get('CRAB.server_name',None) or cfgParams.get('CRAB.use_server',None):
|
| 63 |
pass
|
| 64 |
else:
|
| 65 |
self.checkCondorSetup()
|
| 66 |
|
| 67 |
# init BlackWhiteListParser
|
| 68 |
ceWhiteList = cfgParams.get('GRID.ce_white_list',[])
|
| 69 |
ceBlackList = cfgParams.get('GRID.ce_black_list',[])
|
| 70 |
self.ceBlackWhiteListParser = \
|
| 71 |
CEBlackWhiteListParser(ceWhiteList, ceBlackList, common.logger())
|
| 72 |
|
| 73 |
try:
|
| 74 |
self.GLOBUS_RSL = cfgParams['CONDORG.globus_rsl']
|
| 75 |
except KeyError:
|
| 76 |
self.GLOBUS_RSL = ''
|
| 77 |
|
| 78 |
# Provide an override for the batchsystem that condor_g specifies
|
| 79 |
# as a grid resource. This is to handle the case where the site
|
| 80 |
# supports several batchsystem but bdii only allows a site
|
| 81 |
# to public one.
|
| 82 |
try:
|
| 83 |
self.batchsystem = cfgParams['CONDORG.batchsystem']
|
| 84 |
msg = self.msgPre + 'batchsystem overide specified in your crab.cfg'
|
| 85 |
common.logger.debug(msg)
|
| 86 |
except KeyError:
|
| 87 |
self.batchsystem = ''
|
| 88 |
|
| 89 |
self.datasetPath = ''
|
| 90 |
|
| 91 |
try:
|
| 92 |
tmp = cfgParams['CMSSW.datasetpath']
|
| 93 |
if tmp.lower() == 'none':
|
| 94 |
self.datasetPath = None
|
| 95 |
self.selectNoInput = 1
|
| 96 |
else:
|
| 97 |
self.datasetPath = tmp
|
| 98 |
self.selectNoInput = 0
|
| 99 |
except KeyError:
|
| 100 |
msg = "Error: datasetpath not defined "
|
| 101 |
raise CrabException(msg)
|
| 102 |
|
| 103 |
return
|
| 104 |
|
| 105 |
def envUniqueID(self):
|
| 106 |
taskHash = sha1(common._db.queryTask('name')).hexdigest()
|
| 107 |
id = 'https://' + self.name() + '/' + taskHash + '/${NJob}'
|
| 108 |
msg = 'JobID for ML monitoring is created for OSG scheduler: %s'%id
|
| 109 |
common.logger.debug( msg)
|
| 110 |
return id
|
| 111 |
|
| 112 |
def realSchedParams(self, cfgParams):
|
| 113 |
"""
|
| 114 |
Return dictionary with specific parameters, to use
|
| 115 |
with real scheduler
|
| 116 |
"""
|
| 117 |
|
| 118 |
tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
|
| 119 |
jobDir = common.work_space.jobDir()
|
| 120 |
params = {'tmpDir':tmpDir,
|
| 121 |
'jobDir':jobDir}
|
| 122 |
return params
|
| 123 |
|
| 124 |
|
| 125 |
def sched_parameter(self, i, task):
|
| 126 |
"""
|
| 127 |
Returns scheduler-specific parameters
|
| 128 |
"""
|
| 129 |
jobParams = ''
|
| 130 |
|
| 131 |
return jobParams
|
| 132 |
|
| 133 |
|
| 134 |
def decodeLogInfo(self, theFile):
|
| 135 |
"""
|
| 136 |
Parse logging info file and return main info
|
| 137 |
"""
|
| 138 |
|
| 139 |
loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
|
| 140 |
reason = loggingInfo.decodeReason(theFile)
|
| 141 |
return reason
|
| 142 |
|
| 143 |
|
| 144 |
def listMatch(self, seList, full, onlyOSG=True):
|
| 145 |
"""
|
| 146 |
Check the compatibility of available resources
|
| 147 |
"""
|
| 148 |
|
| 149 |
scram = Scram.Scram(None)
|
| 150 |
versionCMSSW = scram.getSWVersion()
|
| 151 |
arch = scram.getArch()
|
| 152 |
|
| 153 |
if self.selectNoInput:
|
| 154 |
availCEs = listAllCEs(versionCMSSW, arch, onlyOSG=onlyOSG)
|
| 155 |
else:
|
| 156 |
seDest = self.blackWhiteListParser.cleanForBlackWhiteList(seList, "list")
|
| 157 |
availCEs = getJobManagerList(seDest, versionCMSSW,
|
| 158 |
arch, onlyOSG=onlyOSG)
|
| 159 |
|
| 160 |
uniqCEs = []
|
| 161 |
for ce in availCEs:
|
| 162 |
if ce not in uniqCEs:
|
| 163 |
uniqCEs.append(ce)
|
| 164 |
|
| 165 |
ceDest = self.ceBlackWhiteListParser.cleanForBlackWhiteList(uniqCEs, "list")
|
| 166 |
|
| 167 |
return ceDest
|
| 168 |
|
| 169 |
def ce_list(self):
|
| 170 |
"""
|
| 171 |
Returns string with requirement CE related, dummy for now
|
| 172 |
"""
|
| 173 |
|
| 174 |
req = ''
|
| 175 |
|
| 176 |
return req, self.EDG_ce_white_list, self.EDG_ce_black_list
|
| 177 |
|
| 178 |
def seListToCElist(self, seList, onlyOSG=True):
|
| 179 |
"""
|
| 180 |
Convert the list of SEs into a list of CEs
|
| 181 |
"""
|
| 182 |
|
| 183 |
ceDest = self.listMatch(seList, onlyOSG)
|
| 184 |
|
| 185 |
if (not ceDest):
|
| 186 |
msg = 'No sites found hosting the data or ' \
|
| 187 |
+ 'all sites blocked by CE/SE white/blacklisting'
|
| 188 |
print msg
|
| 189 |
raise CrabException(msg)
|
| 190 |
|
| 191 |
return ceDest
|
| 192 |
|
| 193 |
|
| 194 |
def wsExitFunc(self):
|
| 195 |
"""
|
| 196 |
Returns the part of the job script which runs prior to exit
|
| 197 |
"""
|
| 198 |
|
| 199 |
txt = '\n'
|
| 200 |
|
| 201 |
txt += '#\n'
|
| 202 |
txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
|
| 203 |
txt += '#\n\n'
|
| 204 |
|
| 205 |
txt += 'func_exit() { \n'
|
| 206 |
txt += self.wsExitFunc_common()
|
| 207 |
txt += ' exit $job_exit_code\n'
|
| 208 |
txt += '}\n'
|
| 209 |
return txt
|
| 210 |
|
| 211 |
|
| 212 |
def checkCondorSetup(self):
|
| 213 |
"""
|
| 214 |
Check the local machine for a properly set up and running condor install
|
| 215 |
"""
|
| 216 |
|
| 217 |
# check for locally running schedd
|
| 218 |
cmd = 'ps xau | grep -i condor_schedd | grep -v grep'
|
| 219 |
cmdOut = runCommand(cmd)
|
| 220 |
if cmdOut == None:
|
| 221 |
msg = self.msgPre + 'condor_schedd is not running on this machine.\n'
|
| 222 |
msg += self.msgPre + 'Please use a machine with condor installed and running\n'
|
| 223 |
msg += self.msgPre + 'condor_schedd or change the Scheduler in your crab.cfg.'
|
| 224 |
common.logger.debug( msg)
|
| 225 |
raise CrabException(msg)
|
| 226 |
|
| 227 |
self.checkExecutableInPath('condor_q')
|
| 228 |
self.checkExecutableInPath('condor_submit')
|
| 229 |
self.checkExecutableInPath('condor_version')
|
| 230 |
|
| 231 |
# get version number
|
| 232 |
cmd = 'condor_version'
|
| 233 |
cmdOut = runCommand(cmd)
|
| 234 |
if cmdOut != None :
|
| 235 |
pass
|
| 236 |
#tmp = cmdOut.find('CondorVersion') + 15
|
| 237 |
#version = cmdOut[tmp:tmp+6].split('.')
|
| 238 |
#version_master = int(version[0])
|
| 239 |
#version_major = int(version[1])
|
| 240 |
#version_minor = int(version[2])
|
| 241 |
else :
|
| 242 |
msg = self.msgPre + 'condor_version was not able to determine the installed condor version.\n'
|
| 243 |
msg += self.msgPre + 'Please use a machine with a properly installed condor\n'
|
| 244 |
msg += self.msgPre + 'or change the Scheduler in your crab.cfg.'
|
| 245 |
common.logger.debug( msg)
|
| 246 |
raise CrabException(msg)
|
| 247 |
|
| 248 |
self.checkExecutableInPath('condor_config_val')
|
| 249 |
self.checkCondorVariablePointsToFile('GRIDMANAGER')
|
| 250 |
self.checkCondorVariablePointsToFile('GT2_GAHP', alternateName='GAHP')
|
| 251 |
self.checkCondorVariablePointsToFile('GRID_MONITOR')
|
| 252 |
self.checkCondorVariableIsTrue('ENABLE_GRID_MONITOR')
|
| 253 |
|
| 254 |
return
|
| 255 |
|
| 256 |
def checkExecutableInPath(self, name):
|
| 257 |
"""
|
| 258 |
check if executable is in PATH
|
| 259 |
"""
|
| 260 |
|
| 261 |
cmd = 'which '+name
|
| 262 |
cmdOut = runCommand(cmd)
|
| 263 |
if cmdOut == None:
|
| 264 |
msg = self.msgPre + name + ' is not in the $PATH on this machine.\n'
|
| 265 |
msg += self.msgPre + 'Please use a machine with a properly installed condor\n'
|
| 266 |
msg += self.msgPre + 'or change the Scheduler in your crab.cfg.'
|
| 267 |
common.logger.debug( msg)
|
| 268 |
raise CrabException(msg)
|
| 269 |
|
| 270 |
def checkCondorVariablePointsToFile(self, name, alternateName=None):
|
| 271 |
"""
|
| 272 |
check for condor variable
|
| 273 |
"""
|
| 274 |
|
| 275 |
cmd = 'condor_config_val ' + name
|
| 276 |
cmdOut = runCommand(cmd)
|
| 277 |
if alternateName and not cmdOut:
|
| 278 |
cmd = 'condor_config_val ' + alternateName
|
| 279 |
cmdOut = runCommand(cmd)
|
| 280 |
if cmdOut:
|
| 281 |
cmdOut = cmdOut.strip()
|
| 282 |
if not cmdOut or not os.path.isfile(cmdOut) :
|
| 283 |
msg = self.msgPre + 'the variable ' + name
|
| 284 |
msg += ' is not properly set for the condor installation.\n'
|
| 285 |
msg += self.msgPre + 'Please ask the administrator of the local condor '
|
| 286 |
msg += 'installation to set the variable ' + name + ' properly, '
|
| 287 |
msg += 'use another machine with a properly installed condor\n'
|
| 288 |
msg += 'or change the Scheduler in your crab.cfg.'
|
| 289 |
common.logger.debug( msg)
|
| 290 |
raise CrabException(msg)
|
| 291 |
|
| 292 |
def checkCondorVariableIsTrue(self, name):
|
| 293 |
"""
|
| 294 |
check for condor variable
|
| 295 |
"""
|
| 296 |
|
| 297 |
cmd = 'condor_config_val '+name
|
| 298 |
cmdOut = runCommand(cmd)
|
| 299 |
if cmdOut == 'TRUE' :
|
| 300 |
msg = self.msgPre + 'the variable ' + name
|
| 301 |
msg += ' is not set to true for the condor installation.\n'
|
| 302 |
msg += self.msgPre + 'Please ask the administrator of the local condor installation '
|
| 303 |
msg += 'to set the variable ' + name + ' to true, '
|
| 304 |
msg += 'use another machine with a properly installed condor or '
|
| 305 |
msg += 'change the Scheduler in your crab.cfg.'
|
| 306 |
common.logger.debug( msg)
|
| 307 |
raise CrabException(msg)
|
| 308 |
|
| 309 |
def queryCondorVariable(self, name, default):
|
| 310 |
"""
|
| 311 |
check for condor variable
|
| 312 |
"""
|
| 313 |
|
| 314 |
cmd = 'condor_config_val '+name
|
| 315 |
cmdOut = runCommand(cmd)
|
| 316 |
if cmdOut:
|
| 317 |
# out = popen2.Popen3(cmd, 1)
|
| 318 |
# exitCode = out.wait()
|
| 319 |
# cmdOut = out.fromchild.readline().strip()
|
| 320 |
# if exitCode != 0 :
|
| 321 |
cmdOut = str(default)
|
| 322 |
|
| 323 |
return cmdOut
|
| 324 |
|
