1 |
ewv |
1.79 |
from SchedulerGrid import SchedulerGrid
|
2 |
gutsche |
1.1 |
from JobList import JobList
|
3 |
|
|
from crab_logger import Logger
|
4 |
|
|
from crab_exceptions import *
|
5 |
|
|
from crab_util import *
|
6 |
gutsche |
1.42 |
from osg_bdii import *
|
7 |
gutsche |
1.1 |
import time
|
8 |
|
|
import common
|
9 |
gutsche |
1.25 |
import popen2
|
10 |
slacapra |
1.39 |
import os
|
11 |
gutsche |
1.49 |
from BlackWhiteListParser import BlackWhiteListParser
|
12 |
spiga |
1.92 |
import CondorGLoggingInfo
|
13 |
gutsche |
1.1 |
|
14 |
ewv |
1.83 |
import pdb # Use while debugging
|
15 |
|
|
|
16 |
ewv |
1.95 |
__revision__ = "$Id: SchedulerCondor_g.py,v 1.94 2008/04/15 17:50:42 ewv Exp $"
|
17 |
|
|
__version__ = "$Revision: 1.94 $"
|
18 |
ewv |
1.83 |
|
19 |
ewv |
1.79 |
class SchedulerCondor_g(SchedulerGrid):
|
20 |
gutsche |
1.1 |
def __init__(self):
|
21 |
ewv |
1.79 |
SchedulerGrid.__init__(self,"CONDOR_G")
|
22 |
gutsche |
1.1 |
|
23 |
|
|
# check for locally running condor scheduler
|
24 |
|
|
cmd = 'ps xau | grep -i condor_schedd | grep -v grep'
|
25 |
|
|
cmd_out = runCommand(cmd)
|
26 |
|
|
if cmd_out == None:
|
27 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: condor_schedd is not running on this machine.\n'
|
28 |
|
|
msg += '[Condor-G Scheduler]: Please use another machine with installed condor and running condor_schedd or change the Scheduler in your crab.cfg.'
|
29 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
30 |
gutsche |
1.20 |
raise CrabException(msg)
|
31 |
gutsche |
1.1 |
|
32 |
|
|
self.checkExecutableInPath('condor_q')
|
33 |
|
|
self.checkExecutableInPath('condor_submit')
|
34 |
|
|
self.checkExecutableInPath('condor_version')
|
35 |
|
|
|
36 |
|
|
# get version number
|
37 |
|
|
cmd = 'condor_version'
|
38 |
|
|
cmd_out = runCommand(cmd)
|
39 |
|
|
if cmd != None :
|
40 |
|
|
tmp = cmd_out.find('CondorVersion') + 15
|
41 |
|
|
version = cmd_out[tmp:tmp+6].split('.')
|
42 |
|
|
version_master = int(version[0])
|
43 |
|
|
version_major = int(version[1])
|
44 |
|
|
version_minor = int(version[2])
|
45 |
|
|
else :
|
46 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: condor_version was not able to determine the installed condor version.\n'
|
47 |
|
|
msg += '[Condor-G Scheduler]: Please use another machine with properly installed condor or change the Scheduler in your crab.cfg.'
|
48 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
49 |
gutsche |
1.20 |
raise CrabException(msg)
|
50 |
gutsche |
1.1 |
|
51 |
|
|
self.checkExecutableInPath('condor_config_val')
|
52 |
|
|
|
53 |
|
|
self.checkCondorVariablePointsToFile('GRIDMANAGER')
|
54 |
|
|
|
55 |
ewv |
1.76 |
self.checkCondorVariablePointsToFile('GT2_GAHP',alternate_name='GAHP')
|
56 |
gutsche |
1.1 |
|
57 |
|
|
self.checkCondorVariablePointsToFile('GRID_MONITOR')
|
58 |
|
|
|
59 |
|
|
self.checkCondorVariableIsTrue('ENABLE_GRID_MONITOR')
|
60 |
|
|
|
61 |
gutsche |
1.25 |
max_submit = self.queryCondorVariable('GRIDMANAGER_MAX_SUBMITTED_JOBS_PER_RESOURCE',100).strip()
|
62 |
|
|
max_pending = self.queryCondorVariable('GRIDMANAGER_MAX_PENDING_SUBMITS_PER_RESOURCE','Unlimited').strip()
|
63 |
gutsche |
1.1 |
|
64 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]\n'
|
65 |
|
|
msg += 'Maximal number of jobs submitted to the grid : GRIDMANAGER_MAX_SUBMITTED_JOBS_PER_RESOURCE = '+max_submit+'\n'
|
66 |
|
|
msg += 'Maximal number of parallel submits to the grid : GRIDMANAGER_MAX_PENDING_SUBMITS_PER_RESOURCE = '+max_pending+'\n'
|
67 |
|
|
msg += 'Ask the administrator of your local condor installation to increase these variables to enable more jobs to be executed on the grid in parallel.\n'
|
68 |
|
|
common.logger.debug(2,msg)
|
69 |
mkirn |
1.15 |
|
70 |
gutsche |
1.29 |
# create hash
|
71 |
|
|
self.hash = makeCksum(common.work_space.cfgFileName())
|
72 |
|
|
|
73 |
gutsche |
1.1 |
return
|
74 |
|
|
|
75 |
gutsche |
1.38 |
def getCEfromSE(self, seSite):
|
76 |
ewv |
1.84 |
# returns the ce including jobmanager
|
77 |
|
|
ces = jm_from_se_bdii(seSite)
|
78 |
gutsche |
1.42 |
|
79 |
ewv |
1.84 |
# mapping ce_hostname to full ce name including jobmanager
|
80 |
|
|
ce_hostnames = {}
|
81 |
|
|
for ce in ces :
|
82 |
|
|
ce_hostnames[ce.split(':')[0].strip()] = ce
|
83 |
|
|
|
84 |
|
|
oneSite=''
|
85 |
|
|
if len(ce_hostnames.keys()) == 1:
|
86 |
|
|
oneSite=ce_hostnames[ce_hostnames.keys()[0]]
|
87 |
|
|
elif len(ce_hostnames.keys()) > 1:
|
88 |
|
|
if self.EDG_ce_white_list and len(self.EDG_ce_white_list) == 1 and self.EDG_ce_white_list[0] in ce_hostnames.keys():
|
89 |
|
|
oneSite = self.EDG_ce_white_list[0]
|
90 |
gutsche |
1.38 |
else :
|
91 |
ewv |
1.84 |
msg = '[Condor-G Scheduler]: More than one Compute Element (CE) is available for job submission.\n'
|
92 |
|
|
msg += '[Condor-G Scheduler]: Please select one of the following CEs:\n'
|
93 |
|
|
msg += '[Condor-G Scheduler]:'
|
94 |
|
|
for host in ce_hostnames.keys() :
|
95 |
|
|
msg += ' ' + host
|
96 |
|
|
msg += '\n'
|
97 |
|
|
msg += '[Condor-G Scheduler]: and enter this CE in the ce_white_list variable of the [EDG] section in your crab.cfg.\n'
|
98 |
|
|
common.logger.debug(2,msg)
|
99 |
|
|
raise CrabException(msg)
|
100 |
|
|
else :
|
101 |
|
|
raise CrabException('[Condor-G Scheduler]: CE hostname(s) for SE '+seSite+' could not be determined from BDII.')
|
102 |
gutsche |
1.40 |
|
103 |
ewv |
1.84 |
return oneSite
|
104 |
gutsche |
1.40 |
|
105 |
gutsche |
1.1 |
def checkExecutableInPath(self, name):
|
106 |
|
|
# check if executable is in PATH
|
107 |
|
|
cmd = 'which '+name
|
108 |
|
|
cmd_out = runCommand(cmd)
|
109 |
|
|
if cmd_out == None:
|
110 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: '+name+' is not in the $PATH on this machine.\n'
|
111 |
|
|
msg += '[Condor-G Scheduler]: Please use another machine with installed condor or change the Scheduler in your crab.cfg.'
|
112 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
113 |
gutsche |
1.20 |
raise CrabException(msg)
|
114 |
gutsche |
1.1 |
|
115 |
ewv |
1.76 |
def checkCondorVariablePointsToFile(self, name, alternate_name=None):
|
116 |
gutsche |
1.1 |
## check for condor variable
|
117 |
|
|
cmd = 'condor_config_val '+name
|
118 |
|
|
cmd_out = runCommand(cmd)
|
119 |
ewv |
1.76 |
if alternate_name and not cmd_out:
|
120 |
|
|
cmd = 'condor_config_val '+alternate_name
|
121 |
|
|
cmd_out = runCommand(cmd)
|
122 |
|
|
if cmd_out:
|
123 |
|
|
cmd_out = string.strip(cmd_out)
|
124 |
|
|
if not cmd_out or not os.path.isfile(cmd_out) :
|
125 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: the variable '+name+' is not properly set for the condor installation on this machine.\n'
|
126 |
ewv |
1.76 |
msg += '[Condor-G Scheduler]: Please ask the administrator of the local condor installation to set the variable '+name+' properly, ' + \
|
127 |
gutsche |
1.1 |
'use another machine with properly installed condor or change the Scheduler in your crab.cfg.'
|
128 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
129 |
gutsche |
1.20 |
raise CrabException(msg)
|
130 |
gutsche |
1.1 |
|
131 |
|
|
def checkCondorVariableIsTrue(self, name):
|
132 |
|
|
## check for condor variable
|
133 |
|
|
cmd = 'condor_config_val '+name
|
134 |
|
|
cmd_out = runCommand(cmd)
|
135 |
|
|
if cmd_out == 'TRUE' :
|
136 |
gutsche |
1.38 |
msg = '[Condor-G Scheduler]: the variable '+name+' is not set to true for the condor installation on this machine.\n'
|
137 |
gutsche |
1.20 |
msg += '[Condor-G Scheduler]: Please ask the administrator of the local condor installation to set the variable '+name+' to true,',
|
138 |
gutsche |
1.1 |
'use another machine with properly installed condor or change the Scheduler in your crab.cfg.'
|
139 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
140 |
gutsche |
1.20 |
raise CrabException(msg)
|
141 |
gutsche |
1.1 |
|
142 |
gutsche |
1.25 |
def queryCondorVariable(self, name, default):
|
143 |
gutsche |
1.1 |
## check for condor variable
|
144 |
|
|
cmd = 'condor_config_val '+name
|
145 |
gutsche |
1.25 |
out = popen2.Popen3(cmd,1)
|
146 |
|
|
exit_code = out.wait()
|
147 |
|
|
cmd_out = out.fromchild.readline().strip()
|
148 |
|
|
if exit_code != 0 :
|
149 |
|
|
cmd_out = str(default)
|
150 |
|
|
|
151 |
|
|
return cmd_out
|
152 |
gutsche |
1.1 |
|
153 |
|
|
def configure(self, cfg_params):
|
154 |
ewv |
1.79 |
SchedulerGrid.configure(self,cfg_params)
|
155 |
gutsche |
1.1 |
|
156 |
gutsche |
1.49 |
# init BlackWhiteListParser
|
157 |
ewv |
1.88 |
#self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
|
158 |
gutsche |
1.49 |
|
159 |
gutsche |
1.18 |
try:
|
160 |
ewv |
1.79 |
self.GLOBUS_RSL = cfg_params['CONDORG.globus_rsl']
|
161 |
gutsche |
1.30 |
except KeyError:
|
162 |
ewv |
1.79 |
self.GLOBUS_RSL = ''
|
163 |
gutsche |
1.40 |
|
164 |
|
|
# Provide an override for the batchsystem that condor_g specifies as a grid resource.
|
165 |
gutsche |
1.42 |
# this is to handle the case where the site supports several batchsystem but bdii
|
166 |
gutsche |
1.40 |
# only allows a site to public one.
|
167 |
ewv |
1.63 |
try:
|
168 |
ewv |
1.79 |
self.batchsystem = cfg_params['CONDORG.batchsystem']
|
169 |
|
|
msg = '[Condor-G Scheduler]: batchsystem overide specified in your crab.cfg'
|
170 |
|
|
common.logger.debug(2,msg)
|
171 |
|
|
except KeyError:
|
172 |
|
|
self.batchsystem = ''
|
173 |
gutsche |
1.1 |
|
174 |
|
|
# check if one and only one entry is in $CE_WHITELIST
|
175 |
|
|
|
176 |
ewv |
1.79 |
# redo this with SchedulerGrid SE list
|
177 |
|
|
|
178 |
gutsche |
1.1 |
try:
|
179 |
gutsche |
1.32 |
tmpGood = string.split(cfg_params['EDG.se_white_list'],',')
|
180 |
gutsche |
1.1 |
except KeyError:
|
181 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: destination site is not selected properly.\n'
|
182 |
gutsche |
1.32 |
msg += '[Condor-G Scheduler]: Please select your destination site and only your destination site in the SE_white_list variable of the [EDG] section in your crab.cfg.'
|
183 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
184 |
gutsche |
1.20 |
raise CrabException(msg)
|
185 |
gutsche |
1.40 |
|
186 |
gutsche |
1.1 |
if len(tmpGood) != 1 :
|
187 |
gutsche |
1.20 |
msg = '[Condor-G Scheduler]: destination site is not selected properly.\n'
|
188 |
gutsche |
1.32 |
msg += '[Condor-G Scheduler]: Please select your destination site and only your destination site in the SE_white_list variable of the [EDG] section in your crab.cfg.'
|
189 |
gutsche |
1.42 |
common.logger.debug(2,msg)
|
190 |
gutsche |
1.20 |
raise CrabException(msg)
|
191 |
gutsche |
1.1 |
|
192 |
|
|
try:
|
193 |
|
|
self.UseGT4 = cfg_params['USER.use_gt_4'];
|
194 |
|
|
except KeyError:
|
195 |
|
|
self.UseGT4 = 0;
|
196 |
|
|
|
197 |
|
|
# added here because checklistmatch is not used
|
198 |
|
|
self.checkProxy()
|
199 |
ewv |
1.89 |
self.environment_unique_identifier = 'GLOBUS_GRAM_JOB_CONTACT'
|
200 |
gutsche |
1.6 |
|
201 |
gutsche |
1.50 |
self.datasetPath = ''
|
202 |
|
|
try:
|
203 |
|
|
tmp = cfg_params['CMSSW.datasetpath']
|
204 |
|
|
if string.lower(tmp)=='none':
|
205 |
|
|
self.datasetPath = None
|
206 |
|
|
self.selectNoInput = 1
|
207 |
|
|
else:
|
208 |
|
|
self.datasetPath = tmp
|
209 |
|
|
self.selectNoInput = 0
|
210 |
|
|
except KeyError:
|
211 |
ewv |
1.58 |
msg = "Error: datasetpath not defined "
|
212 |
gutsche |
1.50 |
raise CrabException(msg)
|
213 |
ewv |
1.71 |
|
214 |
gutsche |
1.1 |
return
|
215 |
gutsche |
1.40 |
|
216 |
ewv |
1.82 |
def sched_parameter(self,i,task):
|
217 |
ewv |
1.84 |
"""
|
218 |
|
|
Returns file with scheduler-specific parameters
|
219 |
|
|
"""
|
220 |
|
|
lastDest=''
|
221 |
|
|
first = []
|
222 |
|
|
last = []
|
223 |
|
|
|
224 |
ewv |
1.95 |
seDest = self.blackWhiteListParser.cleanForBlackWhiteList(eval(task.jobs[i-1]['dlsDestination']))
|
225 |
ewv |
1.84 |
ceDest = self.getCEfromSE(seDest)
|
226 |
|
|
|
227 |
ewv |
1.87 |
jobParams = "globusscheduler = "+ceDest+":2119/jobmanager-condor; "
|
228 |
|
|
globusRSL = self.GLOBUS_RSL
|
229 |
|
|
if (self.EDG_clock_time):
|
230 |
|
|
globusRSL += '(maxWalltime='+self.EDG_clock_time+')'
|
231 |
|
|
if (globusRSL != ''):
|
232 |
|
|
jobParams += 'globusrsl = ' + globusRSL + '; '
|
233 |
ewv |
1.84 |
|
234 |
|
|
common._db.updateTask_({'jobType':jobParams})
|
235 |
|
|
return jobParams # Not sure I even need to return anything
|
236 |
gutsche |
1.40 |
|
237 |
spiga |
1.92 |
def decodeLogInfo(self, file):
|
238 |
|
|
"""
|
239 |
|
|
Parse logging info file and return main info
|
240 |
|
|
"""
|
241 |
|
|
loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
|
242 |
|
|
reason = loggingInfo.decodeReason(file)
|
243 |
|
|
return reason
|
244 |
|
|
|
245 |
gutsche |
1.1 |
def wsSetupEnvironment(self):
|
246 |
ewv |
1.91 |
"""
|
247 |
|
|
Returns part of a job script which does scheduler-specific work.
|
248 |
|
|
"""
|
249 |
|
|
txt = SchedulerGrid.wsSetupEnvironment(self)
|
250 |
ewv |
1.86 |
|
251 |
ewv |
1.91 |
if int(self.copy_data) == 1:
|
252 |
|
|
if self.SE:
|
253 |
|
|
txt += 'export SE='+self.SE+'\n'
|
254 |
|
|
txt += 'echo "SE = $SE"\n'
|
255 |
|
|
if self.SE_PATH:
|
256 |
|
|
if self.SE_PATH[-1] != '/':
|
257 |
|
|
self.SE_PATH = self.SE_PATH + '/'
|
258 |
|
|
txt += 'export SE_PATH='+self.SE_PATH+'\n'
|
259 |
|
|
txt += 'echo "SE_PATH = $SE_PATH"\n'
|
260 |
|
|
|
261 |
|
|
txt += 'export VO='+self.VO+'\n'
|
262 |
|
|
txt += 'CE=${args[3]}\n'
|
263 |
|
|
txt += 'echo "CE = $CE"\n'
|
264 |
|
|
return txt
|
265 |
gutsche |
1.1 |
|
266 |
|
|
def wsCopyInput(self):
|
267 |
|
|
"""
|
268 |
ewv |
1.58 |
Copy input data from SE to WN
|
269 |
gutsche |
1.1 |
"""
|
270 |
|
|
txt = '\n'
|
271 |
|
|
return txt
|
272 |
|
|
|
273 |
|
|
def listMatch(self, nj):
|
274 |
|
|
"""
|
275 |
|
|
Check the compatibility of available resources
|
276 |
|
|
"""
|
277 |
slacapra |
1.39 |
#self.checkProxy()
|
278 |
gutsche |
1.1 |
return 1
|
279 |
|
|
|
280 |
slacapra |
1.65 |
def userName(self):
|
281 |
|
|
""" return the user name """
|
282 |
|
|
self.checkProxy()
|
283 |
|
|
return runCommand("voms-proxy-info -identity")
|
284 |
|
|
|
285 |
ewv |
1.81 |
def ce_list(self):
|
286 |
|
|
"""
|
287 |
|
|
Returns string with requirement CE related, dummy for now
|
288 |
|
|
"""
|
289 |
|
|
req = ''
|
290 |
|
|
|
291 |
|
|
return req,self.EDG_ce_white_list,self.EDG_ce_black_list
|