ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerRcondor.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerRcondor.py (file contents):
Revision 1.10 by belforte, Fri Aug 10 17:54:30 2012 UTC vs.
Revision 1.12 by belforte, Thu Aug 23 13:25:47 2012 UTC

# Line 5 | Line 5 | Implements the vanilla (local) Remote Co
5   from SchedulerGrid  import SchedulerGrid
6   from crab_exceptions import CrabException
7   from crab_util import runCommand
8 < #from WMCore.SiteScreening.BlackWhiteListParser import CEBlackWhiteListParser
8 > from ServerConfig import *
9   from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
10   import Scram
11  
12  
13 –
13   import common
14   import os
15   import socket
16   import re
17 + import commands
18  
19   # FUTURE: for python 2.4 & 2.6
20   try:
# Line 24 | Line 24 | except:
24  
25   class SchedulerRcondor(SchedulerGrid) :
26      """
27 <    Class to implement the vanilla (local) Condor scheduler
27 >    Class to implement the vanilla remote Condor scheduler
28       Naming convention:  Methods starting with 'ws' provide
29       the corresponding part of the job script
30       ('ws' stands for 'write script').
# Line 32 | Line 32 | class SchedulerRcondor(SchedulerGrid) :
32  
33      def __init__(self):
34          SchedulerGrid.__init__(self,"RCONDOR")
35 <        self.rcondorHost   = os.getenv('RCONDOR_HOST')
36 <        if self.rcondorHost == None:
37 <            raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
35 >
36          self.datasetPath   = None
37          self.selectNoInput = None
38          self.OSBsize = 50*1000*1000 # 50 MB
# Line 48 | Line 46 | class SchedulerRcondor(SchedulerGrid) :
46          """
47          Configure the scheduler with the config settings from the user
48          """
49 +        
50 + #        task  = common._db.getTask()
51 + #        #print task.__dict__
52 + #
53 + #        if task['serverName']!=None and task['serverName']!="":
54 + #            # cast to string to avoid issues with unicode :-(
55 + #            self.rcondorUserHost=str(task['serverName'])
56 + #            common.logger.info("serverName from Task DB is %s" %
57 + #                               self.rcondorUserHost)
58 + #        else :
59 + #            # get an rcondor host from config and save
60 + #            common.logger.info("no serverName in Task DB, use env.var.")
61 + #
62 + #            self.rcondorHost   = os.getenv('RCONDOR_HOST')
63 + #            if not self.rcondorHost :
64 + #                raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
65 + #            self.rcondorUser = os.getenv('RCONDOR_USER')
66 + #            if not self.rcondorUser :
67 + #                common.logger.info("$RCONDOR_USER not defined, try to find out via uberftp ...")
68 + #                command="uberftp $RCONDOR_HOST pwd|grep User|awk '{print $3}'"
69 + #                (status, output) = commands.getstatusoutput(command)
70 + #                if status == 0:
71 + #                    self.rcondorUser = output
72 + #                    common.logger.info("rcondorUser set to %s" % self.rcondorUser)
73 + #                if self.rcondorUser==None:
74 + #                    raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
75 + #
76 + #                self.rcondorUserHost = self.rcondorUser + '@' + self.rcondorHost
77 + #
78 + #            print "will set server name to : ", self.rcondorUserHost
79 + #            common._db.updateTask_({'serverName':self.rcondorUserHost})
80 + #            print "ok"
81  
82          SchedulerGrid.configure(self, cfg_params)
83  
# Line 58 | Line 88 | class SchedulerRcondor(SchedulerGrid) :
88          self.group = cfg_params.get("GRID.group", None)
89          self.role = cfg_params.get("GRID.role", None)
90          self.VO = cfg_params.get('GRID.virtual_organization','cms')
91 <        
91 >
92 >        self.checkProxy()
93 >
94 >
95          try:
96              tmp =  cfg_params['CMSSW.datasetpath']
97              if tmp.lower() == 'none':
# Line 71 | Line 104 | class SchedulerRcondor(SchedulerGrid) :
104              msg = "Error: datasetpath not defined "
105              raise CrabException(msg)
106  
107 <        self.checkProxy()
107 >        if cfg_params.get('GRID.ce_black_list', None) or \
108 >           cfg_params.get('GRID.ce_white_list', None) :
109 >            msg="BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
110 >            msg+="\n Remove them from crab configuration to proceed."
111 >            msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
112 >            raise CrabException(msg)
113  
114          return
115      
# Line 88 | Line 126 | class SchedulerRcondor(SchedulerGrid) :
126      def sched_parameter(self, i, task):
127          """
128          Return scheduler-specific parameters. Used at crab -submit time
129 +        by $CRABPYTHON/Scheduler.py
130          """
131  
132   #SB paste from crab ScheduerGlidein
133  
134          jobParams = ""
135  
136 +        (self.rcondorHost,self.rcondorUserHost) = self.pickRcondorSubmissionHost(task)
137 +
138          seDest = task.jobs[i-1]['dlsDestination']
139  
140          if seDest == [''] :
# Line 114 | Line 155 | class SchedulerRcondor(SchedulerGrid) :
155          jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";'
156          jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";'
157          
158 <        myschedName = self.rcondorHost
159 <        jobParams += '+Glidein_MonitorID = "https://'+ myschedName + '//$(Cluster).$(Process)"; '
158 >        myscheddName = self.rcondorHost
159 >        jobParams += '+Glidein_MonitorID = "https://'+ myscheddName  + '//$(Cluster).$(Process)"; '
160  
161          if (self.EDG_clock_time):
162              jobParams += '+MaxWallTimeMins = '+self.EDG_clock_time+'; '
# Line 139 | Line 180 | class SchedulerRcondor(SchedulerGrid) :
180          jobDir = common.work_space.jobDir()
181          taskDir=common.work_space.topDir().split('/')[-2]
182          shareDir = common.work_space.shareDir()
142 –        #SBtmpDir = common.work_space.tmpDir()
183          
184 <        params = {'rcondorHost':self.rcondorHost,
145 <                  'shareDir':shareDir,
146 <                  #SB'tmpDir':tmpDir,
184 >        params = {'shareDir':shareDir,
185                    'jobDir':jobDir,
186                    'taskDir':taskDir}
187  
# Line 233 | Line 271 | class SchedulerRcondor(SchedulerGrid) :
271              taskReq = {'commonRequirements':req}
272              common._db.updateTask_(taskReq)
273  
274 +    def pickRcondorSubmissionHost(self, task):
275 +    
276 +        task  = common._db.getTask()
277 +
278 +        if task['serverName']!=None and task['serverName']!="":
279 +            # rcondorHost is already defined and stored for this task
280 +            # so pick it from DB
281 +            # cast to string to avoid issues with unicode :-(
282 +            rcondorUserHost=str(task['serverName'])
283 +            common.logger.info("serverName from Task DB is %s" %
284 +                               rcondorUserHost)
285 +            if '@' in rcondorUserHost:
286 +                rcondorHost = rcondorUserHost.split('@')[1]
287 +            else:
288 +                rcondorHost = rcondorUserHost
289 +        else:
290 +            if self.cfg_params.has_key('CRAB.submit_host'):
291 +                # get an rcondor host from crab config file
292 +                srvCfg=ServerConfig(self.cfg_params['CRAB.submit_host']).config()
293 +                rcondorHost=srvCfg['serverName']
294 +                common.logger.info("rcondorhost from crab.cfg = %s" % rcondorHost)
295 +            else:
296 +                # pick from Available Servers List
297 +                srvCfg=ServerConfig('default').config()
298 +                print srvCfg
299 +                rcondorHost = srvCfg['serverName']
300 +                common.logger.info("rcondorhost from Avail.List = %s" % rcondorHost)
301 +
302 +            if not rcondorHost:
303 +                raise CrabException('FATAL ERROR: condorHost not defined')
304 +                # fall back to env.
305 +                #common.logger.info("no serverName in Task DB, use env.var.")
306 +                #rcondorHost   = os.getenv('RCONDOR_HOST')
307 +                #if not rcondorHost :
308 +                #    raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
309 +            
310 +            #rcondorUser = os.getenv('RCONDOR_USER')
311 +            #if not rcondorUser :
312 +            common.logger.info("try to find out RCONDOR_USER via uberftp ...")
313 +            command="uberftp %s pwd|grep User|awk '{print $3}'" % rcondorHost
314 +            (status, output) = commands.getstatusoutput(command)
315 +            if status == 0:
316 +                rcondorUser = output
317 +                common.logger.info("rcondorUser set to %s" % rcondorUser)
318 +                if rcondorUser==None:
319 +                    raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
320 +
321 +            rcondorUserHost = rcondorUser + '@' + rcondorHost
322 +
323 +        print "will set server name to : ", rcondorUserHost
324 +        common._db.updateTask_({'serverName':rcondorUserHost})
325  
326 +        return (rcondorHost, rcondorUserHost)

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines