ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerGlite.py
Revision: 1.42
Committed: Thu Mar 27 15:22:31 2008 UTC (17 years, 1 month ago) by ewv
Content type: text/x-python
Branch: MAIN
Changes since 1.41: +14 -13 lines
Log Message:
Move checkProxy() to SchedulerGlite

File Contents

# User Rev Content
1 slacapra 1.31 from SchedulerGrid import SchedulerGrid
2 spiga 1.2 from crab_logger import Logger
3     from crab_exceptions import *
4     from crab_util import *
5 fanzago 1.4 from GliteConfig import *
6 spiga 1.2 import common
7    
8     import os, sys, time
9    
10 slacapra 1.31 class SchedulerGlite(SchedulerGrid):
11 slacapra 1.33 def __init__(self, name="GLITE"):
12     SchedulerGrid.__init__(self,name)
13 slacapra 1.31
14     def configure(self,cfg_params):
15     SchedulerGrid.configure(self, cfg_params)
16 ewv 1.42 self.checkProxy()
17 slacapra 1.31 self.environment_unique_identifier = 'GLITE_WMS_JOBID'
18 slacapra 1.3
19 fanzago 1.4 def rb_configure(self, RB):
20 slacapra 1.31 if not RB: return None
21     glite_config = None
22     rb_param_file = None
23 fanzago 1.4
24     gliteConfig = GliteConfig(RB)
25 slacapra 1.31 glite_config = gliteConfig.config()
26 fanzago 1.4
27 slacapra 1.31 if (glite_config ):
28 spiga 1.40 rb_param_file = glite_config
29 slacapra 1.31 return rb_param_file
30 fanzago 1.4
31 spiga 1.36 def ce_list(self):
32 spiga 1.2 """
33 ewv 1.42 Returns string with requirement CE related
34 spiga 1.2 """
35 ewv 1.42 req = ''
36 spiga 1.2 if self.EDG_ce_white_list:
37 mcinquil 1.32 ce_white_list = self.EDG_ce_white_list
38 slacapra 1.13 tmpCe=[]
39     concString = '&&'
40     for ce in ce_white_list:
41     tmpCe.append('RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId)')
42 mcinquil 1.20 if len(tmpCe) == 1:
43     req += " && (" + concString.join(tmpCe) + ") "
44     elif len(tmpCe) > 1:
45     firstCE = 0
46     for reqTemp in tmpCe:
47     if firstCE == 0:
48     req += " && ( (" + reqTemp + ") "
49     firstCE = 1
50     elif firstCE > 0:
51     req += " || (" + reqTemp + ") "
52     if firstCE > 0:
53     req += ") "
54 ewv 1.28
55 spiga 1.2 if self.EDG_ce_black_list:
56 mcinquil 1.32 ce_black_list = self.EDG_ce_black_list
57 slacapra 1.13 tmpCe=[]
58     concString = '&&'
59 spiga 1.2 for ce in ce_black_list:
60 slacapra 1.13 tmpCe.append('(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))')
61 spiga 1.36 if len(tmpCe): req += " && (" + concString.join(tmpCe) + ") "
62    
63     # requirement added to skip gliteCE
64     req += '&& (!RegExp("blah", other.GlueCEUniqueId))'
65    
66 spiga 1.41 return req,self.EDG_ce_white_list,self.EDG_ce_black_list
67 spiga 1.36
68 spiga 1.39 def se_list(self, id, dest):
69 spiga 1.36 """
70 ewv 1.42 Returns string with requirement SE related
71     """
72 spiga 1.39 hostList=self.findSites_(id,dest)
73 spiga 1.36 req=''
74     reqtmp=[]
75     concString = '||'
76    
77     for arg in hostList:
78     reqtmp.append(' Member("'+arg+'" , other.GlueCESEBindGroupSEUniqueID) ')
79    
80     if len(reqtmp): req += " && (" + concString.join(reqtmp) + ") "
81 spiga 1.11
82 spiga 1.36 return req
83    
84     def jdlParam(self):
85     """
86 ewv 1.42 Returns
87     """
88 spiga 1.36 req=''
89     if self.EDG_addJdlParam:
90 ewv 1.42 if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam= self.EDG_addJdlParam[:-1]
91 spiga 1.36 for p in self.EDG_addJdlParam:
92     # param_file.write(string.strip(p)+';\n')
93     req+=string.strip(p)+';\n' ## BL--DS
94     return req
95    
96     def specific_req(self):
97     """
98     Returns string with specific requirements
99 ewv 1.42 """
100 spiga 1.36 req=''
101 spiga 1.2 if self.EDG_clock_time:
102 slacapra 1.13 if (not req == ' '): req = req + ' && '
103     req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
104 spiga 1.2
105     if self.EDG_cpu_time:
106 slacapra 1.13 if (not req == ' '): req = req + ' && '
107     req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
108 ewv 1.42
109 spiga 1.36 return req
110 ewv 1.28
111 spiga 1.39 def sched_fix_parameter(self):
112 spiga 1.36 """
113     Returns string with requirements and scheduler-specific parameters
114     """
115 ewv 1.42 index = int(common._db.nJobs())
116 spiga 1.39 job = common.job_list[index-1]
117 spiga 1.36 jbt = job.type()
118     req = ''
119     req = req + jbt.getRequirements()
120 slacapra 1.13
121 spiga 1.36 if self.EDG_requirements:
122     if (not req == ' '): req = req + ' && '
123     req = req + self.EDG_requirements
124 spiga 1.2
125 spiga 1.39 Task_Req={'jobType':req}## DS--BL
126 ewv 1.42 common._db.updateTask_(Task_Req)
127 spiga 1.39
128     def sched_parameter(self,i,task):
129     """
130     Returns string with requirements and scheduler-specific parameters
131     """
132     dest= eval(task.jobs[i]['dlsDestination']) ## DS--BL
133    
134     req=''
135     req +=task['jobType']
136    
137     sched_param=''
138     sched_param+='Requirements = ' + req +self.specific_req() + self.se_list(i,dest) +\
139 spiga 1.41 self.ce_list()[0] +';\n' ## BL--DS
140 spiga 1.39 if self.EDG_addJdlParam: sched_param+=self.jdlParam() ## BL--DS
141     sched_param+='MyProxyServer = "' + self.proxyServer + '";\n'
142     sched_param+='VirtualOrganisation = "' + self.VO + '";\n'
143     sched_param+='RetryCount = '+str(self.EDG_retry_count)+';\n'
144     sched_param+='ShallowRetryCount = '+str(self.EDG_shallow_retry_count)+';\n'
145    
146     return sched_param
147 ewv 1.42
148 spiga 1.2 def wsSetupEnvironment(self):
149     """
150     Returns part of a job script which does scheduler-specific work.
151     """
152     txt = ''
153     txt += '# strip arguments\n'
154     txt += 'echo "strip arguments"\n'
155     txt += 'args=("$@")\n'
156     txt += 'nargs=$#\n'
157     txt += 'shift $nargs\n'
158     txt += "# job number (first parameter for job wrapper)\n"
159 ewv 1.34 txt += "NJob=${args[0]}; export NJob\n"
160 spiga 1.2
161     txt += '# job identification to DashBoard \n'
162 fanzago 1.37 #txt += 'MonitorJobID=`echo ${NJob}_$GLITE_WMS_JOBID`\n'
163     #txt += 'SyncGridJobId=`echo $GLITE_WMS_JOBID`\n'
164     #txt += 'MonitorID=`echo ' + self._taskId + '`\n'
165     txt += 'MonitorJobID=${NJob}_$GLITE_WMS_JOBID \n'
166     txt += 'SyncGridJobId=$GLITE_WMS_JOBID \n'
167     txt += 'MonitorID='+self._taskId+' \n'
168     txt += 'echo "MonitorJobID=$MonitorJobID" > $RUNTIME_AREA/$repo \n'
169     txt += 'echo "SyncGridJobId=$SyncGridJobId" >> $RUNTIME_AREA/$repo \n'
170     txt += 'echo "MonitorID=$MonitorID" >> $RUNTIME_AREA/$repo\n'
171     #txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
172     #txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
173     #txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
174 spiga 1.2
175 fanzago 1.37 #txt += 'echo "middleware discovery: " \n'
176     txt += 'echo ">>> GridFlavour discovery: " \n'
177 spiga 1.2 txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
178 fanzago 1.37 txt += ' middleware=LCG\n'
179     #txt += ' echo "SyncCE=`glite-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
180     txt += ' echo "SyncCE=`glite-brokerinfo getCE`" >> $RUNTIME_AREA/$repo \n'
181     #txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
182     txt += ' echo "GridFlavour=$middleware" | tee -a $RUNTIME_AREA/$repo \n'
183     #txt += ' echo ">>> middleware =$middleware" \n'
184 spiga 1.2 txt += 'elif [ $OSG_APP ]; then \n'
185     txt += ' middleware=OSG \n'
186 mcinquil 1.25 txt += ' if [ $OSG_JOB_CONTACT ]; then \n'
187     txt += ' SyncCE="$OSG_JOB_CONTACT"; \n'
188 fanzago 1.37 #txt += ' echo "SyncCE=$SyncCE" | tee -a $RUNTIME_AREA/$repo ;\n'
189     txt += ' echo "SyncCE=$SyncCE" >> $RUNTIME_AREA/$repo ;\n'
190 mcinquil 1.22 txt += ' else\n'
191     txt += ' echo "not reporting SyncCE";\n'
192     txt += ' fi\n';
193 fanzago 1.37 txt += ' echo "GridFlavour=$middleware" | tee -a $RUNTIME_AREA/$repo \n'
194     #txt += ' echo ">>> middleware =$middleware" \n'
195 spiga 1.2 txt += 'else \n'
196 fanzago 1.37 txt += ' echo "ERROR ==> GridFlavour not identified" \n'
197     txt += ' job_exit_code=10030\n'
198     txt += ' func_exit \n'
199     #txt += ' echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
200     #txt += ' echo "JOB_EXIT_STATUS = 10030" \n'
201     #txt += ' echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n'
202     #txt += ' dumpStatus $RUNTIME_AREA/$repo \n'
203     #txt += ' exit 1 \n'
204 spiga 1.2 txt += 'fi \n'
205    
206     txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
207 ewv 1.28
208 spiga 1.2 txt += '\n\n'
209    
210     txt += 'export VO='+self.VO+'\n'
211 ewv 1.28 txt += 'if [ $middleware == LCG ]; then\n'
212 spiga 1.2 txt += ' CloseCEs=`glite-brokerinfo getCE`\n'
213     txt += ' echo "CloseCEs = $CloseCEs"\n'
214     txt += ' CE=`echo $CloseCEs | sed -e "s/:.*//"`\n'
215     txt += ' echo "CE = $CE"\n'
216     txt += 'elif [ $middleware == OSG ]; then \n'
217     txt += ' if [ $OSG_JOB_CONTACT ]; then \n'
218     txt += ' CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n'
219     txt += ' else \n'
220 fanzago 1.35 txt += ' echo "ERROR ==> OSG mode in setting CE name from OSG_JOB_CONTACT" \n'
221     txt += ' job_exit_code=10099\n'
222     txt += ' func_exit \n'
223     #txt += ' echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n'
224     #txt += ' echo "JOB_EXIT_STATUS = 10099" \n'
225     #txt += ' echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n'
226     #txt += ' dumpStatus $RUNTIME_AREA/$repo \n'
227     #txt += ' exit 1 \n'
228 spiga 1.2 txt += ' fi \n'
229 ewv 1.28 txt += 'fi \n'
230 spiga 1.2
231     return txt
232 ewv 1.28
233 spiga 1.2 def loggingInfo(self, id):
234     """
235     retrieve the logging info from logging and bookkeeping and return it
236     """
237     self.checkProxy()
238 slacapra 1.8 cmd = 'glite-job-logging-info -v 3 ' + id
239 spiga 1.2 cmd_out = runCommand(cmd)
240     return cmd_out
241    
242     def queryDetailedStatus(self, id):
243     """ Query a detailed status of the job with id """
244     cmd = 'glite-job-status '+id
245     cmd_out = runCommand(cmd)
246     return cmd_out
247    
248 spiga 1.39 def findSites_(self, n, sites):
249     itr4 =[]
250     if len(sites)>0 and sites[0]=="":
251     return itr4
252     if sites != [""]:
253     ##Addedd Daniele
254     replicas = self.blackWhiteListParser.checkBlackList(sites,n)
255     if len(replicas)!=0:
256     replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
257 ewv 1.42
258 spiga 1.39 itr4 = replicas
259     #####
260     return itr4
261    
262    
263    
264 spiga 1.14
265 mcinquil 1.27 def tOut(self, list):
266 mcinquil 1.18 return 180