2 |
|
from crab_logger import Logger |
3 |
|
from crab_exceptions import * |
4 |
|
from crab_util import * |
5 |
+ |
from EdgConfig import * |
6 |
|
import common |
7 |
|
|
8 |
< |
import os, sys, tempfile |
8 |
> |
import os, sys, time |
9 |
|
|
10 |
|
class SchedulerEdg(Scheduler): |
11 |
|
def __init__(self): |
21 |
|
return |
22 |
|
|
23 |
|
def configure(self, cfg_params): |
24 |
+ |
# test branch 2 |
25 |
+ |
try: |
26 |
+ |
RB = cfg_params["EDG.rb"] |
27 |
+ |
edgConfig = EdgConfig(RB) |
28 |
+ |
self.edg_config = edgConfig.config() |
29 |
+ |
self.edg_config_vo = edgConfig.configVO() |
30 |
+ |
except KeyError: |
31 |
+ |
self.edg_config = '' |
32 |
+ |
self.edg_config_vo = '' |
33 |
|
|
34 |
< |
try: self.edg_config = cfg_params["EDG.config"] |
35 |
< |
except KeyError: self.edg_config = '' |
36 |
< |
|
37 |
< |
try: self.edg_config_vo = cfg_params["EDG.config_vo"] |
38 |
< |
except KeyError: self.edg_config_vo = '' |
34 |
> |
try: |
35 |
> |
self.proxyServer = cfg_params["EDG.proxy_server"] |
36 |
> |
except KeyError: |
37 |
> |
self.proxyServer = 'myproxy.cern.ch' |
38 |
> |
common.logger.debug(5,'Setting myproxy server to '+self.proxyServer) |
39 |
|
|
40 |
|
try: self.LCG_version = cfg_params["EDG.lcg_version"] |
41 |
|
except KeyError: self.LCG_version = '2' |
46 |
|
try: self.EDG_retry_count = cfg_params['EDG.retry_count'] |
47 |
|
except KeyError: self.EDG_retry_count = '' |
48 |
|
|
49 |
+ |
try: |
50 |
+ |
self.EDG_ce_black_list = cfg_params['EDG.ce_black_list'] |
51 |
+ |
#print "self.EDG_ce_black_list = ", self.EDG_ce_black_list |
52 |
+ |
except KeyError: |
53 |
+ |
self.EDG_ce_black_list = '' |
54 |
+ |
|
55 |
+ |
try: |
56 |
+ |
self.EDG_ce_white_list = cfg_params['EDG.ce_white_list'] |
57 |
+ |
#print "self.EDG_ce_white_list = ", self.EDG_ce_white_list |
58 |
+ |
except KeyError: self.EDG_ce_white_list = '' |
59 |
+ |
|
60 |
+ |
try: self.VO = cfg_params['EDG.virtual_organization'] |
61 |
+ |
except KeyError: self.VO = 'cms' |
62 |
+ |
|
63 |
+ |
try: self.return_data = cfg_params['USER.return_data'] |
64 |
+ |
except KeyError: self.return_data = 1 |
65 |
+ |
|
66 |
+ |
try: |
67 |
+ |
self.copy_input_data = common.analisys_common_info['copy_input_data'] |
68 |
+ |
#print "self.copy_input_data = ", self.copy_input_data |
69 |
+ |
except KeyError: self.copy_input_data = 0 |
70 |
+ |
|
71 |
+ |
try: |
72 |
+ |
self.copy_data = cfg_params["USER.copy_data"] |
73 |
+ |
if int(self.copy_data) == 1: |
74 |
+ |
try: |
75 |
+ |
self.SE = cfg_params['USER.storage_element'] |
76 |
+ |
self.SE_PATH = cfg_params['USER.storage_path'] |
77 |
+ |
except KeyError: |
78 |
+ |
msg = "Error. The [USER] section does not have 'storage_element'" |
79 |
+ |
msg = msg + " and/or 'storage_path' entries, necessary to copy the output" |
80 |
+ |
common.logger.message(msg) |
81 |
+ |
raise CrabException(msg) |
82 |
+ |
except KeyError: self.copy_data = 0 |
83 |
+ |
|
84 |
+ |
if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ): |
85 |
+ |
msg = 'Warning: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n' |
86 |
+ |
msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n' |
87 |
+ |
raise CrabException(msg) |
88 |
+ |
|
89 |
+ |
try: |
90 |
+ |
self.lfc_host = cfg_params['EDG.lfc_host'] |
91 |
+ |
except KeyError: |
92 |
+ |
msg = "Error. The [EDG] section does not have 'lfc_host' value" |
93 |
+ |
msg = msg + " it's necessary to know the LFC host name" |
94 |
+ |
common.logger.message(msg) |
95 |
+ |
raise CrabException(msg) |
96 |
+ |
try: |
97 |
+ |
self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type'] |
98 |
+ |
except KeyError: |
99 |
+ |
msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value" |
100 |
+ |
msg = msg + " it's necessary to know the catalog type" |
101 |
+ |
common.logger.message(msg) |
102 |
+ |
raise CrabException(msg) |
103 |
|
try: |
104 |
< |
self.VO = cfg_params['EDG.virtual_organization'] |
104 |
> |
self.lfc_home = cfg_params['EDG.lfc_home'] |
105 |
|
except KeyError: |
106 |
< |
self.VO = 'cms' |
106 |
> |
msg = "Error. The [EDG] section does not have 'lfc_home' value" |
107 |
> |
msg = msg + " it's necessary to know the home catalog dir" |
108 |
> |
common.logger.message(msg) |
109 |
> |
raise CrabException(msg) |
110 |
> |
|
111 |
> |
try: |
112 |
> |
self.register_data = cfg_params["USER.register_data"] |
113 |
> |
if int(self.register_data) == 1: |
114 |
> |
try: |
115 |
> |
self.LFN = cfg_params['USER.lfn_dir'] |
116 |
> |
except KeyError: |
117 |
> |
msg = "Error. The [USER] section does not have 'lfn_dir' value" |
118 |
> |
msg = msg + " it's necessary for LCF registration" |
119 |
> |
common.logger.message(msg) |
120 |
> |
raise CrabException(msg) |
121 |
> |
except KeyError: self.register_data = 0 |
122 |
> |
|
123 |
> |
if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ): |
124 |
> |
msg = 'Warning: register_data = 1 must be used with copy_data = 1\n' |
125 |
> |
msg = msg + 'Please modify copy_data value in your crab.cfg file\n' |
126 |
> |
common.logger.message(msg) |
127 |
> |
raise CrabException(msg) |
128 |
> |
|
129 |
> |
try: self.EDG_requirements = cfg_params['EDG.requirements'] |
130 |
> |
except KeyError: self.EDG_requirements = '' |
131 |
> |
|
132 |
> |
try: self.EDG_retry_count = cfg_params['EDG.retry_count'] |
133 |
> |
except KeyError: self.EDG_retry_count = '' |
134 |
> |
|
135 |
> |
try: self.EDG_clock_time = cfg_params['EDG.max_wall_clock_time'] |
136 |
> |
except KeyError: self.EDG_clock_time= '' |
137 |
> |
|
138 |
> |
try: self.EDG_cpu_time = cfg_params['EDG.max_cpu_time'] |
139 |
> |
except KeyError: self.EDG_cpu_time = '' |
140 |
|
|
141 |
|
# Add EDG_WL_LOCATION to the python path |
142 |
|
|
151 |
|
libPath=os.path.join(path, "lib", "python") |
152 |
|
sys.path.append(libPath) |
153 |
|
|
154 |
< |
self.checkProxy_() |
154 |
> |
self.proxyValid=0 |
155 |
> |
|
156 |
> |
try: |
157 |
> |
self._taskId = cfg_params['taskId'] |
158 |
> |
except: |
159 |
> |
self._taskId = '' |
160 |
> |
|
161 |
|
return |
162 |
|
|
163 |
|
|
175 |
|
return 1 |
176 |
|
else: |
177 |
|
return 0 |
178 |
+ |
|
179 |
|
def wsSetupEnvironment(self): |
180 |
|
""" |
181 |
|
Returns part of a job script which does scheduler-specific work. |
182 |
|
""" |
183 |
< |
txt = '\n' |
184 |
< |
txt += 'CloseCEs=`edg-brokerinfo getCE`\n' |
185 |
< |
txt += 'echo "CloseCEs = $CloseCEs"\n' |
186 |
< |
txt += 'CE=`echo $CloseCEs | sed -e "s/:.*//"`\n' |
187 |
< |
txt += 'echo "CE = $CE"\n' |
183 |
> |
txt = '' |
184 |
> |
txt += "# job number (first parameter for job wrapper)\n" |
185 |
> |
txt += "NJob=$1\n" |
186 |
> |
|
187 |
> |
txt += '# job identification to DashBoard \n' |
188 |
> |
txt += 'MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`\n' |
189 |
> |
txt += 'SyncGridJobId=`echo $EDG_WL_JOBID`\n' |
190 |
> |
txt += 'MonitorID=`echo ' + self._taskId + '`\n' |
191 |
> |
txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
192 |
> |
txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n' |
193 |
> |
txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
194 |
> |
|
195 |
> |
txt += 'echo "middleware discovery " \n' |
196 |
> |
txt += 'if [ $VO_CMS_SW_DIR ]; then \n' |
197 |
> |
txt += ' middleware=LCG \n' |
198 |
> |
txt += ' echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n' |
199 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
200 |
> |
txt += ' echo "middleware =$middleware" \n' |
201 |
> |
txt += 'elif [ $GRID3_APP_DIR ]; then\n' |
202 |
> |
txt += ' middleware=OSG \n' |
203 |
> |
txt += ' echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n' |
204 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
205 |
> |
txt += ' echo "middleware =$middleware" \n' |
206 |
> |
txt += 'elif [ $OSG_APP ]; then \n' |
207 |
> |
txt += ' middleware=OSG \n' |
208 |
> |
txt += ' echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n' |
209 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
210 |
> |
txt += ' echo "middleware =$middleware" \n' |
211 |
> |
txt += 'else \n' |
212 |
> |
txt += ' echo "SET_CMS_ENV 10030 ==> middleware not identified" \n' |
213 |
> |
txt += ' echo "JOB_EXIT_STATUS = 10030" \n' |
214 |
> |
txt += ' echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n' |
215 |
> |
txt += ' dumpStatus $RUNTIME_AREA/$repo \n' |
216 |
> |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
217 |
> |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
218 |
> |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
219 |
> |
txt += ' exit 1 \n' |
220 |
> |
txt += 'fi \n' |
221 |
> |
|
222 |
> |
txt += '# report first time to DashBoard \n' |
223 |
> |
txt += 'dumpStatus $RUNTIME_AREA/$repo \n' |
224 |
> |
txt += 'rm -f $RUNTIME_AREA/$repo \n' |
225 |
> |
txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
226 |
> |
txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
227 |
> |
|
228 |
> |
txt += '\n\n' |
229 |
> |
|
230 |
> |
if int(self.copy_data) == 1: |
231 |
> |
if self.SE: |
232 |
> |
txt += 'export SE='+self.SE+'\n' |
233 |
> |
txt += 'echo "SE = $SE"\n' |
234 |
> |
if self.SE_PATH: |
235 |
> |
if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/' |
236 |
> |
txt += 'export SE_PATH='+self.SE_PATH+'\n' |
237 |
> |
txt += 'echo "SE_PATH = $SE_PATH"\n' |
238 |
> |
|
239 |
> |
txt += 'export VO='+self.VO+'\n' |
240 |
> |
### FEDE: add some line for LFC catalog setting |
241 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
242 |
> |
txt += ' if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n' |
243 |
> |
txt += ' export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n' |
244 |
> |
txt += ' fi\n' |
245 |
> |
txt += ' if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n' |
246 |
> |
txt += ' export LFC_HOST='+self.lfc_host+'\n' |
247 |
> |
txt += ' fi\n' |
248 |
> |
txt += ' if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n' |
249 |
> |
txt += ' export LFC_HOME='+self.lfc_home+'\n' |
250 |
> |
txt += ' fi\n' |
251 |
> |
txt += 'elif [ $middleware == OSG ]; then\n' |
252 |
> |
txt += ' echo "LFC catalog setting to be implemented for OSG"\n' |
253 |
> |
txt += 'fi\n' |
254 |
> |
##### |
255 |
> |
if int(self.register_data) == 1: |
256 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
257 |
> |
txt += ' export LFN='+self.LFN+'\n' |
258 |
> |
txt += ' lfc-ls $LFN\n' |
259 |
> |
txt += ' result=$?\n' |
260 |
> |
txt += ' echo $result\n' |
261 |
> |
### creation of LFN dir in LFC catalog, under /grid/cms dir |
262 |
> |
txt += ' if [ $result != 0 ]; then\n' |
263 |
> |
txt += ' lfc-mkdir $LFN\n' |
264 |
> |
txt += ' result=$?\n' |
265 |
> |
txt += ' echo $result\n' |
266 |
> |
txt += ' fi\n' |
267 |
> |
txt += 'elif [ $middleware == OSG ]; then\n' |
268 |
> |
txt += ' echo " Files registration to be implemented for OSG"\n' |
269 |
> |
txt += 'fi\n' |
270 |
> |
txt += '\n' |
271 |
> |
|
272 |
> |
if self.VO: |
273 |
> |
txt += 'export VO='+self.VO+'\n' |
274 |
> |
if self.LFN: |
275 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
276 |
> |
txt += ' export LFN='+self.LFN+'\n' |
277 |
> |
txt += 'fi\n' |
278 |
> |
txt += '\n' |
279 |
> |
|
280 |
> |
txt += 'if [ $middleware == LCG ]; then\n' |
281 |
> |
txt += ' CloseCEs=`edg-brokerinfo getCE`\n' |
282 |
> |
txt += ' echo "CloseCEs = $CloseCEs"\n' |
283 |
> |
txt += ' CE=`echo $CloseCEs | sed -e "s/:.*//"`\n' |
284 |
> |
txt += ' echo "CE = $CE"\n' |
285 |
> |
txt += 'elif [ $middleware == OSG ]; then \n' |
286 |
> |
txt += ' if [ $OSG_JOB_CONTACT ]; then \n' |
287 |
> |
txt += ' CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n' |
288 |
> |
txt += ' else \n' |
289 |
> |
txt += ' echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n' |
290 |
> |
txt += ' echo "JOB_EXIT_STATUS = 10099" \n' |
291 |
> |
txt += ' echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n' |
292 |
> |
txt += ' dumpStatus $RUNTIME_AREA/$repo \n' |
293 |
> |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
294 |
> |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
295 |
> |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
296 |
> |
txt += ' exit 1 \n' |
297 |
> |
txt += ' fi \n' |
298 |
> |
txt += 'fi \n' |
299 |
> |
|
300 |
> |
return txt |
301 |
> |
|
302 |
> |
def wsCopyInput(self): |
303 |
> |
""" |
304 |
> |
Copy input data from SE to WN |
305 |
> |
""" |
306 |
> |
txt = '' |
307 |
> |
try: |
308 |
> |
self.copy_input_data = common.analisys_common_info['copy_input_data'] |
309 |
> |
#print "self.copy_input_data = ", self.copy_input_data |
310 |
> |
except KeyError: self.copy_input_data = 0 |
311 |
> |
if int(self.copy_input_data) == 1: |
312 |
> |
## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG) |
313 |
> |
txt += 'if [ $middleware == OSG ]; then\n' |
314 |
> |
txt += ' #\n' |
315 |
> |
txt += ' # Copy Input Data from SE to this WN deactivated in OSG mode\n' |
316 |
> |
txt += ' #\n' |
317 |
> |
txt += ' echo "Copy Input Data from SE to this WN deactivated in OSG mode"\n' |
318 |
> |
txt += 'elif [ $middleware == LCG ]; then \n' |
319 |
> |
txt += ' #\n' |
320 |
> |
txt += ' # Copy Input Data from SE to this WN\n' |
321 |
> |
txt += ' #\n' |
322 |
> |
### changed by georgia (put a loop copying more than one input files per jobs) |
323 |
> |
txt += ' for input_file in $cur_file_list \n' |
324 |
> |
txt += ' do \n' |
325 |
> |
#### FEDE |
326 |
> |
#txt += ' echo "which lcg-cp" \n' |
327 |
> |
#txt += ' which lcg-cp \n' |
328 |
> |
######### |
329 |
> |
txt += ' lcg-cp --vo $VO --verbose -t 1200 lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n' |
330 |
> |
txt += ' copy_input_exit_status=$?\n' |
331 |
> |
txt += ' echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n' |
332 |
> |
txt += ' if [ $copy_input_exit_status -ne 0 ]; then \n' |
333 |
> |
txt += ' echo "Problems with copying to WN" \n' |
334 |
> |
txt += ' else \n' |
335 |
> |
txt += ' echo "input copied into WN" \n' |
336 |
> |
txt += ' fi \n' |
337 |
> |
txt += ' done \n' |
338 |
> |
### copy a set of PU ntuples (same for each jobs -- but accessed randomly) |
339 |
> |
txt += ' for file in $cur_pu_list \n' |
340 |
> |
txt += ' do \n' |
341 |
> |
#### FEDE |
342 |
> |
#txt += ' echo "which lcg-cp" \n' |
343 |
> |
#txt += ' which lcg-cp \n' |
344 |
> |
######### |
345 |
> |
txt += ' lcg-cp --vo $VO --verbose -t 1200 lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n' |
346 |
> |
txt += ' copy_input_pu_exit_status=$?\n' |
347 |
> |
txt += ' echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n' |
348 |
> |
txt += ' if [ $copy_input_pu_exit_status -ne 0 ]; then \n' |
349 |
> |
txt += ' echo "Problems with copying pu to WN" \n' |
350 |
> |
txt += ' else \n' |
351 |
> |
txt += ' echo "input pu files copied into WN" \n' |
352 |
> |
txt += ' fi \n' |
353 |
> |
txt += ' done \n' |
354 |
> |
txt += ' \n' |
355 |
> |
txt += ' ### Check SCRATCH space available on WN : \n' |
356 |
> |
txt += ' df -h \n' |
357 |
> |
txt += 'fi \n' |
358 |
> |
|
359 |
> |
return txt |
360 |
> |
|
361 |
> |
def wsCopyOutput(self): |
362 |
> |
""" |
363 |
> |
Write a CopyResults part of a job script, e.g. |
364 |
> |
to copy produced output into a storage element. |
365 |
> |
""" |
366 |
> |
txt = '' |
367 |
> |
if int(self.copy_data) == 1: |
368 |
> |
txt += '#\n' |
369 |
> |
txt += '# Copy output to SE = $SE\n' |
370 |
> |
txt += '#\n' |
371 |
> |
#txt += 'if [ $exe_result -eq 0 ]; then\n' |
372 |
> |
txt += ' if [ $middleware == OSG ]; then\n' |
373 |
> |
txt += ' echo "X509_USER_PROXY = $X509_USER_PROXY"\n' |
374 |
> |
txt += ' echo "source $OSG_APP/glite/setup_glite_ui.sh"\n' |
375 |
> |
txt += ' source $OSG_APP/glite/setup_glite_ui.sh\n' |
376 |
> |
txt += ' export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n' |
377 |
> |
txt += ' echo "export X509_CERT_DIR=$X509_CERT_DIR"\n' |
378 |
> |
txt += ' fi \n' |
379 |
> |
txt += ' for out_file in $file_list ; do\n' |
380 |
> |
txt += ' echo "Trying to copy output file to $SE using lcg-cp"\n' |
381 |
> |
txt += ' echo "lcg-cp --vo $VO -t 1200 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n' |
382 |
> |
txt += ' exitstring=`lcg-cp --vo $VO -t 1200 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n' |
383 |
> |
txt += ' copy_exit_status=$?\n' |
384 |
> |
txt += ' echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n' |
385 |
> |
txt += ' echo "STAGE_OUT = $copy_exit_status"\n' |
386 |
> |
txt += ' if [ $copy_exit_status -ne 0 ]; then\n' |
387 |
> |
txt += ' echo "Possible problem with SE = $SE"\n' |
388 |
> |
txt += ' echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n' |
389 |
> |
txt += ' echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n' |
390 |
> |
txt += ' echo "lcg-cp failed, attempting srmcp"\n' |
391 |
> |
txt += ' echo "mkdir -p $HOME/.srmconfig"\n' |
392 |
> |
txt += ' mkdir -p $HOME/.srmconfig\n' |
393 |
> |
txt += ' if [ $middleware == LCG ]; then\n' |
394 |
> |
txt += ' echo "srmcp -retry_num 5 -retry_timeout 240000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n' |
395 |
> |
txt += ' exitstring=`srmcp -retry_num 5 -retry_timeout 240000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n' |
396 |
> |
txt += ' elif [ $middleware == OSG ]; then\n' |
397 |
> |
txt += ' echo "srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n' |
398 |
> |
txt += ' exitstring=`srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n' |
399 |
> |
txt += ' fi \n' |
400 |
> |
txt += ' copy_exit_status=$?\n' |
401 |
> |
txt += ' echo "COPY_EXIT_STATUS for srm = $copy_exit_status"\n' |
402 |
> |
txt += ' echo "STAGE_OUT = $copy_exit_status"\n' |
403 |
> |
txt += ' if [ $copy_exit_status -ne 0 ]; then\n' |
404 |
> |
txt += ' echo "Problems with SE = $SE"\n' |
405 |
> |
txt += ' echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n' |
406 |
> |
txt += ' echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n' |
407 |
> |
txt += ' echo "lcg-cp and srm failed"\n' |
408 |
> |
txt += ' echo "If storage_path in your config file contains a ? you may need a \? instead."\n' |
409 |
> |
txt += ' else\n' |
410 |
> |
txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n' |
411 |
> |
txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n' |
412 |
> |
txt += ' echo "output copied into $SE/$SE_PATH directory"\n' |
413 |
> |
txt += ' echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n' |
414 |
> |
txt += ' echo "srmcp succeeded"\n' |
415 |
> |
txt += ' fi\n' |
416 |
> |
txt += ' else\n' |
417 |
> |
txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n' |
418 |
> |
txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n' |
419 |
> |
txt += ' echo "output copied into $SE/$SE_PATH directory"\n' |
420 |
> |
txt += ' echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n' |
421 |
> |
txt += ' echo "lcg-cp succeeded"\n' |
422 |
> |
txt += ' fi\n' |
423 |
> |
txt += ' done\n' |
424 |
> |
#txt += 'fi\n' |
425 |
> |
return txt |
426 |
> |
|
427 |
> |
def wsRegisterOutput(self): |
428 |
> |
""" |
429 |
> |
Returns part of a job script which does scheduler-specific work. |
430 |
> |
""" |
431 |
> |
|
432 |
> |
txt = '' |
433 |
> |
if int(self.register_data) == 1: |
434 |
> |
## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG) |
435 |
> |
txt += 'if [ $middleware == OSG ]; then\n' |
436 |
> |
txt += ' #\n' |
437 |
> |
txt += ' # Register output to LFC deactivated in OSG mode\n' |
438 |
> |
txt += ' #\n' |
439 |
> |
txt += ' echo "Register output to LFC deactivated in OSG mode"\n' |
440 |
> |
txt += 'elif [ $middleware == LCG ]; then \n' |
441 |
> |
txt += '#\n' |
442 |
> |
txt += '# Register output to LFC\n' |
443 |
> |
txt += '#\n' |
444 |
> |
#txt += ' if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n' |
445 |
> |
txt += ' if [ $copy_exit_status -eq 0 ]; then\n' |
446 |
> |
txt += ' for out_file in $file_list ; do\n' |
447 |
> |
txt += ' echo "Trying to register the output file into LFC"\n' |
448 |
> |
#### FEDE |
449 |
> |
#txt += ' echo "which lcg-rf" \n' |
450 |
> |
#txt += ' which lcg-rf \n' |
451 |
> |
######### |
452 |
> |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n' |
453 |
> |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n' |
454 |
> |
txt += ' register_exit_status=$?\n' |
455 |
> |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
456 |
> |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
457 |
> |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
458 |
> |
txt += ' echo "Problems with the registration to LFC" \n' |
459 |
> |
txt += ' echo "Try with srm protocol" \n' |
460 |
> |
#### FEDE |
461 |
> |
#txt += ' echo "which lcg-rf" \n' |
462 |
> |
#txt += ' which lcg-rf \n' |
463 |
> |
######### |
464 |
> |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n' |
465 |
> |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n' |
466 |
> |
txt += ' register_exit_status=$?\n' |
467 |
> |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
468 |
> |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
469 |
> |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
470 |
> |
txt += ' echo "Problems with the registration into LFC" \n' |
471 |
> |
txt += ' fi \n' |
472 |
> |
txt += ' else \n' |
473 |
> |
txt += ' echo "output registered to LFC"\n' |
474 |
> |
txt += ' fi \n' |
475 |
> |
txt += ' echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
476 |
> |
txt += ' done\n' |
477 |
> |
#txt += ' elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n' |
478 |
> |
txt += ' else \n' |
479 |
> |
txt += ' echo "Trying to copy output file to CloseSE"\n' |
480 |
> |
txt += ' CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n' |
481 |
> |
txt += ' for out_file in $file_list ; do\n' |
482 |
> |
#### FEDE |
483 |
> |
#txt += ' echo "which lcg-cr" \n' |
484 |
> |
#txt += ' which lcg-cr \n' |
485 |
> |
######### |
486 |
> |
txt += ' echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n' |
487 |
> |
txt += ' lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n' |
488 |
> |
txt += ' register_exit_status=$?\n' |
489 |
> |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
490 |
> |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
491 |
> |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
492 |
> |
txt += ' echo "Problems with CloseSE or Catalog" \n' |
493 |
> |
txt += ' else \n' |
494 |
> |
txt += ' echo "The program was successfully executed"\n' |
495 |
> |
txt += ' echo "SE = $CLOSE_SE"\n' |
496 |
> |
txt += ' echo "LFN for the file is LFN=${LFN}/$out_file"\n' |
497 |
> |
txt += ' fi \n' |
498 |
> |
txt += ' echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
499 |
> |
txt += ' done\n' |
500 |
> |
#txt += ' else\n' |
501 |
> |
#txt += ' echo "Problem with the executable"\n' |
502 |
> |
txt += ' fi \n' |
503 |
> |
txt += ' exit_status=$register_exit_status\n' |
504 |
> |
txt += 'fi \n' |
505 |
|
return txt |
506 |
|
|
507 |
< |
def loggingInfo(self, nj): |
507 |
> |
def loggingInfo(self, id): |
508 |
|
""" |
509 |
|
retrieve the logging info from logging and bookkeeping and return it |
510 |
|
""" |
511 |
< |
id = common.jobDB.jobId(nj) |
512 |
< |
edg_ui_cfg_opt = '' |
513 |
< |
if self.edg_config: |
514 |
< |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
94 |
< |
cmd = 'edg-job-get-logging-info -v 2 ' + edg_ui_cfg_opt + id |
95 |
< |
print cmd |
96 |
< |
myCmd = os.popen(cmd) |
97 |
< |
cmd_out = myCmd.readlines() |
98 |
< |
myCmd.close() |
511 |
> |
self.checkProxy() |
512 |
> |
cmd = 'edg-job-get-logging-info -v 2 ' + id |
513 |
> |
#cmd_out = os.popen(cmd) |
514 |
> |
cmd_out = runCommand(cmd) |
515 |
|
return cmd_out |
516 |
|
|
517 |
|
def listMatch(self, nj): |
518 |
|
""" |
519 |
|
Check the compatibility of available resources |
520 |
|
""" |
521 |
+ |
self.checkProxy() |
522 |
|
jdl = common.job_list[nj].jdlFilename() |
523 |
< |
edg_ui_cfg_opt = '' |
524 |
< |
if self.edg_config: |
525 |
< |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
526 |
< |
if self.edg_config_vo: |
527 |
< |
edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' ' |
111 |
< |
cmd = 'edg-job-list-match ' + edg_ui_cfg_opt + jdl |
112 |
< |
myCmd = os.popen(cmd) |
113 |
< |
cmd_out = myCmd.readlines() |
114 |
< |
myCmd.close() |
523 |
> |
cmd = 'edg-job-list-match ' + self.configOpt_() + jdl |
524 |
> |
cmd_out = runCommand(cmd,0,10) |
525 |
> |
if not cmd_out: |
526 |
> |
raise CrabException("ERROR: "+cmd+" failed!") |
527 |
> |
|
528 |
|
return self.parseListMatch_(cmd_out, jdl) |
529 |
|
|
530 |
|
def parseListMatch_(self, out, jdl): |
531 |
+ |
""" |
532 |
+ |
Parse the f* output of edg-list-match and produce something sensible |
533 |
+ |
""" |
534 |
|
reComment = re.compile( r'^\**$' ) |
535 |
|
reEmptyLine = re.compile( r'^$' ) |
536 |
|
reVO = re.compile( r'Selected Virtual Organisation name.*' ) |
537 |
< |
reCE = re.compile( r'CEId' ) |
537 |
> |
reLine = re.compile( r'.*') |
538 |
> |
reCE = re.compile( r'(.*:.*)') |
539 |
> |
reCEId = re.compile( r'CEId.*') |
540 |
|
reNO = re.compile( r'No Computing Element matching' ) |
541 |
|
reRB = re.compile( r'Connecting to host' ) |
542 |
|
next = 0 |
543 |
|
CEs=[] |
544 |
|
Match=0 |
545 |
< |
for line in out: |
546 |
< |
line = line.strip() |
547 |
< |
if reComment.match( line ): |
548 |
< |
next = 0 |
549 |
< |
continue |
550 |
< |
if reEmptyLine.match(line): |
551 |
< |
continue |
545 |
> |
|
546 |
> |
#print out |
547 |
> |
lines = reLine.findall(out) |
548 |
> |
|
549 |
> |
i=0 |
550 |
> |
CEs=[] |
551 |
> |
for line in lines: |
552 |
> |
string.strip(line) |
553 |
> |
#print line |
554 |
> |
if reNO.match( line ): |
555 |
> |
common.logger.debug(5,line) |
556 |
> |
return 0 |
557 |
> |
pass |
558 |
|
if reVO.match( line ): |
559 |
< |
VO =line.split()[-1] |
560 |
< |
common.logger.debug(5, 'VO :'+VO) |
559 |
> |
VO =reVO.match( line ).group() |
560 |
> |
common.logger.debug(5,"VO "+VO) |
561 |
|
pass |
562 |
+ |
|
563 |
|
if reRB.match( line ): |
564 |
< |
RB =line.split()[3] |
565 |
< |
common.logger.debug(5, 'Using RB :'+RB) |
564 |
> |
RB = reRB.match(line).group() |
565 |
> |
common.logger.debug(5,"RB "+RB) |
566 |
|
pass |
567 |
< |
if reCE.search( line ): |
568 |
< |
next = 1 |
569 |
< |
continue |
570 |
< |
if next: |
571 |
< |
CE=line.split(':')[0] |
572 |
< |
CEs.append(CE) |
573 |
< |
common.logger.debug(5, 'Matched CE :'+CE) |
574 |
< |
Match=Match+1 |
150 |
< |
pass |
151 |
< |
if reNO.match( line ): |
152 |
< |
common.logger.debug(5,line) |
153 |
< |
self.noMatchFound_(jdl) |
154 |
< |
Match=0 |
567 |
> |
|
568 |
> |
if reCEId.search( line ): |
569 |
> |
for lineCE in lines[i:-1]: |
570 |
> |
if reCE.match( lineCE ): |
571 |
> |
CE = string.strip(reCE.search(lineCE).group(1)) |
572 |
> |
CEs.append(CE.split(':')[0]) |
573 |
> |
pass |
574 |
> |
pass |
575 |
|
pass |
576 |
< |
return Match |
576 |
> |
i=i+1 |
577 |
> |
pass |
578 |
> |
|
579 |
> |
common.logger.debug(5,"All CE :"+str(CEs)) |
580 |
> |
|
581 |
> |
sites = [] |
582 |
> |
[sites.append(it) for it in CEs if not sites.count(it)] |
583 |
> |
|
584 |
> |
common.logger.debug(5,"All Sites :"+str(sites)) |
585 |
> |
common.logger.message("Matched Sites :"+str(sites)) |
586 |
> |
return len(sites) |
587 |
|
|
588 |
|
def noMatchFound_(self, jdl): |
589 |
|
reReq = re.compile( r'Requirements' ) |
609 |
|
Submit one EDG job. |
610 |
|
""" |
611 |
|
|
612 |
+ |
self.checkProxy() |
613 |
|
jid = None |
614 |
|
jdl = common.job_list[nj].jdlFilename() |
615 |
< |
id_tmp = tempfile.mktemp() |
616 |
< |
edg_ui_cfg_opt = ' ' |
186 |
< |
if self.edg_config: |
187 |
< |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
188 |
< |
if self.edg_config_vo: |
189 |
< |
edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' ' |
190 |
< |
cmd = 'edg-job-submit -o ' + id_tmp + edg_ui_cfg_opt + jdl |
615 |
> |
|
616 |
> |
cmd = 'edg-job-submit ' + self.configOpt_() + jdl |
617 |
|
cmd_out = runCommand(cmd) |
618 |
|
if cmd_out != None: |
619 |
< |
idfile = open(id_tmp) |
620 |
< |
jid_line = idfile.readline() |
195 |
< |
while jid_line[0] == '#': |
196 |
< |
jid_line = idfile.readline() |
197 |
< |
pass |
198 |
< |
jid = string.strip(jid_line) |
199 |
< |
os.unlink(id_tmp) |
619 |
> |
reSid = re.compile( r'https.+' ) |
620 |
> |
jid = reSid.search(cmd_out).group() |
621 |
|
pass |
622 |
|
return jid |
623 |
|
|
624 |
+ |
def resubmit(self, nj_list): |
625 |
+ |
""" |
626 |
+ |
Prepare jobs to be submit |
627 |
+ |
""" |
628 |
+ |
return |
629 |
+ |
|
630 |
|
def getExitStatus(self, id): |
631 |
|
return self.getStatusAttribute_(id, 'exit_code') |
632 |
|
|
640 |
|
def getStatusAttribute_(self, id, attr): |
641 |
|
""" Query a status of the job with id """ |
642 |
|
|
643 |
+ |
self.checkProxy() |
644 |
|
hstates = {} |
645 |
|
Status = importName('edg_wl_userinterface_common_LbWrapper', 'Status') |
646 |
|
# Bypass edg-job-status interfacing directly to C++ API |
652 |
|
jobStat.getStatus(id, level) |
653 |
|
err, apiMsg = jobStat.get_error() |
654 |
|
if err: |
655 |
< |
print 'Error caught', apiMsg |
228 |
< |
common.log.message(apiMsg) |
655 |
> |
common.logger.debug(5,'Error caught' + apiMsg) |
656 |
|
return None |
657 |
|
else: |
658 |
|
for i in range(len(self.states)): |
232 |
– |
#print "states = ", states |
659 |
|
# Fill an hash table with all information retrieved from LB API |
660 |
|
hstates[ self.states[i] ] = jobStat.loadStatus(st)[i] |
661 |
|
result = jobStat.loadStatus(st)[ self.states.index(attr) ] |
673 |
|
Returns the name of directory with results. |
674 |
|
""" |
675 |
|
|
676 |
+ |
self.checkProxy() |
677 |
|
cmd = 'edg-job-get-output --dir ' + common.work_space.resDir() + ' ' + id |
678 |
|
cmd_out = runCommand(cmd) |
679 |
|
|
680 |
|
# Determine the output directory name |
681 |
|
dir = common.work_space.resDir() |
682 |
< |
dir += os.getlogin() |
682 |
> |
dir += os.environ['USER'] |
683 |
|
dir += '_' + os.path.basename(id) |
684 |
|
return dir |
685 |
|
|
686 |
|
def cancel(self, id): |
687 |
|
""" Cancel the EDG job with id """ |
688 |
+ |
self.checkProxy() |
689 |
|
cmd = 'edg-job-cancel --noint ' + id |
690 |
|
cmd_out = runCommand(cmd) |
691 |
|
return cmd_out |
692 |
|
|
265 |
– |
def checkProxy_(self): |
266 |
– |
""" |
267 |
– |
Function to check the Globus proxy. |
268 |
– |
""" |
269 |
– |
cmd = 'grid-proxy-info -timeleft' |
270 |
– |
cmd_out = runCommand(cmd) |
271 |
– |
ok = 1 |
272 |
– |
timeleft = -999 |
273 |
– |
try: timeleft = int(cmd_out) |
274 |
– |
except ValueError: ok=0 |
275 |
– |
except TypeError: ok=0 |
276 |
– |
if timeleft < 1: ok=0 |
277 |
– |
|
278 |
– |
if ok==0: |
279 |
– |
msg = 'No valid proxy found !\n' |
280 |
– |
msg += "Please do 'grid-proxy-init'." |
281 |
– |
raise CrabException(msg) |
282 |
– |
return |
283 |
– |
|
693 |
|
def createSchScript(self, nj): |
694 |
|
""" |
695 |
|
Create a JDL-file for EDG. |
704 |
|
title = '# This JDL was generated by '+\ |
705 |
|
common.prog_name+' (version '+common.prog_version_str+')\n' |
706 |
|
jt_string = '' |
707 |
+ |
|
708 |
+ |
|
709 |
|
|
710 |
|
SPL = inp_storage_subdir |
711 |
|
if ( SPL and SPL[-1] != '/' ) : SPL = SPL + '/' |
718 |
|
jdl.write('Executable = "' + os.path.basename(script) +'";\n') |
719 |
|
jdl.write(jt_string) |
720 |
|
|
721 |
< |
firstEvent = common.jobDB.firstEvent(nj) |
311 |
< |
maxEvents = common.jobDB.maxEvents(nj) |
312 |
< |
jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n') |
721 |
> |
### only one .sh JDL has arguments: |
722 |
|
|
723 |
+ |
### Fabio |
724 |
+ |
jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n') |
725 |
|
inp_box = 'InputSandbox = { ' |
726 |
|
inp_box = inp_box + '"' + script + '",' |
727 |
|
|
734 |
|
#if common.use_jam: |
735 |
|
# inp_box = inp_box+' "'+common.bin_dir+'/'+common.run_jam+'",' |
736 |
|
|
737 |
< |
for addFile in jbt.additional_inbox_files: |
738 |
< |
addFile = os.path.abspath(addFile) |
739 |
< |
inp_box = inp_box+' "'+addFile+'",' |
740 |
< |
pass |
737 |
> |
# Marco (VERY TEMPORARY ML STUFF) |
738 |
> |
inp_box = inp_box+' "' + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + '", "' +\ |
739 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + '", "'+\ |
740 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + '", "'+\ |
741 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + '", "'+\ |
742 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py') + '"' |
743 |
> |
# End Marco |
744 |
> |
|
745 |
> |
if (not jbt.additional_inbox_files == []): |
746 |
> |
inp_box = inp_box + ', ' |
747 |
> |
for addFile in jbt.additional_inbox_files: |
748 |
> |
addFile = os.path.abspath(addFile) |
749 |
> |
inp_box = inp_box+' "'+addFile+'",' |
750 |
> |
pass |
751 |
|
|
752 |
|
if inp_box[-1] == ',' : inp_box = inp_box[:-1] |
753 |
|
inp_box = inp_box + ' };\n' |
755 |
|
|
756 |
|
jdl.write('StdOutput = "' + job.stdout() + '";\n') |
757 |
|
jdl.write('StdError = "' + job.stderr() + '";\n') |
758 |
< |
|
759 |
< |
#if common.flag_return_data : |
760 |
< |
# for fl in job.outputDataFiles(): |
761 |
< |
# out_box = out_box + ' "' + fl + '",' |
762 |
< |
# pass |
763 |
< |
# pass |
764 |
< |
|
765 |
< |
out_box = 'OutputSandbox = { ' |
766 |
< |
if out_sandbox != None: |
767 |
< |
for fl in out_sandbox: |
768 |
< |
out_box = out_box + ' "' + fl + '",' |
758 |
> |
|
759 |
> |
|
760 |
> |
if job.stdout() == job.stderr(): |
761 |
> |
out_box = 'OutputSandbox = { "' + \ |
762 |
> |
job.stdout() + '", ".BrokerInfo",' |
763 |
> |
else: |
764 |
> |
out_box = 'OutputSandbox = { "' + \ |
765 |
> |
job.stdout() + '", "' + \ |
766 |
> |
job.stderr() + '", ".BrokerInfo",' |
767 |
> |
|
768 |
> |
if int(self.return_data) == 1: |
769 |
> |
if out_sandbox != None: |
770 |
> |
for fl in out_sandbox: |
771 |
> |
out_box = out_box + ' "' + fl + '",' |
772 |
> |
pass |
773 |
|
pass |
774 |
|
pass |
775 |
< |
|
775 |
> |
|
776 |
|
if out_box[-1] == ',' : out_box = out_box[:-1] |
777 |
|
out_box = out_box + ' };' |
778 |
|
jdl.write(out_box+'\n') |
779 |
|
|
355 |
– |
# If CloseCE is used ... |
356 |
– |
#if common.flag_usecloseCE and job.inputDataFiles(): |
357 |
– |
# indata = 'InputData = { ' |
358 |
– |
# for fl in job.inputDataFiles(): |
359 |
– |
# indata = indata + ' "lfn:' + SPL + fl + '",' |
360 |
– |
# if indata[-1] == ',' : indata = indata[:-1] |
361 |
– |
# indata = indata + ' };' |
362 |
– |
# jdl.write(indata+'\n') |
363 |
– |
# jdl.write('DataAccessProtocol = { "gsiftp" };\n') |
364 |
– |
|
365 |
– |
if common.analisys_common_info['sites']: |
366 |
– |
if common.analisys_common_info['sw_version']: |
367 |
– |
|
368 |
– |
req='Requirements = ' |
369 |
– |
### First ORCA version |
370 |
– |
req=req + 'Member("VO-cms-' + \ |
371 |
– |
common.analisys_common_info['sw_version'] + \ |
372 |
– |
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
373 |
– |
## then sites |
374 |
– |
if len(common.analisys_common_info['sites'])>0: |
375 |
– |
req = req + ' && (' |
376 |
– |
for i in range(len(common.analisys_common_info['sites'])): |
377 |
– |
req = req + 'other.GlueCEInfoHostName == "' \ |
378 |
– |
+ common.analisys_common_info['sites'][i] + '"' |
379 |
– |
if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ): |
380 |
– |
req = req + ' || ' |
381 |
– |
req = req + ')' |
382 |
– |
## then user requirement |
383 |
– |
if self.EDG_requirements: |
384 |
– |
req = req + ' && ' + self.EDG_requirements |
385 |
– |
req = req + ';\n' |
386 |
– |
jdl.write(req) |
780 |
|
|
781 |
+ |
req='Requirements = ' |
782 |
+ |
noreq=req |
783 |
+ |
req = req + jbt.getRequirements() |
784 |
+ |
#### and USER REQUIREMENT |
785 |
+ |
if self.EDG_requirements: |
786 |
+ |
if (req != noreq): |
787 |
+ |
req = req + ' && ' |
788 |
+ |
req = req + self.EDG_requirements |
789 |
+ |
#### FEDE ##### |
790 |
+ |
if self.EDG_ce_white_list: |
791 |
+ |
ce_white_list = string.split(self.EDG_ce_white_list,',') |
792 |
+ |
#print "req = ", req |
793 |
+ |
for i in range(len(ce_white_list)): |
794 |
+ |
if i == 0: |
795 |
+ |
if (req != noreq): |
796 |
+ |
req = req + ' && ' |
797 |
+ |
req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
798 |
+ |
pass |
799 |
+ |
else: |
800 |
+ |
req = req + ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
801 |
+ |
req = req + ')' |
802 |
+ |
|
803 |
+ |
if self.EDG_ce_black_list: |
804 |
+ |
ce_black_list = string.split(self.EDG_ce_black_list,',') |
805 |
+ |
for ce in ce_black_list: |
806 |
+ |
if (req != noreq): |
807 |
+ |
req = req + ' && ' |
808 |
+ |
req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))' |
809 |
+ |
pass |
810 |
+ |
|
811 |
+ |
############### |
812 |
+ |
clockTime=480 |
813 |
+ |
if self.EDG_clock_time: |
814 |
+ |
clockTime= self.EDG_clock_time |
815 |
+ |
if (req != noreq): |
816 |
+ |
req = req + ' && ' |
817 |
+ |
req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))' |
818 |
+ |
|
819 |
+ |
cpuTime=1000 |
820 |
+ |
if self.EDG_cpu_time: |
821 |
+ |
cpuTime=self.EDG_cpu_time |
822 |
+ |
if (req != noreq): |
823 |
+ |
req = req + ' && ' |
824 |
+ |
req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))' |
825 |
+ |
|
826 |
+ |
if (req != noreq): |
827 |
+ |
req = req + ';\n' |
828 |
+ |
jdl.write(req) |
829 |
+ |
|
830 |
|
jdl.write('VirtualOrganisation = "' + self.VO + '";\n') |
831 |
|
|
832 |
|
if ( self.EDG_retry_count ): |
833 |
|
jdl.write('RetryCount = '+self.EDG_retry_count+';\n') |
834 |
|
pass |
835 |
|
|
836 |
+ |
jdl.write('MyProxyServer = "' + self.proxyServer + '";\n') |
837 |
+ |
|
838 |
|
jdl.close() |
839 |
|
return |
840 |
+ |
|
841 |
+ |
def checkProxy(self): |
842 |
+ |
""" |
843 |
+ |
Function to check the Globus proxy. |
844 |
+ |
""" |
845 |
+ |
if (self.proxyValid): return |
846 |
+ |
timeleft = -999 |
847 |
+ |
minTimeLeft=10*3600 # in seconds |
848 |
+ |
|
849 |
+ |
minTimeLeftServer = 100 # in hours |
850 |
+ |
|
851 |
+ |
#cmd = 'voms-proxy-info -exists -valid '+str(minTimeLeft)+':00' |
852 |
+ |
#cmd = 'voms-proxy-info -timeleft' |
853 |
+ |
mustRenew = 0 |
854 |
+ |
timeLeftLocal = runCommand('voms-proxy-info -timeleft 2>/dev/null') |
855 |
+ |
timeLeftServer = -999 |
856 |
+ |
if not timeLeftLocal or int(timeLeftLocal) <= 0 or not isInt(timeLeftLocal): |
857 |
+ |
mustRenew = 1 |
858 |
+ |
else: |
859 |
+ |
timeLeftServer = runCommand('voms-proxy-info -actimeleft 2>/dev/null | head -1') |
860 |
+ |
if not timeLeftServer or not isInt(timeLeftServer): |
861 |
+ |
mustRenew = 1 |
862 |
+ |
elif timeLeftLocal<minTimeLeft or timeLeftServer<minTimeLeft: |
863 |
+ |
mustRenew = 1 |
864 |
+ |
pass |
865 |
+ |
pass |
866 |
+ |
|
867 |
+ |
if mustRenew: |
868 |
+ |
common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n") |
869 |
+ |
cmd = 'voms-proxy-init -voms cms -valid 96:00' |
870 |
+ |
try: |
871 |
+ |
# SL as above: damn it! |
872 |
+ |
out = os.system(cmd) |
873 |
+ |
if (out>0): raise CrabException("Unable to create a valid proxy!\n") |
874 |
+ |
except: |
875 |
+ |
msg = "Unable to create a valid proxy!\n" |
876 |
+ |
raise CrabException(msg) |
877 |
+ |
# cmd = 'grid-proxy-info -timeleft' |
878 |
+ |
# cmd_out = runCommand(cmd,0,20) |
879 |
+ |
pass |
880 |
+ |
|
881 |
+ |
## now I do have a voms proxy valid, and I check the myproxy server |
882 |
+ |
renewProxy = 0 |
883 |
+ |
cmd = 'myproxy-info -d -s '+self.proxyServer |
884 |
+ |
cmd_out = runCommand(cmd,0,20) |
885 |
+ |
if not cmd_out: |
886 |
+ |
common.logger.message('No credential delegated to myproxy server '+self.proxyServer+' will do now') |
887 |
+ |
renewProxy = 1 |
888 |
+ |
else: |
889 |
+ |
# if myproxy exist but not long enough, renew |
890 |
+ |
reTime = re.compile( r'timeleft: (\d+)' ) |
891 |
+ |
#print "<"+str(reTime.search( cmd_out ).group(1))+">" |
892 |
+ |
if reTime.match( cmd_out ): |
893 |
+ |
time = reTime.search( line ).group(1) |
894 |
+ |
if time < minTimeLeftServer: |
895 |
+ |
renewProxy = 1 |
896 |
+ |
common.logger.message('No credential delegation will expire in '+time+' hours: renew it') |
897 |
+ |
pass |
898 |
+ |
pass |
899 |
+ |
|
900 |
+ |
# if not, create one. |
901 |
+ |
if renewProxy: |
902 |
+ |
cmd = 'myproxy-init -d -n -s '+self.proxyServer |
903 |
+ |
out = os.system(cmd) |
904 |
+ |
if (out>0): |
905 |
+ |
raise CrabException("Unable to delegate the proxy to myproxyserver "+self.proxyServer+" !\n") |
906 |
+ |
pass |
907 |
+ |
|
908 |
+ |
# cache proxy validity |
909 |
+ |
self.proxyValid=1 |
910 |
+ |
return |
911 |
+ |
|
912 |
+ |
def configOpt_(self): |
913 |
+ |
edg_ui_cfg_opt = ' ' |
914 |
+ |
if self.edg_config: |
915 |
+ |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
916 |
+ |
if self.edg_config_vo: |
917 |
+ |
edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' ' |
918 |
+ |
return edg_ui_cfg_opt |