2 |
|
from crab_logger import Logger |
3 |
|
from crab_exceptions import * |
4 |
|
from crab_util import * |
5 |
+ |
from EdgConfig import * |
6 |
|
import common |
7 |
|
|
8 |
|
import os, sys, time |
22 |
|
|
23 |
|
def configure(self, cfg_params): |
24 |
|
|
25 |
< |
try: self.edg_config = cfg_params["EDG.config"] |
26 |
< |
except KeyError: self.edg_config = '' |
25 |
> |
try: |
26 |
> |
RB = cfg_params["EDG.rb"] |
27 |
> |
edgConfig = EdgConfig(RB) |
28 |
> |
self.edg_config = edgConfig.config() |
29 |
> |
self.edg_config_vo = edgConfig.configVO() |
30 |
> |
except KeyError: |
31 |
> |
self.edg_config = '' |
32 |
> |
self.edg_config_vo = '' |
33 |
|
|
34 |
< |
try: self.edg_config_vo = cfg_params["EDG.config_vo"] |
35 |
< |
except KeyError: self.edg_config_vo = '' |
34 |
> |
try: |
35 |
> |
self.proxyServer = cfg_params["EDG.proxy_server"] |
36 |
> |
except KeyError: |
37 |
> |
self.proxyServer = 'myproxy.cern.ch' |
38 |
> |
common.logger.debug(5,'Setting myproxy server to '+self.proxyServer) |
39 |
|
|
40 |
|
try: self.LCG_version = cfg_params["EDG.lcg_version"] |
41 |
|
except KeyError: self.LCG_version = '2' |
46 |
|
try: self.EDG_retry_count = cfg_params['EDG.retry_count'] |
47 |
|
except KeyError: self.EDG_retry_count = '' |
48 |
|
|
49 |
+ |
try: |
50 |
+ |
self.EDG_ce_black_list = cfg_params['EDG.ce_black_list'] |
51 |
+ |
#print "self.EDG_ce_black_list = ", self.EDG_ce_black_list |
52 |
+ |
except KeyError: |
53 |
+ |
self.EDG_ce_black_list = '' |
54 |
+ |
|
55 |
+ |
try: |
56 |
+ |
self.EDG_ce_white_list = cfg_params['EDG.ce_white_list'] |
57 |
+ |
#print "self.EDG_ce_white_list = ", self.EDG_ce_white_list |
58 |
+ |
except KeyError: self.EDG_ce_white_list = '' |
59 |
+ |
|
60 |
|
try: self.VO = cfg_params['EDG.virtual_organization'] |
61 |
|
except KeyError: self.VO = 'cms' |
62 |
|
|
63 |
|
try: self.return_data = cfg_params['USER.return_data'] |
64 |
< |
except KeyError: self.return_data = '' |
64 |
> |
except KeyError: self.return_data = 1 |
65 |
> |
|
66 |
> |
try: |
67 |
> |
self.copy_input_data = common.analisys_common_info['copy_input_data'] |
68 |
> |
#print "self.copy_input_data = ", self.copy_input_data |
69 |
> |
except KeyError: self.copy_input_data = 0 |
70 |
|
|
71 |
|
try: |
72 |
|
self.copy_data = cfg_params["USER.copy_data"] |
73 |
< |
try: |
74 |
< |
self.SE = cfg_params['USER.storage_element'] |
75 |
< |
self.SE_PATH = cfg_params['USER.storage_path'] |
76 |
< |
except KeyError: |
77 |
< |
msg = "Error. The [USER] section does not have 'storage_element'" |
78 |
< |
msg = msg + " and/or 'storage_path' entries, necessary to copy the output" |
79 |
< |
common.logger.message(msg) |
80 |
< |
raise CrabException(msg) |
81 |
< |
except KeyError: self.copy_data = '' |
73 |
> |
if int(self.copy_data) == 1: |
74 |
> |
try: |
75 |
> |
self.SE = cfg_params['USER.storage_element'] |
76 |
> |
self.SE_PATH = cfg_params['USER.storage_path'] |
77 |
> |
except KeyError: |
78 |
> |
msg = "Error. The [USER] section does not have 'storage_element'" |
79 |
> |
msg = msg + " and/or 'storage_path' entries, necessary to copy the output" |
80 |
> |
common.logger.message(msg) |
81 |
> |
raise CrabException(msg) |
82 |
> |
except KeyError: self.copy_data = 0 |
83 |
> |
|
84 |
> |
if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ): |
85 |
> |
msg = 'Warning: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n' |
86 |
> |
msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n' |
87 |
> |
raise CrabException(msg) |
88 |
|
|
89 |
+ |
try: |
90 |
+ |
self.lfc_host = cfg_params['EDG.lfc_host'] |
91 |
+ |
except KeyError: |
92 |
+ |
msg = "Error. The [EDG] section does not have 'lfc_host' value" |
93 |
+ |
msg = msg + " it's necessary to know the LFC host name" |
94 |
+ |
common.logger.message(msg) |
95 |
+ |
raise CrabException(msg) |
96 |
+ |
try: |
97 |
+ |
self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type'] |
98 |
+ |
except KeyError: |
99 |
+ |
msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value" |
100 |
+ |
msg = msg + " it's necessary to know the catalog type" |
101 |
+ |
common.logger.message(msg) |
102 |
+ |
raise CrabException(msg) |
103 |
+ |
try: |
104 |
+ |
self.lfc_home = cfg_params['EDG.lfc_home'] |
105 |
+ |
except KeyError: |
106 |
+ |
msg = "Error. The [EDG] section does not have 'lfc_home' value" |
107 |
+ |
msg = msg + " it's necessary to know the home catalog dir" |
108 |
+ |
common.logger.message(msg) |
109 |
+ |
raise CrabException(msg) |
110 |
+ |
|
111 |
|
try: |
112 |
|
self.register_data = cfg_params["USER.register_data"] |
113 |
< |
try: |
114 |
< |
self.LFN = cfg_params['USER.lfn_dir'] |
115 |
< |
except KeyError: |
116 |
< |
msg = "Error. The [USER] section does not have 'lfn_dir' value" |
117 |
< |
msg = msg + " it's necessary for RLS registration" |
118 |
< |
common.logger.message(msg) |
119 |
< |
raise CrabException(msg) |
120 |
< |
except KeyError: self.register_data= '' |
113 |
> |
if int(self.register_data) == 1: |
114 |
> |
try: |
115 |
> |
self.LFN = cfg_params['USER.lfn_dir'] |
116 |
> |
except KeyError: |
117 |
> |
msg = "Error. The [USER] section does not have 'lfn_dir' value" |
118 |
> |
msg = msg + " it's necessary for LCF registration" |
119 |
> |
common.logger.message(msg) |
120 |
> |
raise CrabException(msg) |
121 |
> |
except KeyError: self.register_data = 0 |
122 |
> |
|
123 |
> |
if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ): |
124 |
> |
msg = 'Warning: register_data = 1 must be used with copy_data = 1\n' |
125 |
> |
msg = msg + 'Please modify copy_data value in your crab.cfg file\n' |
126 |
> |
common.logger.message(msg) |
127 |
> |
raise CrabException(msg) |
128 |
|
|
129 |
|
try: self.EDG_requirements = cfg_params['EDG.requirements'] |
130 |
|
except KeyError: self.EDG_requirements = '' |
152 |
|
sys.path.append(libPath) |
153 |
|
|
154 |
|
self.proxyValid=0 |
155 |
+ |
|
156 |
+ |
try: |
157 |
+ |
self._taskId = cfg_params['taskId'] |
158 |
+ |
except: |
159 |
+ |
self._taskId = '' |
160 |
+ |
|
161 |
|
return |
162 |
|
|
163 |
|
|
180 |
|
""" |
181 |
|
Returns part of a job script which does scheduler-specific work. |
182 |
|
""" |
116 |
– |
|
183 |
|
txt = '' |
184 |
< |
if self.copy_data: |
184 |
> |
txt += '# strip arguments\n' |
185 |
> |
txt += 'echo "strip arguments"\n' |
186 |
> |
txt += 'args=("$@")\n' |
187 |
> |
txt += 'nargs=$#\n' |
188 |
> |
txt += 'shift $nargs\n' |
189 |
> |
txt += "# job number (first parameter for job wrapper)\n" |
190 |
> |
#txt += "NJob=$1\n" |
191 |
> |
txt += "NJob=${args[0]}\n" |
192 |
> |
|
193 |
> |
txt += '# job identification to DashBoard \n' |
194 |
> |
txt += 'MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`\n' |
195 |
> |
txt += 'SyncGridJobId=`echo $EDG_WL_JOBID`\n' |
196 |
> |
txt += 'MonitorID=`echo ' + self._taskId + '`\n' |
197 |
> |
txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
198 |
> |
txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n' |
199 |
> |
txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
200 |
> |
|
201 |
> |
txt += 'echo "middleware discovery " \n' |
202 |
> |
txt += 'if [ $VO_CMS_SW_DIR ]; then \n' |
203 |
> |
txt += ' middleware=LCG \n' |
204 |
> |
txt += ' echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n' |
205 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
206 |
> |
txt += ' echo "middleware =$middleware" \n' |
207 |
> |
txt += 'elif [ $GRID3_APP_DIR ]; then\n' |
208 |
> |
txt += ' middleware=OSG \n' |
209 |
> |
txt += ' echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n' |
210 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
211 |
> |
txt += ' echo "middleware =$middleware" \n' |
212 |
> |
txt += 'elif [ $OSG_APP ]; then \n' |
213 |
> |
txt += ' middleware=OSG \n' |
214 |
> |
txt += ' echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n' |
215 |
> |
txt += ' echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n' |
216 |
> |
txt += ' echo "middleware =$middleware" \n' |
217 |
> |
txt += 'else \n' |
218 |
> |
txt += ' echo "SET_CMS_ENV 10030 ==> middleware not identified" \n' |
219 |
> |
txt += ' echo "JOB_EXIT_STATUS = 10030" \n' |
220 |
> |
txt += ' echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n' |
221 |
> |
txt += ' dumpStatus $RUNTIME_AREA/$repo \n' |
222 |
> |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
223 |
> |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
224 |
> |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
225 |
> |
txt += ' exit 1 \n' |
226 |
> |
txt += 'fi \n' |
227 |
> |
|
228 |
> |
txt += '# report first time to DashBoard \n' |
229 |
> |
txt += 'dumpStatus $RUNTIME_AREA/$repo \n' |
230 |
> |
txt += 'rm -f $RUNTIME_AREA/$repo \n' |
231 |
> |
txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
232 |
> |
txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
233 |
> |
|
234 |
> |
txt += '\n\n' |
235 |
> |
|
236 |
> |
if int(self.copy_data) == 1: |
237 |
|
if self.SE: |
238 |
|
txt += 'export SE='+self.SE+'\n' |
239 |
|
txt += 'echo "SE = $SE"\n' |
241 |
|
if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/' |
242 |
|
txt += 'export SE_PATH='+self.SE_PATH+'\n' |
243 |
|
txt += 'echo "SE_PATH = $SE_PATH"\n' |
244 |
< |
|
245 |
< |
if self.register_data: |
244 |
> |
|
245 |
> |
txt += 'export VO='+self.VO+'\n' |
246 |
> |
### FEDE: add some line for LFC catalog setting |
247 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
248 |
> |
txt += ' if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n' |
249 |
> |
txt += ' export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n' |
250 |
> |
txt += ' fi\n' |
251 |
> |
txt += ' if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n' |
252 |
> |
txt += ' export LFC_HOST='+self.lfc_host+'\n' |
253 |
> |
txt += ' fi\n' |
254 |
> |
txt += ' if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n' |
255 |
> |
txt += ' export LFC_HOME='+self.lfc_home+'\n' |
256 |
> |
txt += ' fi\n' |
257 |
> |
txt += 'elif [ $middleware == OSG ]; then\n' |
258 |
> |
txt += ' echo "LFC catalog setting to be implemented for OSG"\n' |
259 |
> |
txt += 'fi\n' |
260 |
> |
##### |
261 |
> |
if int(self.register_data) == 1: |
262 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
263 |
> |
txt += ' export LFN='+self.LFN+'\n' |
264 |
> |
txt += ' lfc-ls $LFN\n' |
265 |
> |
txt += ' result=$?\n' |
266 |
> |
txt += ' echo $result\n' |
267 |
> |
### creation of LFN dir in LFC catalog, under /grid/cms dir |
268 |
> |
txt += ' if [ $result != 0 ]; then\n' |
269 |
> |
txt += ' lfc-mkdir $LFN\n' |
270 |
> |
txt += ' result=$?\n' |
271 |
> |
txt += ' echo $result\n' |
272 |
> |
txt += ' fi\n' |
273 |
> |
txt += 'elif [ $middleware == OSG ]; then\n' |
274 |
> |
txt += ' echo " Files registration to be implemented for OSG"\n' |
275 |
> |
txt += 'fi\n' |
276 |
> |
txt += '\n' |
277 |
> |
|
278 |
|
if self.VO: |
279 |
|
txt += 'export VO='+self.VO+'\n' |
280 |
|
if self.LFN: |
281 |
< |
txt += 'export LFN='+self.LFN+'\n' |
281 |
> |
txt += 'if [ $middleware == LCG ]; then \n' |
282 |
> |
txt += ' export LFN='+self.LFN+'\n' |
283 |
> |
txt += 'fi\n' |
284 |
|
txt += '\n' |
285 |
< |
txt += 'CloseCEs=`edg-brokerinfo getCE`\n' |
286 |
< |
txt += 'echo "CloseCEs = $CloseCEs"\n' |
287 |
< |
txt += 'CE=`echo $CloseCEs | sed -e "s/:.*//"`\n' |
288 |
< |
txt += 'echo "CE = $CE"\n' |
285 |
> |
|
286 |
> |
txt += 'if [ $middleware == LCG ]; then\n' |
287 |
> |
txt += ' CloseCEs=`edg-brokerinfo getCE`\n' |
288 |
> |
txt += ' echo "CloseCEs = $CloseCEs"\n' |
289 |
> |
txt += ' CE=`echo $CloseCEs | sed -e "s/:.*//"`\n' |
290 |
> |
txt += ' echo "CE = $CE"\n' |
291 |
> |
txt += 'elif [ $middleware == OSG ]; then \n' |
292 |
> |
txt += ' if [ $OSG_JOB_CONTACT ]; then \n' |
293 |
> |
# txt += ' CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n' |
294 |
> |
txt += ' CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print ${args[0]}}\'` \n' |
295 |
> |
txt += ' else \n' |
296 |
> |
txt += ' echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n' |
297 |
> |
txt += ' echo "JOB_EXIT_STATUS = 10099" \n' |
298 |
> |
txt += ' echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n' |
299 |
> |
txt += ' dumpStatus $RUNTIME_AREA/$repo \n' |
300 |
> |
txt += ' rm -f $RUNTIME_AREA/$repo \n' |
301 |
> |
txt += ' echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n' |
302 |
> |
txt += ' echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n' |
303 |
> |
txt += ' exit 1 \n' |
304 |
> |
txt += ' fi \n' |
305 |
> |
txt += 'fi \n' |
306 |
> |
|
307 |
> |
return txt |
308 |
> |
|
309 |
> |
def wsCopyInput(self): |
310 |
> |
""" |
311 |
> |
Copy input data from SE to WN |
312 |
> |
""" |
313 |
> |
txt = '' |
314 |
> |
try: |
315 |
> |
self.copy_input_data = common.analisys_common_info['copy_input_data'] |
316 |
> |
#print "self.copy_input_data = ", self.copy_input_data |
317 |
> |
except KeyError: self.copy_input_data = 0 |
318 |
> |
if int(self.copy_input_data) == 1: |
319 |
> |
## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG) |
320 |
> |
txt += 'if [ $middleware == OSG ]; then\n' |
321 |
> |
txt += ' #\n' |
322 |
> |
txt += ' # Copy Input Data from SE to this WN deactivated in OSG mode\n' |
323 |
> |
txt += ' #\n' |
324 |
> |
txt += ' echo "Copy Input Data from SE to this WN deactivated in OSG mode"\n' |
325 |
> |
txt += 'elif [ $middleware == LCG ]; then \n' |
326 |
> |
txt += ' #\n' |
327 |
> |
txt += ' # Copy Input Data from SE to this WN\n' |
328 |
> |
txt += ' #\n' |
329 |
> |
### changed by georgia (put a loop copying more than one input files per jobs) |
330 |
> |
txt += ' for input_file in $cur_file_list \n' |
331 |
> |
txt += ' do \n' |
332 |
> |
#### FEDE |
333 |
> |
#txt += ' echo "which lcg-cp" \n' |
334 |
> |
#txt += ' which lcg-cp \n' |
335 |
> |
######### |
336 |
> |
txt += ' lcg-cp --vo $VO --verbose -t 1200 lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n' |
337 |
> |
txt += ' copy_input_exit_status=$?\n' |
338 |
> |
txt += ' echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n' |
339 |
> |
txt += ' if [ $copy_input_exit_status -ne 0 ]; then \n' |
340 |
> |
txt += ' echo "Problems with copying to WN" \n' |
341 |
> |
txt += ' else \n' |
342 |
> |
txt += ' echo "input copied into WN" \n' |
343 |
> |
txt += ' fi \n' |
344 |
> |
txt += ' done \n' |
345 |
> |
### copy a set of PU ntuples (same for each jobs -- but accessed randomly) |
346 |
> |
txt += ' for file in $cur_pu_list \n' |
347 |
> |
txt += ' do \n' |
348 |
> |
#### FEDE |
349 |
> |
#txt += ' echo "which lcg-cp" \n' |
350 |
> |
#txt += ' which lcg-cp \n' |
351 |
> |
######### |
352 |
> |
txt += ' lcg-cp --vo $VO --verbose -t 1200 lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n' |
353 |
> |
txt += ' copy_input_pu_exit_status=$?\n' |
354 |
> |
txt += ' echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n' |
355 |
> |
txt += ' if [ $copy_input_pu_exit_status -ne 0 ]; then \n' |
356 |
> |
txt += ' echo "Problems with copying pu to WN" \n' |
357 |
> |
txt += ' else \n' |
358 |
> |
txt += ' echo "input pu files copied into WN" \n' |
359 |
> |
txt += ' fi \n' |
360 |
> |
txt += ' done \n' |
361 |
> |
txt += ' \n' |
362 |
> |
txt += ' ### Check SCRATCH space available on WN : \n' |
363 |
> |
txt += ' df -h \n' |
364 |
> |
txt += 'fi \n' |
365 |
> |
|
366 |
|
return txt |
367 |
|
|
368 |
|
def wsCopyOutput(self): |
371 |
|
to copy produced output into a storage element. |
372 |
|
""" |
373 |
|
txt = '' |
374 |
< |
if self.copy_data: |
146 |
< |
copy = 'globus-url-copy file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file' |
374 |
> |
if int(self.copy_data) == 1: |
375 |
|
txt += '#\n' |
376 |
|
txt += '# Copy output to SE = $SE\n' |
377 |
|
txt += '#\n' |
378 |
< |
txt += 'if [ $exe_result -eq 0 ]; then\n' |
379 |
< |
txt += ' for out_file in $file_list ; do\n' |
380 |
< |
txt += ' echo "Trying to copy output file to $SE "\n' |
381 |
< |
txt += ' echo "'+copy+'"\n' |
382 |
< |
txt += ' '+copy+' 2>&1\n' |
383 |
< |
txt += ' copy_exit_status=$?\n' |
384 |
< |
txt += ' echo "COPY_EXIT_STATUS = $copy_exit_status"\n' |
157 |
< |
txt += ' echo "STAGE_OUT = $copy_exit_status"\n' |
158 |
< |
txt += ' if [ $copy_exit_status -ne 0 ]; then \n' |
159 |
< |
txt += ' echo "Problems with SE= $SE" \n' |
160 |
< |
txt += ' else \n' |
161 |
< |
txt += ' echo "output copied into $SE/$SE_PATH directory"\n' |
378 |
> |
#txt += 'if [ $exe_result -eq 0 ]; then\n' |
379 |
> |
txt += ' if [ $middleware == OSG ]; then\n' |
380 |
> |
txt += ' echo "X509_USER_PROXY = $X509_USER_PROXY"\n' |
381 |
> |
txt += ' echo "source $OSG_APP/glite/setup_glite_ui.sh"\n' |
382 |
> |
txt += ' source $OSG_APP/glite/setup_glite_ui.sh\n' |
383 |
> |
txt += ' export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n' |
384 |
> |
txt += ' echo "export X509_CERT_DIR=$X509_CERT_DIR"\n' |
385 |
|
txt += ' fi \n' |
386 |
< |
txt += ' done\n' |
387 |
< |
txt += 'fi \n' |
386 |
> |
txt += ' for out_file in $file_list ; do\n' |
387 |
> |
txt += ' echo "Trying to copy output file to $SE using lcg-cp"\n' |
388 |
> |
if common.logger.debugLevel() >= 5: |
389 |
> |
txt += ' echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n' |
390 |
> |
txt += ' exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n' |
391 |
> |
else: |
392 |
> |
txt += ' echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n' |
393 |
> |
txt += ' exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n' |
394 |
> |
txt += ' copy_exit_status=$?\n' |
395 |
> |
txt += ' echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n' |
396 |
> |
txt += ' echo "STAGE_OUT = $copy_exit_status"\n' |
397 |
> |
txt += ' if [ $copy_exit_status -ne 0 ]; then\n' |
398 |
> |
txt += ' echo "Possible problem with SE = $SE"\n' |
399 |
> |
txt += ' echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n' |
400 |
> |
txt += ' echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n' |
401 |
> |
txt += ' echo "lcg-cp failed. For verbose lcg-cp output, use command line option -debug 5."\n' |
402 |
> |
txt += ' echo "lcg-cp failed, attempting srmcp"\n' |
403 |
> |
txt += ' echo "mkdir -p $HOME/.srmconfig"\n' |
404 |
> |
txt += ' mkdir -p $HOME/.srmconfig\n' |
405 |
> |
txt += ' if [ $middleware == LCG ]; then\n' |
406 |
> |
txt += ' echo "srmcp -retry_num 5 -retry_timeout 480000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n' |
407 |
> |
txt += ' exitstring=`srmcp -retry_num 5 -retry_timeout 480000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n' |
408 |
> |
txt += ' elif [ $middleware == OSG ]; then\n' |
409 |
> |
txt += ' echo "srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n' |
410 |
> |
txt += ' exitstring=`srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n' |
411 |
> |
txt += ' fi \n' |
412 |
> |
txt += ' copy_exit_status=$?\n' |
413 |
> |
txt += ' echo "COPY_EXIT_STATUS for srm = $copy_exit_status"\n' |
414 |
> |
txt += ' echo "STAGE_OUT = $copy_exit_status"\n' |
415 |
> |
txt += ' if [ $copy_exit_status -ne 0 ]; then\n' |
416 |
> |
txt += ' echo "Problems with SE = $SE"\n' |
417 |
> |
txt += ' echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n' |
418 |
> |
txt += ' echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n' |
419 |
> |
txt += ' echo "lcg-cp and srm failed"\n' |
420 |
> |
txt += ' echo "If storage_path in your config file contains a ? you may need a \? instead."\n' |
421 |
> |
txt += ' else\n' |
422 |
> |
txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n' |
423 |
> |
txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n' |
424 |
> |
txt += ' echo "output copied into $SE/$SE_PATH directory"\n' |
425 |
> |
txt += ' echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n' |
426 |
> |
txt += ' echo "srmcp succeeded"\n' |
427 |
> |
txt += ' fi\n' |
428 |
> |
txt += ' else\n' |
429 |
> |
txt += ' echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n' |
430 |
> |
txt += ' echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n' |
431 |
> |
txt += ' echo "output copied into $SE/$SE_PATH directory"\n' |
432 |
> |
txt += ' echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n' |
433 |
> |
txt += ' echo "lcg-cp succeeded"\n' |
434 |
> |
txt += ' fi\n' |
435 |
> |
txt += ' done\n' |
436 |
> |
#txt += 'fi\n' |
437 |
|
return txt |
438 |
|
|
439 |
|
def wsRegisterOutput(self): |
442 |
|
""" |
443 |
|
|
444 |
|
txt = '' |
445 |
< |
if self.register_data: |
445 |
> |
if int(self.register_data) == 1: |
446 |
> |
## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG) |
447 |
> |
txt += 'if [ $middleware == OSG ]; then\n' |
448 |
> |
txt += ' #\n' |
449 |
> |
txt += ' # Register output to LFC deactivated in OSG mode\n' |
450 |
> |
txt += ' #\n' |
451 |
> |
txt += ' echo "Register output to LFC deactivated in OSG mode"\n' |
452 |
> |
txt += 'elif [ $middleware == LCG ]; then \n' |
453 |
|
txt += '#\n' |
454 |
< |
txt += '# Register output to RLS\n' |
454 |
> |
txt += '# Register output to LFC\n' |
455 |
|
txt += '#\n' |
456 |
< |
txt += 'if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n' |
457 |
< |
txt += ' for out_file in $file_list ; do\n' |
458 |
< |
txt += ' echo "Trying to register the output file into RLS"\n' |
459 |
< |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO sfn://$SE$SE_PATH/$out_file"\n' |
460 |
< |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO sfn://$SE$SE_PATH/$out_file 2>&1 \n' |
461 |
< |
txt += ' register_exit_status=$?\n' |
462 |
< |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
463 |
< |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
464 |
< |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
465 |
< |
txt += ' echo "Problems with the registration to RLS" \n' |
187 |
< |
txt += ' echo "Try with srm protocol" \n' |
188 |
< |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO srm://$SE$SE_PATH/$out_file"\n' |
189 |
< |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO srm://$SE$SE_PATH/$out_file 2>&1 \n' |
456 |
> |
#txt += ' if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n' |
457 |
> |
txt += ' if [ $copy_exit_status -eq 0 ]; then\n' |
458 |
> |
txt += ' for out_file in $file_list ; do\n' |
459 |
> |
txt += ' echo "Trying to register the output file into LFC"\n' |
460 |
> |
#### FEDE |
461 |
> |
#txt += ' echo "which lcg-rf" \n' |
462 |
> |
#txt += ' which lcg-rf \n' |
463 |
> |
######### |
464 |
> |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n' |
465 |
> |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n' |
466 |
|
txt += ' register_exit_status=$?\n' |
467 |
|
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
468 |
|
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
469 |
|
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
470 |
< |
txt += ' echo "Problems with the registration into RLS" \n' |
470 |
> |
txt += ' echo "Problems with the registration to LFC" \n' |
471 |
> |
txt += ' echo "Try with srm protocol" \n' |
472 |
> |
#### FEDE |
473 |
> |
#txt += ' echo "which lcg-rf" \n' |
474 |
> |
#txt += ' which lcg-rf \n' |
475 |
> |
######### |
476 |
> |
txt += ' echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n' |
477 |
> |
txt += ' lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n' |
478 |
> |
txt += ' register_exit_status=$?\n' |
479 |
> |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
480 |
> |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
481 |
> |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
482 |
> |
txt += ' echo "Problems with the registration into LFC" \n' |
483 |
> |
txt += ' fi \n' |
484 |
> |
txt += ' else \n' |
485 |
> |
txt += ' echo "output registered to LFC"\n' |
486 |
|
txt += ' fi \n' |
487 |
< |
txt += ' else \n' |
488 |
< |
txt += ' echo "output registered to RLS"\n' |
489 |
< |
txt += ' fi \n' |
490 |
< |
txt += ' done\n' |
491 |
< |
txt += 'elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n' |
492 |
< |
txt += ' echo "Trying to copy output file to CloseSE"\n' |
493 |
< |
txt += ' CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n' |
494 |
< |
txt += ' for out_file in $file_list ; do\n' |
495 |
< |
txt += ' echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://`pwd`/$out_file" \n' |
496 |
< |
txt += ' lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://`pwd`/$out_file 2>&1 \n' |
497 |
< |
txt += ' register_exit_status=$?\n' |
498 |
< |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
499 |
< |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
500 |
< |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
501 |
< |
txt += ' echo "Problems with CloseSE" \n' |
502 |
< |
txt += ' else \n' |
503 |
< |
txt += ' echo "The program was successfully executed"\n' |
504 |
< |
txt += ' echo "SE = $CLOSE_SE"\n' |
505 |
< |
txt += ' echo "LFN for the file is LFN=${LFN}/$out_file"\n' |
506 |
< |
txt += ' fi \n' |
507 |
< |
txt += ' done\n' |
508 |
< |
txt += 'else\n' |
509 |
< |
txt += ' echo "Problem with the executable"\n' |
487 |
> |
txt += ' echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
488 |
> |
txt += ' done\n' |
489 |
> |
#txt += ' elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n' |
490 |
> |
txt += ' else \n' |
491 |
> |
txt += ' echo "Trying to copy output file to CloseSE"\n' |
492 |
> |
txt += ' CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n' |
493 |
> |
txt += ' for out_file in $file_list ; do\n' |
494 |
> |
#### FEDE |
495 |
> |
#txt += ' echo "which lcg-cr" \n' |
496 |
> |
#txt += ' which lcg-cr \n' |
497 |
> |
######### |
498 |
> |
txt += ' echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n' |
499 |
> |
txt += ' lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n' |
500 |
> |
txt += ' register_exit_status=$?\n' |
501 |
> |
txt += ' echo "REGISTER_EXIT_STATUS = $register_exit_status"\n' |
502 |
> |
txt += ' echo "STAGE_OUT = $register_exit_status"\n' |
503 |
> |
txt += ' if [ $register_exit_status -ne 0 ]; then \n' |
504 |
> |
txt += ' echo "Problems with CloseSE or Catalog" \n' |
505 |
> |
txt += ' else \n' |
506 |
> |
txt += ' echo "The program was successfully executed"\n' |
507 |
> |
txt += ' echo "SE = $CLOSE_SE"\n' |
508 |
> |
txt += ' echo "LFN for the file is LFN=${LFN}/$out_file"\n' |
509 |
> |
txt += ' fi \n' |
510 |
> |
txt += ' echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n' |
511 |
> |
txt += ' done\n' |
512 |
> |
#txt += ' else\n' |
513 |
> |
#txt += ' echo "Problem with the executable"\n' |
514 |
> |
txt += ' fi \n' |
515 |
> |
txt += ' exit_status=$register_exit_status\n' |
516 |
|
txt += 'fi \n' |
517 |
|
return txt |
518 |
|
|
521 |
|
retrieve the logging info from logging and bookkeeping and return it |
522 |
|
""" |
523 |
|
self.checkProxy() |
227 |
– |
# id = common.jobDB.jobId(nj) |
524 |
|
cmd = 'edg-job-get-logging-info -v 2 ' + id |
525 |
< |
cmd_out = os.popen(cmd) |
526 |
< |
# cmd_out = runCommand(cmd) |
525 |
> |
#cmd_out = os.popen(cmd) |
526 |
> |
cmd_out = runCommand(cmd) |
527 |
|
return cmd_out |
528 |
|
|
529 |
|
def listMatch(self, nj): |
533 |
|
self.checkProxy() |
534 |
|
jdl = common.job_list[nj].jdlFilename() |
535 |
|
cmd = 'edg-job-list-match ' + self.configOpt_() + jdl |
536 |
< |
myCmd = os.popen(cmd) |
537 |
< |
cmd_out = myCmd.readlines() |
538 |
< |
myCmd.close() |
536 |
> |
cmd_out = runCommand(cmd,0,45) |
537 |
> |
if not cmd_out: |
538 |
> |
raise CrabException("ERROR: "+cmd+" failed!") |
539 |
> |
|
540 |
|
return self.parseListMatch_(cmd_out, jdl) |
541 |
|
|
542 |
|
def parseListMatch_(self, out, jdl): |
543 |
< |
|
543 |
> |
""" |
544 |
> |
Parse the f* output of edg-list-match and produce something sensible |
545 |
> |
""" |
546 |
|
reComment = re.compile( r'^\**$' ) |
547 |
|
reEmptyLine = re.compile( r'^$' ) |
548 |
|
reVO = re.compile( r'Selected Virtual Organisation name.*' ) |
549 |
< |
reCE = re.compile( r'CEId' ) |
549 |
> |
reLine = re.compile( r'.*') |
550 |
> |
reCE = re.compile( r'(.*:.*)') |
551 |
> |
reCEId = re.compile( r'CEId.*') |
552 |
|
reNO = re.compile( r'No Computing Element matching' ) |
553 |
|
reRB = re.compile( r'Connecting to host' ) |
554 |
|
next = 0 |
555 |
|
CEs=[] |
556 |
|
Match=0 |
557 |
|
|
558 |
< |
for line in out: |
559 |
< |
line = line.strip() |
560 |
< |
if reComment.match( line ): |
561 |
< |
next = 0 |
562 |
< |
continue |
563 |
< |
if reEmptyLine.match(line): |
564 |
< |
continue |
558 |
> |
#print out |
559 |
> |
lines = reLine.findall(out) |
560 |
> |
|
561 |
> |
i=0 |
562 |
> |
CEs=[] |
563 |
> |
for line in lines: |
564 |
> |
string.strip(line) |
565 |
> |
#print line |
566 |
> |
if reNO.match( line ): |
567 |
> |
common.logger.debug(5,line) |
568 |
> |
return 0 |
569 |
> |
pass |
570 |
|
if reVO.match( line ): |
571 |
< |
VO =line.split()[-1] |
572 |
< |
common.logger.debug(5, 'VO :'+VO) |
571 |
> |
VO =reVO.match( line ).group() |
572 |
> |
common.logger.debug(5,"VO "+VO) |
573 |
|
pass |
574 |
+ |
|
575 |
|
if reRB.match( line ): |
576 |
< |
RB =line.split()[3] |
577 |
< |
common.logger.debug(5, 'Using RB :'+RB) |
576 |
> |
RB = reRB.match(line).group() |
577 |
> |
common.logger.debug(5,"RB "+RB) |
578 |
|
pass |
579 |
< |
if reCE.search( line ): |
580 |
< |
next = 1 |
581 |
< |
continue |
582 |
< |
if next: |
583 |
< |
CE=line.split(':')[0] |
584 |
< |
if (CEs.count(CE) > 0): |
585 |
< |
pass |
586 |
< |
else: |
280 |
< |
CEs.append(CE) |
281 |
< |
Match=Match+1 |
282 |
< |
common.logger.debug(5, 'Matched CE :'+CE) |
283 |
< |
pass |
284 |
< |
if reNO.match( line ): |
285 |
< |
common.logger.debug(5,line) |
286 |
< |
self.noMatchFound_(jdl) |
287 |
< |
Match=0 |
579 |
> |
|
580 |
> |
if reCEId.search( line ): |
581 |
> |
for lineCE in lines[i:-1]: |
582 |
> |
if reCE.match( lineCE ): |
583 |
> |
CE = string.strip(reCE.search(lineCE).group(1)) |
584 |
> |
CEs.append(CE.split(':')[0]) |
585 |
> |
pass |
586 |
> |
pass |
587 |
|
pass |
588 |
< |
return Match |
588 |
> |
i=i+1 |
589 |
> |
pass |
590 |
> |
|
591 |
> |
common.logger.debug(5,"All CE :"+str(CEs)) |
592 |
> |
|
593 |
> |
sites = [] |
594 |
> |
[sites.append(it) for it in CEs if not sites.count(it)] |
595 |
> |
|
596 |
> |
common.logger.debug(5,"All Sites :"+str(sites)) |
597 |
> |
common.logger.message("Matched Sites :"+str(sites)) |
598 |
> |
return len(sites) |
599 |
|
|
600 |
|
def noMatchFound_(self, jdl): |
601 |
|
reReq = re.compile( r'Requirements' ) |
633 |
|
pass |
634 |
|
return jid |
635 |
|
|
636 |
+ |
def resubmit(self, nj_list): |
637 |
+ |
""" |
638 |
+ |
Prepare jobs to be submit |
639 |
+ |
""" |
640 |
+ |
return |
641 |
+ |
|
642 |
|
def getExitStatus(self, id): |
643 |
|
return self.getStatusAttribute_(id, 'exit_code') |
644 |
|
|
664 |
|
jobStat.getStatus(id, level) |
665 |
|
err, apiMsg = jobStat.get_error() |
666 |
|
if err: |
352 |
– |
#print 'Error caught', apiMsg |
353 |
– |
#common.log.message(apiMsg) |
667 |
|
common.logger.debug(5,'Error caught' + apiMsg) |
668 |
|
return None |
669 |
|
else: |
691 |
|
|
692 |
|
# Determine the output directory name |
693 |
|
dir = common.work_space.resDir() |
694 |
< |
dir += os.getlogin() |
694 |
> |
dir += os.environ['USER'] |
695 |
|
dir += '_' + os.path.basename(id) |
696 |
|
return dir |
697 |
|
|
731 |
|
jdl.write(jt_string) |
732 |
|
|
733 |
|
### only one .sh JDL has arguments: |
421 |
– |
firstEvent = common.jobDB.firstEvent(nj) |
422 |
– |
maxEvents = common.jobDB.maxEvents(nj) |
423 |
– |
jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n') |
734 |
|
|
735 |
+ |
### Fabio |
736 |
+ |
jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n') |
737 |
|
inp_box = 'InputSandbox = { ' |
738 |
|
inp_box = inp_box + '"' + script + '",' |
739 |
|
|
746 |
|
#if common.use_jam: |
747 |
|
# inp_box = inp_box+' "'+common.bin_dir+'/'+common.run_jam+'",' |
748 |
|
|
749 |
< |
for addFile in jbt.additional_inbox_files: |
750 |
< |
addFile = os.path.abspath(addFile) |
751 |
< |
inp_box = inp_box+' "'+addFile+'",' |
752 |
< |
pass |
749 |
> |
# Marco (VERY TEMPORARY ML STUFF) |
750 |
> |
inp_box = inp_box+' "' + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + '", "' +\ |
751 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + '", "'+\ |
752 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + '", "'+\ |
753 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + '", "'+\ |
754 |
> |
os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py') + '"' |
755 |
> |
# End Marco |
756 |
> |
|
757 |
> |
if (not jbt.additional_inbox_files == []): |
758 |
> |
inp_box = inp_box + ', ' |
759 |
> |
for addFile in jbt.additional_inbox_files: |
760 |
> |
addFile = os.path.abspath(addFile) |
761 |
> |
inp_box = inp_box+' "'+addFile+'",' |
762 |
> |
pass |
763 |
|
|
764 |
|
if inp_box[-1] == ',' : inp_box = inp_box[:-1] |
765 |
|
inp_box = inp_box + ' };\n' |
777 |
|
job.stdout() + '", "' + \ |
778 |
|
job.stderr() + '", ".BrokerInfo",' |
779 |
|
|
780 |
< |
if self.return_data : |
780 |
> |
if int(self.return_data) == 1: |
781 |
|
if out_sandbox != None: |
782 |
|
for fl in out_sandbox: |
783 |
|
out_box = out_box + ' "' + fl + '",' |
789 |
|
out_box = out_box + ' };' |
790 |
|
jdl.write(out_box+'\n') |
791 |
|
|
792 |
< |
### if at least a CE exists ... |
793 |
< |
if common.analisys_common_info['sites']: |
794 |
< |
if common.analisys_common_info['sw_version']: |
795 |
< |
req='Requirements = ' |
796 |
< |
req=req + 'Member("VO-cms-' + \ |
797 |
< |
common.analisys_common_info['sw_version'] + \ |
798 |
< |
'", other.GlueHostApplicationSoftwareRunTimeEnvironment)' |
799 |
< |
if len(common.analisys_common_info['sites'])>0: |
800 |
< |
req = req + ' && (' |
801 |
< |
for i in range(len(common.analisys_common_info['sites'])): |
802 |
< |
req = req + 'other.GlueCEInfoHostName == "' \ |
803 |
< |
+ common.analisys_common_info['sites'][i] + '"' |
804 |
< |
if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ): |
805 |
< |
req = req + ' || ' |
792 |
> |
|
793 |
> |
req='Requirements = ' |
794 |
> |
noreq=req |
795 |
> |
req = req + jbt.getRequirements(nj) |
796 |
> |
#### and USER REQUIREMENT |
797 |
> |
if self.EDG_requirements: |
798 |
> |
if (req != noreq): |
799 |
> |
req = req + ' && ' |
800 |
> |
req = req + self.EDG_requirements |
801 |
> |
#### FEDE ##### |
802 |
> |
if self.EDG_ce_white_list: |
803 |
> |
ce_white_list = string.split(self.EDG_ce_white_list,',') |
804 |
> |
#print "req = ", req |
805 |
> |
for i in range(len(ce_white_list)): |
806 |
> |
if i == 0: |
807 |
> |
if (req != noreq): |
808 |
> |
req = req + ' && ' |
809 |
> |
req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
810 |
> |
pass |
811 |
> |
else: |
812 |
> |
req = req + ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))' |
813 |
|
req = req + ')' |
814 |
+ |
|
815 |
+ |
if self.EDG_ce_black_list: |
816 |
+ |
ce_black_list = string.split(self.EDG_ce_black_list,',') |
817 |
+ |
for ce in ce_black_list: |
818 |
+ |
if (req != noreq): |
819 |
+ |
req = req + ' && ' |
820 |
+ |
req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))' |
821 |
+ |
pass |
822 |
+ |
|
823 |
+ |
############### |
824 |
+ |
clockTime=480 |
825 |
+ |
if self.EDG_clock_time: |
826 |
+ |
clockTime= self.EDG_clock_time |
827 |
+ |
if (req != noreq): |
828 |
+ |
req = req + ' && ' |
829 |
+ |
req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))' |
830 |
+ |
|
831 |
+ |
cpuTime=1000 |
832 |
+ |
if self.EDG_cpu_time: |
833 |
+ |
cpuTime=self.EDG_cpu_time |
834 |
+ |
if (req != noreq): |
835 |
+ |
req = req + ' && ' |
836 |
+ |
req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))' |
837 |
|
|
838 |
< |
#### and USER REQUIREMENT |
487 |
< |
if self.EDG_requirements: |
488 |
< |
req = req + ' && ' + self.EDG_requirements |
489 |
< |
if self.EDG_clock_time: |
490 |
< |
req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time |
491 |
< |
if self.EDG_cpu_time: |
492 |
< |
req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time |
838 |
> |
if (req != noreq): |
839 |
|
req = req + ';\n' |
840 |
|
jdl.write(req) |
841 |
|
|
845 |
|
jdl.write('RetryCount = '+self.EDG_retry_count+';\n') |
846 |
|
pass |
847 |
|
|
848 |
+ |
jdl.write('MyProxyServer = "' + self.proxyServer + '";\n') |
849 |
+ |
|
850 |
|
jdl.close() |
851 |
|
return |
852 |
|
|
856 |
|
""" |
857 |
|
if (self.proxyValid): return |
858 |
|
timeleft = -999 |
859 |
< |
minTimeLeft=10 # in hours |
860 |
< |
cmd = 'grid-proxy-info -e -v '+str(minTimeLeft)+':00' |
861 |
< |
try: cmd_out = runCommand(cmd,0) |
862 |
< |
except: print cmd_out |
863 |
< |
if (cmd_out == None or cmd_out=='1'): |
864 |
< |
common.logger.message( "No valid proxy found or timeleft too short!\n Creating a user proxy with default length of 100h\n") |
865 |
< |
cmd = 'grid-proxy-init -valid 100:00' |
859 |
> |
minTimeLeft=10*3600 # in seconds |
860 |
> |
|
861 |
> |
minTimeLeftServer = 100 # in hours |
862 |
> |
|
863 |
> |
#cmd = 'voms-proxy-info -exists -valid '+str(minTimeLeft)+':00' |
864 |
> |
#cmd = 'voms-proxy-info -timeleft' |
865 |
> |
mustRenew = 0 |
866 |
> |
timeLeftLocal = runCommand('voms-proxy-info -timeleft 2>/dev/null') |
867 |
> |
timeLeftServer = -999 |
868 |
> |
if not timeLeftLocal or int(timeLeftLocal) <= 0 or not isInt(timeLeftLocal): |
869 |
> |
mustRenew = 1 |
870 |
> |
else: |
871 |
> |
timeLeftServer = runCommand('voms-proxy-info -actimeleft 2>/dev/null | head -1') |
872 |
> |
if not timeLeftServer or not isInt(timeLeftServer): |
873 |
> |
mustRenew = 1 |
874 |
> |
elif timeLeftLocal<minTimeLeft or timeLeftServer<minTimeLeft: |
875 |
> |
mustRenew = 1 |
876 |
> |
pass |
877 |
> |
pass |
878 |
> |
|
879 |
> |
if mustRenew: |
880 |
> |
common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n") |
881 |
> |
cmd = 'voms-proxy-init -voms cms -valid 96:00' |
882 |
|
try: |
883 |
+ |
# SL as above: damn it! |
884 |
|
out = os.system(cmd) |
885 |
|
if (out>0): raise CrabException("Unable to create a valid proxy!\n") |
886 |
|
except: |
887 |
|
msg = "Unable to create a valid proxy!\n" |
888 |
|
raise CrabException(msg) |
889 |
< |
cmd = 'grid-proxy-info -timeleft' |
890 |
< |
cmd_out = runCommand(cmd,0) |
891 |
< |
#print cmd_out, time.time() |
892 |
< |
#time.time(cms_out) |
889 |
> |
# cmd = 'grid-proxy-info -timeleft' |
890 |
> |
# cmd_out = runCommand(cmd,0,20) |
891 |
> |
pass |
892 |
> |
|
893 |
> |
## now I do have a voms proxy valid, and I check the myproxy server |
894 |
> |
renewProxy = 0 |
895 |
> |
cmd = 'myproxy-info -d -s '+self.proxyServer |
896 |
> |
cmd_out = runCommand(cmd,0,20) |
897 |
> |
if not cmd_out: |
898 |
> |
common.logger.message('No credential delegated to myproxy server '+self.proxyServer+' will do now') |
899 |
> |
renewProxy = 1 |
900 |
> |
else: |
901 |
> |
# if myproxy exist but not long enough, renew |
902 |
> |
reTime = re.compile( r'timeleft: (\d+)' ) |
903 |
> |
#print "<"+str(reTime.search( cmd_out ).group(1))+">" |
904 |
> |
if reTime.match( cmd_out ): |
905 |
> |
time = reTime.search( line ).group(1) |
906 |
> |
if time < minTimeLeftServer: |
907 |
> |
renewProxy = 1 |
908 |
> |
common.logger.message('No credential delegation will expire in '+time+' hours: renew it') |
909 |
> |
pass |
910 |
> |
pass |
911 |
> |
|
912 |
> |
# if not, create one. |
913 |
> |
if renewProxy: |
914 |
> |
cmd = 'myproxy-init -d -n -s '+self.proxyServer |
915 |
> |
out = os.system(cmd) |
916 |
> |
if (out>0): |
917 |
> |
raise CrabException("Unable to delegate the proxy to myproxyserver "+self.proxyServer+" !\n") |
918 |
|
pass |
919 |
+ |
|
920 |
+ |
# cache proxy validity |
921 |
|
self.proxyValid=1 |
922 |
|
return |
923 |
< |
|
923 |
> |
|
924 |
|
def configOpt_(self): |
925 |
|
edg_ui_cfg_opt = ' ' |
926 |
|
if self.edg_config: |
927 |
< |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
927 |
> |
edg_ui_cfg_opt = ' -c ' + self.edg_config + ' ' |
928 |
|
if self.edg_config_vo: |
929 |
< |
edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' ' |
929 |
> |
edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' ' |
930 |
|
return edg_ui_cfg_opt |