ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.55 by corvo, Wed May 3 08:44:30 2006 UTC vs.
Revision 1.73.2.5.2.8 by spiga, Thu Sep 14 09:16:39 2006 UTC

# Line 48 | Line 48 | class SchedulerEdg(Scheduler):
48  
49          try:
50              self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
51 –            #print "self.EDG_ce_black_list = ", self.EDG_ce_black_list
51          except KeyError:
52              self.EDG_ce_black_list  = ''
53  
54          try:
55              self.EDG_ce_white_list = cfg_params['EDG.ce_white_list']
57 –            #print "self.EDG_ce_white_list = ", self.EDG_ce_white_list
56          except KeyError: self.EDG_ce_white_list = ''
57  
58          try: self.VO = cfg_params['EDG.virtual_organization']
# Line 65 | Line 63 | class SchedulerEdg(Scheduler):
63  
64          try:
65               self.copy_input_data = common.analisys_common_info['copy_input_data']
68 –             #print "self.copy_input_data = ", self.copy_input_data
66          except KeyError: self.copy_input_data = 0
67  
68          try:
# Line 139 | Line 136 | class SchedulerEdg(Scheduler):
136          except KeyError: self.EDG_cpu_time = ''
137  
138          # Add EDG_WL_LOCATION to the python path
142 –
139          try:
140              path = os.environ['EDG_WL_LOCATION']
141          except:
# Line 152 | Line 148 | class SchedulerEdg(Scheduler):
148          sys.path.append(libPath)
149  
150          self.proxyValid=0
151 +
152 +        try:
153 +            self._taskId = cfg_params['taskId']
154 +        except:
155 +            self._taskId = ''
156 +
157 +        try: self.jobtypeName = cfg_params['CRAB.jobtype']
158 +        except KeyError: self.jobtypeName = ''
159 +
160 +        try: self.schedulerName = cfg_params['CRAB.scheduler']
161 +        except KeyError: self.scheduler = ''
162 +
163          return
164      
165  
# Line 159 | Line 167 | class SchedulerEdg(Scheduler):
167          """
168          Returns file with scheduler-specific parameters
169          """
162 –      
170          if (self.edg_config and self.edg_config_vo != ''):
171              self.param='sched_param.clad'
172              param_file = open(common.work_space.shareDir()+'/'+self.param, 'w')
# Line 175 | Line 182 | class SchedulerEdg(Scheduler):
182          Returns part of a job script which does scheduler-specific work.
183          """
184          txt = ''
185 +        txt += '# strip arguments\n'
186 +        txt += 'echo "strip arguments"\n'
187 +        txt += 'args=("$@")\n'
188 +        txt += 'nargs=$#\n'
189 +        txt += 'shift $nargs\n'
190 +        txt += "# job number (first parameter for job wrapper)\n"
191 +        #txt += "NJob=$1\n"
192 +        txt += "NJob=${args[0]}\n"
193 +
194 +        txt += '# job identification to DashBoard \n'
195 +        txt += 'MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`\n'
196 +        txt += 'SyncGridJobId=`echo $EDG_WL_JOBID`\n'
197 +        txt += 'MonitorID=`echo ' + self._taskId + '`\n'
198 +        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
199 +        txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
200 +        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
201 +
202          txt += 'echo "middleware discovery " \n'
203 <        txt += 'if [ $VO_CMS_SW_DIR ]; then\n'
203 >        txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
204          txt += '    middleware=LCG \n'
205 +        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
206 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
207          txt += '    echo "middleware =$middleware" \n'
208          txt += 'elif [ $GRID3_APP_DIR ]; then\n'
209          txt += '    middleware=OSG \n'
210 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
211 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
212          txt += '    echo "middleware =$middleware" \n'
213          txt += 'elif [ $OSG_APP ]; then \n'
214          txt += '    middleware=OSG \n'
215 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
216 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
217          txt += '    echo "middleware =$middleware" \n'
218          txt += 'else \n'
219 <        txt += '    echo "SET_CMS_ENV 1 ==> middleware not identified" \n'
220 <        txt += '    echo "JOB_EXIT_STATUS = 1"\n'
221 <        txt += '    exit 1\n'
222 <        txt += 'fi\n'
223 <
219 >        txt += '    echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
220 >        txt += '    echo "JOB_EXIT_STATUS = 10030" \n'
221 >        txt += '    echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n'
222 >        txt += '    dumpStatus $RUNTIME_AREA/$repo \n'
223 >        txt += '    rm -f $RUNTIME_AREA/$repo \n'
224 >        txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
225 >        txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
226 >        txt += '    exit 1 \n'
227 >        txt += 'fi \n'
228 >
229 >        txt += '# report first time to DashBoard \n'
230 >        txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
231 >        txt += 'rm -f $RUNTIME_AREA/$repo \n'
232 >        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
233 >        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
234 >        
235          txt += '\n\n'
236  
196 –        txt += 'if [ $middleware == LCG ]; then \n'
197 –        txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
198 –        txt += 'fi\n'
199 –
237          if int(self.copy_data) == 1:
238             if self.SE:
239                txt += 'export SE='+self.SE+'\n'
# Line 256 | Line 293 | class SchedulerEdg(Scheduler):
293          txt += '    if [ $OSG_JOB_CONTACT ]; then \n'
294          txt += '        CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n'
295          txt += '    else \n'
296 <        txt += '        echo "SET_ENV 1 ==> ERROR in setting CE name - OSG mode -" \n'
296 >        txt += '        echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n'
297 >        txt += '        echo "JOB_EXIT_STATUS = 10099" \n'
298 >        txt += '        echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n'
299 >        txt += '        dumpStatus $RUNTIME_AREA/$repo \n'
300 >        txt += '        rm -f $RUNTIME_AREA/$repo \n'
301 >        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
302 >        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
303          txt += '        exit 1 \n'
304          txt += '    fi \n'
305          txt += 'fi \n'
# Line 270 | Line 313 | class SchedulerEdg(Scheduler):
313          txt = ''
314          try:
315              self.copy_input_data = common.analisys_common_info['copy_input_data']
273 –            #print "self.copy_input_data = ", self.copy_input_data
316          except KeyError: self.copy_input_data = 0
317          if int(self.copy_input_data) == 1:
318          ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
# Line 283 | Line 325 | class SchedulerEdg(Scheduler):
325             txt += '   #\n'
326             txt += '   #   Copy Input Data from SE to this WN\n'
327             txt += '   #\n'
328 < ### changed by georgia (put a loop copying more than one input files per jobs)          
328 >           ### changed by georgia (put a loop copying more than one input files per jobs)          
329             txt += '   for input_file in $cur_file_list \n'
330             txt += '   do \n'
331 <           txt += '    lcg-cp --vo $VO lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
332 <           txt += '    copy_input_exit_status=$?\n'
333 <           txt += '    echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n'
334 <           txt += '    if [ $copy_input_exit_status -ne 0 ]; then \n'
335 <           txt += '       echo "Problems with copying to WN" \n'
336 <           txt += '    else \n'
337 <           txt += '       echo "input copied into WN" \n'
338 <           txt += '    fi \n'
331 >           txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
332 >           txt += '      copy_input_exit_status=$?\n'
333 >           txt += '      echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n'
334 >           txt += '      if [ $copy_input_exit_status -ne 0 ]; then \n'
335 >           txt += '         echo "Problems with copying to WN" \n'
336 >           txt += '      else \n'
337 >           txt += '         echo "input copied into WN" \n'
338 >           txt += '      fi \n'
339             txt += '   done \n'
340 < ### copy a set of PU ntuples (same for each jobs -- but accessed randomly)
340 >           ### copy a set of PU ntuples (same for each jobs -- but accessed randomly)
341             txt += '   for file in $cur_pu_list \n'
342             txt += '   do \n'
343 <           txt += '    lcg-cp --vo $VO lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
344 <           txt += '    copy_input_exit_status=$?\n'
345 <           txt += '    echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n'
346 <           txt += '    if [ $copy_input_pu_exit_status -ne 0 ]; then \n'
347 <           txt += '       echo "Problems with copying pu to WN" \n'
348 <           txt += '    else \n'
349 <           txt += '       echo "input pu files copied into WN" \n'
350 <           txt += '    fi \n'
343 >           txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
344 >           txt += '      copy_input_pu_exit_status=$?\n'
345 >           txt += '      echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n'
346 >           txt += '      if [ $copy_input_pu_exit_status -ne 0 ]; then \n'
347 >           txt += '         echo "Problems with copying pu to WN" \n'
348 >           txt += '      else \n'
349 >           txt += '         echo "input pu files copied into WN" \n'
350 >           txt += '      fi \n'
351             txt += '   done \n'
352             txt += '   \n'
353             txt += '   ### Check SCRATCH space available on WN : \n'
# Line 324 | Line 366 | class SchedulerEdg(Scheduler):
366             txt += '#\n'
367             txt += '#   Copy output to SE = $SE\n'
368             txt += '#\n'
369 <           txt += 'if [ $exe_result -eq 0 ]; then\n'
369 >           txt += '    if [ $middleware == OSG ]; then\n'
370 >           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
371 >           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
372 >           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
373 >           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
374 >           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
375 >           txt += '    fi \n'
376             txt += '    for out_file in $file_list ; do\n'
377 <           txt += '        echo "Trying to copy output file to $SE "\n'
378 <           ## OLI_Daniele globus-* for OSG, lcg-* for LCG
379 <           txt += '        if [ $middleware == OSG ]; then\n'
380 <           txt += '           echo "globus-url-copy file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
381 <           txt += '           copy_exit_status=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
334 <           #txt += '           exitstring=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
335 <           txt += '        elif [ $middleware == LCG ]; then \n'
336 <           txt += '           echo "lcg-cp --vo cms -t 1200 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
337 <           txt += '           copy_exit_status=`lcg-cp --vo cms -t 1200 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
338 <           #txt += '           exitstring=`lcg-cp --vo cms -t 30 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
339 <           txt += '        fi \n'
340 <           #txt += '        copy_exit_status=$?\n'
341 <           txt += '        echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
377 >           txt += '        echo "Trying to copy output file to $SE using lcg-cp"\n'
378 >           txt += '        echo "lcg-cp --vo $VO -t 1200 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
379 >           txt += '        exitstring=`lcg-cp --vo $VO -t 1200 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
380 >           txt += '        copy_exit_status=$?\n'
381 >           txt += '        echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
382             txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
383             txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
384 <           txt += '            echo "Problems with SE = $SE"\n'
384 >           txt += '            echo "Possible problem with SE = $SE"\n'
385             txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
386             txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
387 +           txt += '            echo "lcg-cp failed, attempting srmcp"\n'
388 +           txt += '            echo "mkdir -p $HOME/.srmconfig"\n'
389 +           txt += '            mkdir -p $HOME/.srmconfig\n'
390 +           txt += '            if [ $middleware == LCG ]; then\n'
391 +           txt += '               echo "srmcp -retry_num 5 -retry_timeout 240000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
392 +           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 240000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
393 +           txt += '            elif [ $middleware == OSG ]; then\n'
394 +           txt += '               echo "srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
395 +           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
396 +           txt += '            fi \n'
397 +           txt += '            copy_exit_status=$?\n'
398 +           txt += '            echo "COPY_EXIT_STATUS for srm = $copy_exit_status"\n'
399 +           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
400 +           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
401 +           txt += '               echo "Problems with SE = $SE"\n'
402 +           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
403 +           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
404 +           txt += '               echo "lcg-cp and srm failed"\n'
405 +           txt += '               echo "If storage_path in your config file contains a ? you may need a \? instead."\n'
406 +           txt += '            else\n'
407 +           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
408 +           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
409 +           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
410 +           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
411 +           txt += '               echo "srmcp succeeded"\n'
412 +           txt += '            fi\n'
413             txt += '        else\n'
414             txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
415             txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
416             txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
417             txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
418 +           txt += '            echo "lcg-cp succeeded"\n'
419             txt += '         fi\n'
420             txt += '     done\n'
354 –           txt += 'fi\n'
421          return txt
422  
423      def wsRegisterOutput(self):
# Line 361 | Line 427 | class SchedulerEdg(Scheduler):
427  
428          txt = ''
429          if int(self.register_data) == 1:
430 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
430 >           ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
431             txt += 'if [ $middleware == OSG ]; then\n'
432             txt += '   #\n'
433             txt += '   #   Register output to LFC deactivated in OSG mode\n'
# Line 371 | Line 437 | class SchedulerEdg(Scheduler):
437             txt += '#\n'
438             txt += '#  Register output to LFC\n'
439             txt += '#\n'
440 <           txt += '   if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n'
440 >           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
441             txt += '      for out_file in $file_list ; do\n'
442             txt += '         echo "Trying to register the output file into LFC"\n'
443 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO sfn://$SE$SE_PATH/$out_file"\n'
444 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO sfn://$SE$SE_PATH/$out_file 2>&1 \n'
443 >           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
444 >           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
445             txt += '         register_exit_status=$?\n'
446             txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
447             txt += '         echo "STAGE_OUT = $register_exit_status"\n'
448             txt += '         if [ $register_exit_status -ne 0 ]; then \n'
449             txt += '            echo "Problems with the registration to LFC" \n'
450             txt += '            echo "Try with srm protocol" \n'
451 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO srm://$SE$SE_PATH/$out_file"\n'
452 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO srm://$SE$SE_PATH/$out_file 2>&1 \n'
451 >           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
452 >           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
453             txt += '            register_exit_status=$?\n'
454             txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
455             txt += '            echo "STAGE_OUT = $register_exit_status"\n'
# Line 395 | Line 461 | class SchedulerEdg(Scheduler):
461             txt += '         fi \n'
462             txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
463             txt += '      done\n'
464 <           txt += '   elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n'
464 >           txt += '   else \n'
465             txt += '      echo "Trying to copy output file to CloseSE"\n'
466             txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
467             txt += '      for out_file in $file_list ; do\n'
468 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://`pwd`/$out_file" \n'
469 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://`pwd`/$out_file 2>&1 \n'
468 >           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
469 >           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
470             txt += '         register_exit_status=$?\n'
471             txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
472             txt += '         echo "STAGE_OUT = $register_exit_status"\n'
473             txt += '         if [ $register_exit_status -ne 0 ]; then \n'
474 <           txt += '            echo "Problems with CloseSE" \n'
474 >           txt += '            echo "Problems with CloseSE or Catalog" \n'
475             txt += '         else \n'
476             txt += '            echo "The program was successfully executed"\n'
477             txt += '            echo "SE = $CLOSE_SE"\n'
# Line 413 | Line 479 | class SchedulerEdg(Scheduler):
479             txt += '         fi \n'
480             txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
481             txt += '      done\n'
416 –           txt += '   else\n'
417 –           txt += '      echo "Problem with the executable"\n'
482             txt += '   fi \n'
483 +           txt += '   exit_status=$register_exit_status\n'
484             txt += 'fi \n'
485          return txt
486  
# Line 429 | Line 494 | class SchedulerEdg(Scheduler):
494          cmd_out = runCommand(cmd)
495          return cmd_out
496  
497 <    def listMatch(self, nj):
498 <        """
499 <        Check the compatibility of available resources
500 <        """
501 <        self.checkProxy()
502 <        jdl = common.job_list[nj].jdlFilename()
503 <        cmd = 'edg-job-list-match ' + self.configOpt_() + jdl
504 <        cmd_out = runCommand(cmd,0,10)
505 <        if not cmd_out:
506 <            raise CrabException("ERROR: "+cmd+" failed!")
507 <
508 <        return self.parseListMatch_(cmd_out, jdl)
509 <
510 <    def parseListMatch_(self, out, jdl):
511 <        """
512 <        Parse the f* output of edg-list-match and produce something sensible
513 <        """
514 <        reComment = re.compile( r'^\**$' )
515 <        reEmptyLine = re.compile( r'^$' )
516 <        reVO = re.compile( r'Selected Virtual Organisation name.*' )
517 <        reLine = re.compile( r'.*')
518 <        reCE = re.compile( r'(.*:.*)')
519 <        reCEId = re.compile( r'CEId.*')
520 <        reNO = re.compile( r'No Computing Element matching' )
521 <        reRB = re.compile( r'Connecting to host' )
522 <        next = 0
523 <        CEs=[]
524 <        Match=0
525 <
526 <        #print out
527 <        lines = reLine.findall(out)
528 <
529 <        i=0
530 <        CEs=[]
531 <        for line in lines:
532 <            string.strip(line)
533 <            #print line
534 <            if reNO.match( line ):
535 <                common.logger.debug(5,line)
536 <                return 0
537 <                pass
538 <            if reVO.match( line ):
539 <                VO =reVO.match( line ).group()
540 <                common.logger.debug(5,"VO "+VO)
541 <                pass
542 <
543 <            if reRB.match( line ):
544 <                RB = reRB.match(line).group()
545 <                common.logger.debug(5,"RB "+RB)
546 <                pass
547 <
548 <            if reCEId.search( line ):
549 <                for lineCE in lines[i:-1]:
550 <                    if reCE.match( lineCE ):
551 <                        CE = string.strip(reCE.search(lineCE).group(1))
552 <                        CEs.append(CE.split(':')[0])
553 <                        pass
554 <                    pass
555 <                pass
556 <            i=i+1
557 <            pass
558 <
559 <        common.logger.debug(5,"All CE :"+str(CEs))
560 <
561 <        sites = []
562 <        [sites.append(it) for it in CEs if not sites.count(it)]
563 <
564 <        common.logger.debug(5,"All Sites :"+str(sites))
565 <        return len(sites)
566 <
502 <    def noMatchFound_(self, jdl):
503 <        reReq = re.compile( r'Requirements' )
504 <        reString = re.compile( r'"\S*"' )
505 <        f = file(jdl,'r')
506 <        for line in f.readlines():
507 <            line= line.strip()
508 <            if reReq.match(line):
509 <                for req in reString.findall(line):
510 <                    if re.search("VO",req):
511 <                        common.logger.message( "SW required: "+req)
512 <                        continue
513 <                    if re.search('"\d+',req):
514 <                        common.logger.message("Other req  : "+req)
515 <                        continue
516 <                    common.logger.message( "CE required: "+req)
517 <                break
518 <            pass
519 <        raise CrabException("No compatible resources found!")
520 <
521 <    def submit(self, nj):
522 <        """
523 <        Submit one EDG job.
524 <        """
525 <
526 <        self.checkProxy()
527 <        jid = None
528 <        jdl = common.job_list[nj].jdlFilename()
497 >  #  def listMatch(self, nj):
498 >  #      """
499 >  #      Check the compatibility of available resources
500 >  #      """
501 >  #      self.checkProxy()
502 >  #      jdl = common.work_space.shareDir()+"fake.jdl"
503 >  #      cmd = 'edg-job-list-match ' + self.configOpt_() + str(jdl)
504 >  #      cmd_out = runCommand(cmd,0,10)
505 >  #      if not cmd_out:
506 >  #          raise CrabException("ERROR: "+cmd+" failed!")
507 > #
508 > #       return self.parseListMatch_(cmd_out, jdl)
509 >
510 >  #  def parseListMatch_(self, out, jdl):
511 >  #      """
512 >  #      Parse the f* output of edg-list-match and produce something sensible
513 >  #      """
514 >  #      reComment = re.compile( r'^\**$' )
515 >  #      reEmptyLine = re.compile( r'^$' )
516 >  #      reVO = re.compile( r'Selected Virtual Organisation name.*' )
517 >  #      reLine = re.compile( r'.*')
518 >  #      reCE = re.compile( r'(.*:.*)')
519 >  #      reCEId = re.compile( r'CEId.*')
520 >  #      reNO = re.compile( r'No Computing Element matching' )
521 >  #      reRB = re.compile( r'Connecting to host' )
522 >  #      next = 0
523 >  #      CEs=[]
524 >  #      Match=0
525 > #
526 > #        #print out
527 > #        lines = reLine.findall(out)
528 >
529 > #       i=0
530 > #       CEs=[]
531 > #       for line in lines:
532 > #           string.strip(line)
533 > #           #print line
534 > #           if reNO.match( line ):
535 > #               common.logger.debug(5,line)
536 > #               return 0
537 > #               pass
538 > #           if reVO.match( line ):
539 > #               VO =reVO.match( line ).group()
540 > #               common.logger.debug(5,"VO "+VO)
541 > #               pass
542 >
543 > #           if reRB.match( line ):
544 > #               RB = reRB.match(line).group()
545 > #               common.logger.debug(5,"RB "+RB)
546 > #               pass
547 >
548 > #           if reCEId.search( line ):
549 > #               for lineCE in lines[i:-1]:
550 > #                   if reCE.match( lineCE ):
551 > #                       CE = string.strip(reCE.search(lineCE).group(1))
552 > #                       CEs.append(CE.split(':')[0])
553 > #                       pass
554 > #                   pass
555 > #               pass
556 > #           i=i+1
557 > #           pass
558 >
559 > #       common.logger.debug(5,"All CE :"+str(CEs))
560 >
561 > #       sites = []
562 > #       [sites.append(it) for it in CEs if not sites.count(it)]
563 >
564 > #       common.logger.debug(5,"All Sites :"+str(sites))
565 > #       common.logger.message("Matched Sites :"+str(sites))
566 > #       return len(sites)
567  
530 –        cmd = 'edg-job-submit ' + self.configOpt_() + jdl
531 –        cmd_out = runCommand(cmd)
532 –        if cmd_out != None:
533 –            reSid = re.compile( r'https.+' )
534 –            jid = reSid.search(cmd_out).group()
535 –            pass
536 –        return jid
537 –
538 –    def resubmit(self, nj_list):
539 –        """
540 –        Prepare jobs to be submit
541 –        """
542 –        return
568  
569 +    ################################################################ To remove when Boss4 store this info  DS. (start)
570      def getExitStatus(self, id):
571          return self.getStatusAttribute_(id, 'exit_code')
572  
# Line 569 | Line 595 | class SchedulerEdg(Scheduler):
595              common.logger.debug(5,'Error caught' + apiMsg)
596              return None
597          else:
598 <            for i in range(len(self.states)):
598 >           for i in range(len(self.states)):
599                  # Fill an hash table with all information retrieved from LB API
600                  hstates[ self.states[i] ] = jobStat.loadStatus(st)[i]
601 <            result = jobStat.loadStatus(st)[ self.states.index(attr) ]
602 <            return result
601 >           result = jobStat.loadStatus(st)[self.states.index(attr)]
602 >           return result
603 >
604  
605      def queryDetailedStatus(self, id):
606          """ Query a detailed status of the job with id """
# Line 581 | Line 608 | class SchedulerEdg(Scheduler):
608          cmd_out = runCommand(cmd)
609          return cmd_out
610  
584 –    def getOutput(self, id):
585 –        """
586 –        Get output for a finished job with id.
587 –        Returns the name of directory with results.
588 –        """
589 –
590 –        self.checkProxy()
591 –        cmd = 'edg-job-get-output --dir ' + common.work_space.resDir() + ' ' + id
592 –        cmd_out = runCommand(cmd)
611  
612 <        # Determine the output directory name
613 <        dir = common.work_space.resDir()
614 <        dir += os.environ['USER']
615 <        dir += '_' + os.path.basename(id)
616 <        return dir
617 <
618 <    def cancel(self, id):
619 <        """ Cancel the EDG job with id """
620 <        self.checkProxy()
621 <        cmd = 'edg-job-cancel --noint ' + id
622 <        cmd_out = runCommand(cmd)
623 <        return cmd_out
624 <
625 <    def createSchScript(self, nj):
626 <        """
627 <        Create a JDL-file for EDG.
612 >  #  def createFakeJdl(self,nj):  # TMP Just waiting listmatch functionalitly  
613 >  #                                # implementation into BOSS4   Daniele
614 >  #      """                          
615 >  #      Create a fake jdl considering
616 >  #      only requirements  
617 >  #      """
618 >  #      job = common.job_list[0]
619 >  #      jbt = job.type()
620 >  #      inp_storage_subdir = ''
621 >  #      
622 >  #      
623 >  #      SPL = inp_storage_subdir
624 >  #      if ( SPL and SPL[-1] != '/' ) : SPL = SPL + '/'
625 > #
626 > #        jdl = open(common.work_space.shareDir()+"fake.jdl","w")
627 > #
628 > #        script = job.scriptFilename()
629 > #        jdl.write('Executable = "' + os.path.basename(script) +'";\n')
630 > #
631 > #        req='Requirements = '
632 > #        noreq=req
633 > #        ##### 8_9_06 FEDE
634 > #        req = req + jbt.getRequirements(nj)
635 > #        #req = req + jbt.getRequirements()
636 > #        ########
637 > #        #### and USER REQUIREMENT
638 > #       if self.EDG_requirements:
639 > #           if (req != noreq):
640 > #               req = req +  ' && '
641 > #           req = req + self.EDG_requirements
642 > #       #### FEDE #####
643 > #       if self.EDG_ce_white_list:
644 > #           ce_white_list = string.split(self.EDG_ce_white_list,',')
645 > #           #print "req = ", req
646 > #           for i in range(len(ce_white_list)):
647 > #               if i == 0:
648 > #                   if (req != noreq):
649 > #                       req = req +  ' && '
650 > #                   req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
651 >  #                  pass
652 >  #              else:
653 >  #                  req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
654 >  #          req = req + ')'
655 >  #      
656 >  #      if self.EDG_ce_black_list:
657 >  #          ce_black_list = string.split(self.EDG_ce_black_list,',')
658 >  #          for ce in ce_black_list:
659 >  #              if (req != noreq):
660 >  #                  req = req +  ' && '
661 >  #              req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
662 >  #              pass
663 >  #
664 >  #      ###############
665 >  #      clockTime=480
666 >  #      if self.EDG_clock_time:
667 >  #          clockTime= self.EDG_clock_time
668 >  #      if (req != noreq):
669 >  #          req = req + ' && '
670 >  #      req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))'
671 >  #
672 >  #      cpuTime=1000
673 >  #      if self.EDG_cpu_time:
674 >  #          cpuTime=self.EDG_cpu_time
675 >  #      if (req != noreq):
676 >  #          req = req + ' && '
677 >   #     req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))'
678 >   #
679 >   #     if (req != noreq):
680 >   #         req = req + ';\n'
681 >   #         jdl.write(req)
682 >   #                                                                                                                                                          
683 >   #     jdl.write('VirtualOrganisation = "' + self.VO + '";\n')
684 >   #
685 >   #     if ( self.EDG_retry_count ):              
686 >   #         jdl.write('RetryCount = '+self.EDG_retry_count+';\n')
687 >   #         pass
688 >   #
689 >   #     jdl.write('MyProxyServer = "' + self.proxyServer + '";\n')
690 > #
691 > #        jdl.close()
692 > #        return
693 >  
694 >    ##### FEDE ######        
695 >    def findSites_(self, n_tot_job):
696 >        itr4=''
697 >        #print "n_tot_job = ", n_tot_job
698 >        for n in range(n_tot_job):
699 >            sites = common.jobDB.destination(n)
700 >            #job = common.job_list[n]
701 >            #jbt = job.type()
702 >        #    print "common.jobDB.destination(n) = ", common.jobDB.destination(n)
703 >        #    print "sites = ", sites
704 >            for site in sites:
705 >                itr4 = itr4 + 'target.GlueSEUniqueID==&quot;'+site+'&quot; || '
706 >                pass
707 >            # remove last ||
708 >            itr4 = itr4[0:-4]
709 >            itr4 = itr4 + ','
710 >        # remove last ,
711 >        itr4 = itr4[0:-1]
712 >        #print "itr4 = ", itr4
713 >        return itr4
714 >
715 >    def createXMLSchScript(self, nj, argsList, jobList):
716 >   # def createXMLSchScript(self, nj):
717 >        """
718 >        Create a XML-file for BOSS4.
719 >        """
720 >  #      job = common.job_list[nj]
721 >        """
722 >        INDY
723 >        [begin] da rivedere:
724 >        in particolare passerei il jobType ed eliminerei le dipendenze da job
725          """
726 <
727 <        job = common.job_list[nj]
726 >        index = nj - 1
727 >        job = common.job_list[index]
728          jbt = job.type()
614 –        inp_sandbox = jbt.inputSandbox(nj)
615 –        out_sandbox = jbt.outputSandbox(nj)
616 –        inp_storage_subdir = ''
617 –        
618 –        title = '# This JDL was generated by '+\
619 –                common.prog_name+' (version '+common.prog_version_str+')\n'
620 –        jt_string = ''
621 –
622 –
729          
730 <        SPL = inp_storage_subdir
731 <        if ( SPL and SPL[-1] != '/' ) : SPL = SPL + '/'
732 <
733 <        jdl_fname = job.jdlFilename()
734 <        jdl = open(jdl_fname, 'w')
629 <        jdl.write(title)
630 <
631 <        script = job.scriptFilename()
632 <        jdl.write('Executable = "' + os.path.basename(script) +'";\n')
633 <        jdl.write(jt_string)
634 <
635 <        ### only one .sh  JDL has arguments:
636 <        firstEvent = common.jobDB.firstEvent(nj)
637 <        maxEvents = common.jobDB.maxEvents(nj)
638 <        jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n')
639 <
640 <        inp_box = 'InputSandbox = { '
641 <        inp_box = inp_box + '"' + script + '",'
642 <
643 <        if inp_sandbox != None:
644 <            for fl in inp_sandbox:
645 <                inp_box = inp_box + ' "' + fl + '",'
646 <                pass
647 <            pass
648 <
649 <        #if common.use_jam:
650 <        #   inp_box = inp_box+' "'+common.bin_dir+'/'+common.run_jam+'",'
651 <
652 <        # Marco (VERY TEMPORARY ML STUFF)
653 <        inp_box = inp_box+' "' + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + '", "' +\
654 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + '", "'+\
655 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + '", "'+\
656 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + '", "'+\
657 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py') + '"'
658 <        # End Marco
659 <
660 <        if (not jbt.additional_inbox_files == []):
661 <            inp_box = inp_box + ', '
662 <            for addFile in jbt.additional_inbox_files:
663 <                addFile = os.path.abspath(addFile)
664 <                inp_box = inp_box+' "'+addFile+'",'
665 <                pass
666 <
667 <        if inp_box[-1] == ',' : inp_box = inp_box[:-1]
668 <        inp_box = inp_box + ' };\n'
669 <        jdl.write(inp_box)
730 >        inp_sandbox = jbt.inputSandbox(index)
731 >        out_sandbox = jbt.outputSandbox(index)
732 >        """
733 >        [end] da rivedere
734 >        """
735  
671 –        jdl.write('StdOutput     = "' + job.stdout() + '";\n')
672 –        jdl.write('StdError      = "' + job.stderr() + '";\n')
736          
737 +        title = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'
738 +        jt_string = ''
739          
740 <        if job.stdout() == job.stderr():
741 <          out_box = 'OutputSandbox = { "' + \
677 <                    job.stdout() + '", ".BrokerInfo",'
678 <        else:
679 <          out_box = 'OutputSandbox = { "' + \
680 <                    job.stdout() + '", "' + \
681 <                    job.stderr() + '", ".BrokerInfo",'
740 >        xml_fname = str(self.jobtypeName)+'.xml'
741 >        xml = open(common.work_space.shareDir()+'/'+xml_fname, 'a')
742  
743 <        if int(self.return_data) == 1:
744 <            if out_sandbox != None:
745 <                for fl in out_sandbox:
746 <                    out_box = out_box + ' "' + fl + '",'
747 <                    pass
748 <                pass
749 <            pass
750 <                                                                                                                                                            
751 <        if out_box[-1] == ',' : out_box = out_box[:-1]
752 <        out_box = out_box + ' };'
753 <        jdl.write(out_box+'\n')
754 <
755 <
756 <        req='Requirements = '
757 <        req = req + jbt.getRequirements()
758 < #        ### if at least a CE exists ...
699 < #        if common.analisys_common_info['sites']:
700 < #           if common.analisys_common_info['sw_version']:
701 < #                req='Requirements = '
702 < #                req=req + 'Member("VO-cms-' + \
703 < #                     common.analisys_common_info['sw_version'] + \
704 < #                     '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
705 < #            if len(common.analisys_common_info['sites'])>0:
706 < #                req = req + ' && ('
707 < #                for i in range(len(common.analisys_common_info['sites'])):
708 < #                    req = req + 'other.GlueCEInfoHostName == "' \
709 < #                         + common.analisys_common_info['sites'][i] + '"'
710 < #                    if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
711 < #                        req = req + ' || '
712 < #            req = req + ')'
713 <        #### and USER REQUIREMENT
743 >        #TaskName  
744 >        dir = string.split(common.work_space.topDir(), '/')
745 >        taskName = dir[len(dir)-2]
746 >  
747 >        to_writeReq = ''
748 >        to_write = ''
749 >
750 >        req=' '
751 >        req = req + jbt.getRequirements(nj)
752 >
753 >
754 >        #sites = common.jobDB.destination(nj)
755 >        #if len(sites)>0 and sites[0]!="Any":
756 >        #    req = req + ' && anyMatch(other.storage.CloseSEs, (_ITR4_))'
757 >        #req = req    
758 >    
759          if self.EDG_requirements:
760 <            if (req == 'Requirement = '):
760 >            if (req == ' '):
761                  req = req + self.EDG_requirements
762              else:
763                  req = req +  ' && ' + self.EDG_requirements
719 –        #### FEDE #####
764          if self.EDG_ce_white_list:
765              ce_white_list = string.split(self.EDG_ce_white_list,',')
722 –            #print "req = ", req
766              for i in range(len(ce_white_list)):
767                  if i == 0:
768 <                    if (req == 'Requirement = '):
768 >                    if (req == ' '):
769                          req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
770                      else:
771                          req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
# Line 734 | Line 777 | class SchedulerEdg(Scheduler):
777          if self.EDG_ce_black_list:
778              ce_black_list = string.split(self.EDG_ce_black_list,',')
779              for ce in ce_black_list:
780 <                if (req == 'Requirement = '):
780 >                if (req == ' '):
781                      req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
782                  else:
783                      req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
784                  pass
742 –        ###############
785          if self.EDG_clock_time:
786 <            if (req == 'Requirement = '):
786 >            if (req == ' '):
787                  req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
788              else:
789                  req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
790  
791          if self.EDG_cpu_time:
792 <            if (req == 'Requirement = '):
792 >            if (req == ' '):
793                  req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
794              else:
795                  req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
754 –        if (req != 'Requirement = '):
755 –            req = req + ';\n'
756 –            jdl.write(req)
757 –                                                                                                                                                            
758 –        jdl.write('VirtualOrganisation = "' + self.VO + '";\n')
796  
797 +        #if (req != ' '):
798 +        #    req = req + '\n'
799 +        #    to_writeReq = req
800 +                                                                                                                                                            
801          if ( self.EDG_retry_count ):              
802 <            jdl.write('RetryCount = '+self.EDG_retry_count+';\n')
802 >            to_write = to_write + 'RetryCount = "'+self.EDG_retry_count+'"\n'
803 >            pass
804 >
805 >        to_write = to_write + 'MyProxyServer = "&quot;' + self.proxyServer + '&quot;"\n'
806 >        to_write = to_write + 'VirtualOrganisation = "&quot;' + self.VO + '&quot;"\n'
807 >
808 >
809 >        #TaskName  
810 >        dir = string.split(common.work_space.topDir(), '/')
811 >        taskName = dir[len(dir)-2]
812 >
813 >        xml.write(str(title))
814 >        xml.write('<task name="' +str(taskName)+'">\n')
815 >        xml.write(jt_string)
816 >
817 >        xml.write('<iterator>\n')
818 >
819 >        #print str(nj)
820 >        xml.write('\t<iteratorRule name="ITR1" rule="1:'+ str(nj) + '" />\n')
821 >        #print argsList
822 >        xml.write('\t<iteratorRule name="ITR2" rule="'+ argsList + '" />\n')
823 >        #print jobList
824 >        xml.write('\t<iteratorRule name="ITR3" rule="'+ jobList + '" />\n')
825 >
826 >        #### FEDE #####
827 >        itr4=self.findSites_(nj)
828 >        #print "--->>> itr4 = ", itr4
829 >        if (itr4 != ''):
830 >           xml.write('\t<iteratorRule name="ITR4" rule="'+itr4+ '" />\n')
831 >           req = req + ' && anyMatch(other.storage.CloseSEs, (_ITR4_))'
832 >           pass
833 >    #    print "--->>> req= ", req        
834 >  
835 >        if (to_write != ''):
836 >            xml.write('<extraTags\n')
837 >            xml.write(to_write)
838 >            xml.write('/>\n')
839 >            pass
840 >
841 >        xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
842 >        xml.write(jt_string)
843 >
844 >        if (req != ' '):
845 >            req = req + '\n'
846 >            xml.write('<extraTags>\n')
847 >            xml.write('<Requirements>\n')
848 >            xml.write('<![CDATA[\n')
849 >            xml.write(req)
850 >            xml.write(']]>\n')
851 >            xml.write('</Requirements>\n')
852 >            xml.write('</extraTags>\n')
853 >            pass
854 >
855 >        #executable
856 >
857 >        """
858 >        INDY
859 >        script dipende dal jobType: dovrebbe essere semplice tirarlo fuori in altro modo
860 >        """        
861 >        script = job.scriptFilename()
862 >        xml.write('<program exec="' + os.path.basename(script) +'"\n')
863 >        xml.write(jt_string)
864 >    
865 >          
866 >        ### only one .sh  JDL has arguments:
867 >        ### Fabio
868 > #        xml.write('args = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'"\n')
869 >        xml.write('args = "_ITR2_"\n')
870 >        xml.write('program_types="crabjob"\n')
871 >        inp_box = 'infiles="'
872 >        inp_box = inp_box + '' + script + ','
873 >
874 >        if inp_sandbox != None:
875 >            for fl in inp_sandbox:
876 >                inp_box = inp_box + '' + fl + ','
877 >                pass
878              pass
879  
880 <        jdl.write('MyProxyServer = "' + self.proxyServer + '";\n')
880 >        inp_box = inp_box + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + ',' +\
881 >                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + ','+\
882 >                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + ','+\
883 >                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + ','+\
884 >                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py')
885 >
886 >        if (not jbt.additional_inbox_files == []):
887 >            inp_box = inp_box + ', '
888 >            for addFile in jbt.additional_inbox_files:
889 >                addFile = os.path.abspath(addFile)
890 >                inp_box = inp_box+''+addFile+','
891 >                pass
892 >
893 >        if inp_box[-1] == ',' : inp_box = inp_box[:-1]
894 >        inp_box = inp_box + ' "\n'
895 >        xml.write(inp_box)
896 >        
897 >        base = jbt.name()
898 >        stdout = base + '__ITR3_.stdout'
899 >        stderr = base + '__ITR3_.stderr'
900 >        
901 >        xml.write('stderr="' + stderr + '"\n')
902 >        xml.write('stdout="' + stdout + '"\n')
903 >        
904 >
905 >        out_box = 'outfiles="' + \
906 >                  stdout + ',' + \
907 >                  stderr + ',.BrokerInfo,'
908 >
909 >        """
910 >        if int(self.return_data) == 1:
911 >            if out_sandbox != None:
912 >                for fl in out_sandbox:
913 >                    out_box = out_box + '' + fl + ','
914 >                    pass
915 >                pass
916 >            pass
917 >        """
918 >
919 >        """
920 >        INDY
921 >        qualcosa del genere andrebbe fatta per gli infiles
922 >        """        
923 >        if int(self.return_data) == 1:
924 >            for fl in jbt.output_file:
925 >                out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ','
926 >                pass
927 >            pass
928 >
929 >        if out_box[-1] == ',' : out_box = out_box[:-1]
930 >        out_box = out_box + '"'
931 >        xml.write(out_box+'\n')
932 >
933 >        xml.write('group="'+taskName+'"\n')
934 >        xml.write('BossAttr="crabjob.INTERNAL_ID=_ITR1_"\n')
935 >
936 >        xml.write('/>\n')
937 >        xml.write('</chain>\n')
938 >
939 >        xml.write('</iterator>\n')
940 >        xml.write('</task>\n')
941  
942 <        jdl.close()
942 >        xml.close()
943          return
944  
945      def checkProxy(self):
# Line 776 | Line 952 | class SchedulerEdg(Scheduler):
952  
953          minTimeLeftServer = 100 # in hours
954  
779 –        #cmd = 'voms-proxy-info -exists -valid '+str(minTimeLeft)+':00'
780 –        #cmd = 'voms-proxy-info -timeleft'
955          mustRenew = 0
956 <        timeLeftLocal = runCommand('voms-proxy-info -timeleft')
956 >        timeLeftLocal = runCommand('voms-proxy-info -timeleft 2>/dev/null')
957          timeLeftServer = -999
958          if not timeLeftLocal or int(timeLeftLocal) <= 0 or not isInt(timeLeftLocal):
959              mustRenew = 1
960          else:
961 <            timeLeftServer = runCommand('voms-proxy-info -actimeleft | head -1')
961 >            timeLeftServer = runCommand('voms-proxy-info -actimeleft 2>/dev/null | head -1')
962              if not timeLeftServer or not isInt(timeLeftServer):
963                  mustRenew = 1
964              elif timeLeftLocal<minTimeLeft or timeLeftServer<minTimeLeft:
# Line 793 | Line 967 | class SchedulerEdg(Scheduler):
967          pass
968  
969          if mustRenew:
970 <            common.logger.message( "No valid proxy found or timeleft too short!\n Creating a user proxy with default length of 24h\n")
971 <            cmd = 'voms-proxy-init -voms cms -valid 24:00'
970 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
971 >            cmd = 'voms-proxy-init -voms cms -valid 96:00'
972              try:
973                  # SL as above: damn it!
974                  out = os.system(cmd)
# Line 802 | Line 976 | class SchedulerEdg(Scheduler):
976              except:
977                  msg = "Unable to create a valid proxy!\n"
978                  raise CrabException(msg)
805 –            # cmd = 'grid-proxy-info -timeleft'
806 –            # cmd_out = runCommand(cmd,0,20)
979              pass
980  
981          ## now I do have a voms proxy valid, and I check the myproxy server
# Line 816 | Line 988 | class SchedulerEdg(Scheduler):
988          else:
989              # if myproxy exist but not long enough, renew
990              reTime = re.compile( r'timeleft: (\d+)' )
819 –            #print "<"+str(reTime.search( cmd_out ).group(1))+">"
991              if reTime.match( cmd_out ):
992                  time = reTime.search( line ).group(1)
993                  if time < minTimeLeftServer:

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines