ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.55 by corvo, Wed May 3 08:44:30 2006 UTC vs.
Revision 1.66 by fanzago, Fri Jun 30 15:12:30 2006 UTC

# Line 152 | Line 152 | class SchedulerEdg(Scheduler):
152          sys.path.append(libPath)
153  
154          self.proxyValid=0
155 +
156 +        try:
157 +            self._taskId = cfg_params['taskId']
158 +        except:
159 +            self._taskId = ''
160 +
161          return
162      
163  
# Line 175 | Line 181 | class SchedulerEdg(Scheduler):
181          Returns part of a job script which does scheduler-specific work.
182          """
183          txt = ''
184 +        txt += "# job number (first parameter for job wrapper)\n"
185 +        txt += "NJob=$1\n"
186 +
187 +        txt += '# job identification to DashBoard \n'
188 +        txt += 'MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`\n'
189 +        txt += 'SyncGridJobId=`echo $EDG_WL_JOBID`\n'
190 +        txt += 'MonitorID=`echo ' + self._taskId + '`\n'
191 +        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
192 +        txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
193 +        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
194 +
195          txt += 'echo "middleware discovery " \n'
196 <        txt += 'if [ $VO_CMS_SW_DIR ]; then\n'
196 >        txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
197          txt += '    middleware=LCG \n'
198 +        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
199 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
200          txt += '    echo "middleware =$middleware" \n'
201          txt += 'elif [ $GRID3_APP_DIR ]; then\n'
202          txt += '    middleware=OSG \n'
203 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
204 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
205          txt += '    echo "middleware =$middleware" \n'
206          txt += 'elif [ $OSG_APP ]; then \n'
207          txt += '    middleware=OSG \n'
208 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
209 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
210          txt += '    echo "middleware =$middleware" \n'
211          txt += 'else \n'
212 <        txt += '    echo "SET_CMS_ENV 1 ==> middleware not identified" \n'
213 <        txt += '    echo "JOB_EXIT_STATUS = 1"\n'
214 <        txt += '    exit 1\n'
215 <        txt += 'fi\n'
216 <
212 >        txt += '    echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
213 >        txt += '    echo "JOB_EXIT_STATUS = 10030" \n'
214 >        txt += '    echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n'
215 >        txt += '    dumpStatus $RUNTIME_AREA/$repo \n'
216 >        txt += '    rm -f $RUNTIME_AREA/$repo \n'
217 >        txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
218 >        txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
219 >        txt += '    exit 1 \n'
220 >        txt += 'fi \n'
221 >
222 >        txt += '# report first time to DashBoard \n'
223 >        txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
224 >        txt += 'rm -f $RUNTIME_AREA/$repo \n'
225 >        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
226 >        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
227 >        
228          txt += '\n\n'
229  
196 –        txt += 'if [ $middleware == LCG ]; then \n'
197 –        txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
198 –        txt += 'fi\n'
199 –
230          if int(self.copy_data) == 1:
231             if self.SE:
232                txt += 'export SE='+self.SE+'\n'
# Line 256 | Line 286 | class SchedulerEdg(Scheduler):
286          txt += '    if [ $OSG_JOB_CONTACT ]; then \n'
287          txt += '        CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n'
288          txt += '    else \n'
289 <        txt += '        echo "SET_ENV 1 ==> ERROR in setting CE name - OSG mode -" \n'
289 >        txt += '        echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n'
290 >        txt += '        echo "JOB_EXIT_STATUS = 10099" \n'
291 >        txt += '        echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n'
292 >        txt += '        dumpStatus $RUNTIME_AREA/$repo \n'
293 >        txt += '        rm -f $RUNTIME_AREA/$repo \n'
294 >        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
295 >        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
296          txt += '        exit 1 \n'
297          txt += '    fi \n'
298          txt += 'fi \n'
# Line 286 | Line 322 | class SchedulerEdg(Scheduler):
322   ### changed by georgia (put a loop copying more than one input files per jobs)          
323             txt += '   for input_file in $cur_file_list \n'
324             txt += '   do \n'
325 <           txt += '    lcg-cp --vo $VO lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
325 >           txt += '    lcg-cp --vo $VO --verbose lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
326             txt += '    copy_input_exit_status=$?\n'
327             txt += '    echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n'
328             txt += '    if [ $copy_input_exit_status -ne 0 ]; then \n'
# Line 298 | Line 334 | class SchedulerEdg(Scheduler):
334   ### copy a set of PU ntuples (same for each jobs -- but accessed randomly)
335             txt += '   for file in $cur_pu_list \n'
336             txt += '   do \n'
337 <           txt += '    lcg-cp --vo $VO lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
337 >           txt += '    lcg-cp --vo $VO --verbose lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
338             txt += '    copy_input_exit_status=$?\n'
339             txt += '    echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n'
340             txt += '    if [ $copy_input_pu_exit_status -ne 0 ]; then \n'
# Line 324 | Line 360 | class SchedulerEdg(Scheduler):
360             txt += '#\n'
361             txt += '#   Copy output to SE = $SE\n'
362             txt += '#\n'
363 <           txt += 'if [ $exe_result -eq 0 ]; then\n'
363 >           #txt += 'if [ $exe_result -eq 0 ]; then\n'
364             txt += '    for out_file in $file_list ; do\n'
365             txt += '        echo "Trying to copy output file to $SE "\n'
366             ## OLI_Daniele globus-* for OSG, lcg-* for LCG
367             txt += '        if [ $middleware == OSG ]; then\n'
368             txt += '           echo "globus-url-copy file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
369 <           txt += '           copy_exit_status=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
370 <           #txt += '           exitstring=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
369 >           txt += '           globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1 \n'
370 >           txt += '           copy_exit_status=$? \n'
371             txt += '        elif [ $middleware == LCG ]; then \n'
372 +           txt += '           echo "#######################################" \n'
373 +           txt += '           echo "#######################################" \n'
374 +           txt += '           echo "directory: " \n'
375 +           txt += '           pwd \n'
376 +           txt += '           echo "files:" \n'
377 +           txt += '           ls \n'
378 +           txt += '           echo "#######################################" \n'
379 +           txt += '           echo "#######################################" \n'
380             txt += '           echo "lcg-cp --vo cms -t 1200 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
381 <           txt += '           copy_exit_status=`lcg-cp --vo cms -t 1200 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
382 <           #txt += '           exitstring=`lcg-cp --vo cms -t 30 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
381 >           txt += '           lcg-cp --vo cms --verbose -t 1200 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1\n'
382 >           txt += '           copy_exit_status=$? \n'
383             txt += '        fi \n'
340 –           #txt += '        copy_exit_status=$?\n'
384             txt += '        echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
385             txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
386 +           txt += '        exit_status=$copy_exit_status"\n'
387             txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
388             txt += '            echo "Problems with SE = $SE"\n'
389             txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
# Line 351 | Line 395 | class SchedulerEdg(Scheduler):
395             txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
396             txt += '         fi\n'
397             txt += '     done\n'
398 <           txt += 'fi\n'
398 >           #txt += 'fi\n'
399          return txt
400  
401      def wsRegisterOutput(self):
# Line 371 | Line 415 | class SchedulerEdg(Scheduler):
415             txt += '#\n'
416             txt += '#  Register output to LFC\n'
417             txt += '#\n'
418 <           txt += '   if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n'
418 >           #txt += '   if [[ $exe_result -eq 0 && $copy_exit_status -eq 0 ]]; then\n'
419 >           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
420             txt += '      for out_file in $file_list ; do\n'
421             txt += '         echo "Trying to register the output file into LFC"\n'
422             txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO sfn://$SE$SE_PATH/$out_file"\n'
# Line 395 | Line 440 | class SchedulerEdg(Scheduler):
440             txt += '         fi \n'
441             txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
442             txt += '      done\n'
443 <           txt += '   elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n'
443 >           #txt += '   elif [[ $exe_result -eq 0 && $copy_exit_status -ne 0 ]]; then \n'
444 >           txt += '   else \n'
445             txt += '      echo "Trying to copy output file to CloseSE"\n'
446             txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
447             txt += '      for out_file in $file_list ; do\n'
# Line 405 | Line 451 | class SchedulerEdg(Scheduler):
451             txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
452             txt += '         echo "STAGE_OUT = $register_exit_status"\n'
453             txt += '         if [ $register_exit_status -ne 0 ]; then \n'
454 <           txt += '            echo "Problems with CloseSE" \n'
454 >           txt += '            echo "Problems with CloseSE or Catalog" \n'
455             txt += '         else \n'
456             txt += '            echo "The program was successfully executed"\n'
457             txt += '            echo "SE = $CLOSE_SE"\n'
# Line 413 | Line 459 | class SchedulerEdg(Scheduler):
459             txt += '         fi \n'
460             txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
461             txt += '      done\n'
462 <           txt += '   else\n'
463 <           txt += '      echo "Problem with the executable"\n'
462 >           #txt += '   else\n'
463 >           #txt += '      echo "Problem with the executable"\n'
464             txt += '   fi \n'
465 +           txt += '   exit_status=$register_exit_status"\n'
466             txt += 'fi \n'
467          return txt
468  
# Line 497 | Line 544 | class SchedulerEdg(Scheduler):
544          [sites.append(it) for it in CEs if not sites.count(it)]
545  
546          common.logger.debug(5,"All Sites :"+str(sites))
547 +        common.logger.message("Matched Sites :"+str(sites))
548          return len(sites)
549  
550      def noMatchFound_(self, jdl):
# Line 633 | Line 681 | class SchedulerEdg(Scheduler):
681          jdl.write(jt_string)
682  
683          ### only one .sh  JDL has arguments:
636 –        firstEvent = common.jobDB.firstEvent(nj)
637 –        maxEvents = common.jobDB.maxEvents(nj)
638 –        jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n')
684  
685 +        ### Fabio
686 +        jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n')
687          inp_box = 'InputSandbox = { '
688          inp_box = inp_box + '"' + script + '",'
689  
# Line 694 | Line 741 | class SchedulerEdg(Scheduler):
741  
742  
743          req='Requirements = '
744 +        noreq=req
745          req = req + jbt.getRequirements()
698 – #        ### if at least a CE exists ...
699 – #        if common.analisys_common_info['sites']:
700 – #           if common.analisys_common_info['sw_version']:
701 – #                req='Requirements = '
702 – #                req=req + 'Member("VO-cms-' + \
703 – #                     common.analisys_common_info['sw_version'] + \
704 – #                     '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
705 – #            if len(common.analisys_common_info['sites'])>0:
706 – #                req = req + ' && ('
707 – #                for i in range(len(common.analisys_common_info['sites'])):
708 – #                    req = req + 'other.GlueCEInfoHostName == "' \
709 – #                         + common.analisys_common_info['sites'][i] + '"'
710 – #                    if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
711 – #                        req = req + ' || '
712 – #            req = req + ')'
746          #### and USER REQUIREMENT
747          if self.EDG_requirements:
748 <            if (req == 'Requirement = '):
749 <                req = req + self.EDG_requirements
750 <            else:
718 <                req = req +  ' && ' + self.EDG_requirements
748 >            if (req != noreq):
749 >                req = req +  ' && '
750 >            req = req + self.EDG_requirements
751          #### FEDE #####
752          if self.EDG_ce_white_list:
753              ce_white_list = string.split(self.EDG_ce_white_list,',')
754              #print "req = ", req
755              for i in range(len(ce_white_list)):
756                  if i == 0:
757 <                    if (req == 'Requirement = '):
758 <                        req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
759 <                    else:
728 <                        req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
757 >                    if (req != noreq):
758 >                        req = req +  ' && '
759 >                    req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
760                      pass
761                  else:
762                      req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
# Line 734 | Line 765 | class SchedulerEdg(Scheduler):
765          if self.EDG_ce_black_list:
766              ce_black_list = string.split(self.EDG_ce_black_list,',')
767              for ce in ce_black_list:
768 <                if (req == 'Requirement = '):
769 <                    req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
770 <                else:
740 <                    req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
768 >                if (req != noreq):
769 >                    req = req +  ' && '
770 >                req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
771                  pass
772 +
773          ###############
774 +        clockTime=480
775          if self.EDG_clock_time:
776 <            if (req == 'Requirement = '):
777 <                req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
778 <            else:
779 <                req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
776 >            clockTime= self.EDG_clock_time
777 >        if (req != noreq):
778 >            req = req + ' && '
779 >        req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))'
780  
781 +        cpuTime=1000
782          if self.EDG_cpu_time:
783 <            if (req == 'Requirement = '):
784 <                req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
785 <            else:
786 <                req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
787 <        if (req != 'Requirement = '):
783 >            cpuTime=self.EDG_cpu_time
784 >        if (req != noreq):
785 >            req = req + ' && '
786 >        req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))'
787 >
788 >        if (req != noreq):
789              req = req + ';\n'
790              jdl.write(req)
791                                                                                                                                                              
# Line 793 | Line 827 | class SchedulerEdg(Scheduler):
827          pass
828  
829          if mustRenew:
830 <            common.logger.message( "No valid proxy found or timeleft too short!\n Creating a user proxy with default length of 24h\n")
831 <            cmd = 'voms-proxy-init -voms cms -valid 24:00'
830 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
831 >            cmd = 'voms-proxy-init -voms cms -valid 96:00'
832              try:
833                  # SL as above: damn it!
834                  out = os.system(cmd)

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines