ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.133 by fanzago, Wed Aug 15 14:08:18 2007 UTC vs.
Revision 1.136 by spiga, Mon Sep 17 16:42:55 2007 UTC

# Line 204 | Line 204 | class SchedulerEdg(Scheduler):
204          try: self.schedulerName = cfg_params['CRAB.scheduler']
205          except KeyError: self.scheduler = ''
206  
207 +        self.UseServer=0
208 +        try:
209 +            self.UseServer=int(cfg_params['CRAB.server_mode'])
210 +        except KeyError:
211 +            pass
212 +
213          return
214      
215  
# Line 484 | Line 490 | class SchedulerEdg(Scheduler):
490          Write a CopyResults part of a job script, e.g.
491          to copy produced output into a storage element.
492          """
493 <        txt = ''
493 >        txt = '\n'
494 >
495 >        txt += '#\n'
496 >        txt += '# COPY OUTPUT FILE TO SE\n'
497 >        txt += '#\n\n'
498  
489 –        ##### FEDE MOVED FROM SET_ENVIRONMENT ##############
490 –        
499          SE_PATH=''
500          if int(self.copy_data) == 1:
501 <           if self.SE:
502 <              txt += 'export SE='+self.SE+'\n'
503 <              txt += 'echo "SE = $SE"\n'
504 <           if self.SE_PATH:
505 <              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
506 <              SE_PATH=self.SE_PATH
507 <              ####### FEDE FOR DBS2
508 <              if int(self.publish_data) == 1:
509 <                  txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
510 <                  #txt += 'subject=`voms-proxy-info -subject | awk -F\'CN\' \'{print $2$3$4}\' | tr -d \'=/ \'` \n'
511 <                  #txt += 'echo "subject = $subject" \n'
512 <                  #path_add = '${subject}/'+ self.publish_data_name +'_${PSETHASH}/'
513 <                  path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
514 <                  SE_PATH = SE_PATH + path_add
515 <
516 <              txt += 'export SE_PATH='+SE_PATH+'\n'
517 <              txt += 'echo "SE_PATH = $SE_PATH"\n'
518 <
519 <        ##########################################################  
520 <
521 <        #if int(self.copy_data) == 1:
522 <           txt += '#\n'
523 <           txt += '#   Copy output to SE = $SE\n'
524 <           txt += '#\n'
525 <           txt += '    if [ $middleware == OSG ]; then\n'
526 <           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
527 <           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
528 <           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
529 <           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
530 <           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
531 <           txt += '    fi \n'
532 <
533 <           txt += '    for out_file in $file_list ; do\n'
534 <           txt += '        echo "Trying to copy output file to $SE using srmcp"\n'
535 <           # txt += '        echo "mkdir -p $HOME/.srmconfig"\n'
536 <           # txt += '        mkdir -p $HOME/.srmconfig\n'
537 <           txt += '        if [ $middleware == LCG ]; then\n'
538 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 480000 file:///`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
539 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 480000 file:///\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
540 <           txt += '        elif [ $middleware == OSG ]; then\n'
541 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:///`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
542 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:///\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
543 <           txt += '        fi \n'
544 <           txt += '        copy_exit_status=$?\n'
545 <           txt += '        echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
546 <           txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
539 <
540 <           txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
541 <           txt += '            echo "Possible problem with SE = $SE"\n'
542 <           txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
543 <           txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
544 <           txt += '            echo "srmcp failed, attempting lcg-cp."\n'
545 <           if common.logger.debugLevel() >= 5:
546 <               txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
547 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
548 <               #txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
549 <               #txt += '            exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
550 <           else:              
551 <               txt += '            echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
552 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
553 <               #txt += '            echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
554 <               #txt += '            exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
555 <           txt += '            copy_exit_status=$?\n'
556 <           txt += '            echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
557 <           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
558 <
559 <           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
560 <           txt += '               echo "Problems with SE = $SE"\n'
561 <           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
562 <           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
563 <           txt += '               echo "srmcp and lcg-cp and failed!"\n'
564 <           ################### FEDE moved out of for ##############
565 <           #txt += '               SE=""\n'
566 <           #txt += '               echo "SE = $SE"\n'
567 <           #txt += '               SE_PATH=""\n'
568 <           #txt += '               echo "SE_PATH = $SE_PATH"\n'
569 <           ########################################################
570 <           txt += '            else\n'
571 <           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
572 <           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
573 <           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
574 <           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
575 <           txt += '               echo "lcg-cp succeeded"\n'
576 <           txt += '            fi\n'
577 <           txt += '        else\n'
578 <           txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
579 <           txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
580 <           txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
581 <           txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
582 <           txt += '            echo "srmcp succeeded"\n'
583 <           txt += '         fi\n'
584 <           txt += '     done\n'
585 <           txt += '     if [ $copy_exit_status -ne 0 ]; then\n'
586 <           txt += '           SE=""\n'
587 <           txt += '           echo "SE = $SE"\n'
588 <           txt += '           SE_PATH=""\n'
589 <           txt += '           echo "SE_PATH = $SE_PATH"\n'
590 <           txt += '     fi\n'
591 <           txt += '     exit_status=$copy_exit_status\n'
592 <        return txt
593 <
594 <    def wsRegisterOutput(self):
595 <        """
596 <        Returns part of a job script which does scheduler-specific work.
597 <        """
598 <
599 <        txt = ''
600 <        if int(self.register_data) == 1:
601 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
602 <           txt += 'if [ $middleware == OSG ]; then\n'
603 <           txt += '   #\n'
604 <           txt += '   #   Register output to LFC deactivated in OSG mode\n'
605 <           txt += '   #\n'
606 <           txt += '   echo "Register output to LFC deactivated in OSG mode"\n'
607 <           txt += 'elif [ $middleware == LCG ]; then \n'
608 <           txt += '#\n'
609 <           txt += '#  Register output to LFC\n'
610 <           txt += '#\n'
611 <           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
612 <           txt += '      for out_file in $file_list ; do\n'
613 <           txt += '         echo "Trying to register the output file into LFC"\n'
614 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
615 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
616 <           txt += '         register_exit_status=$?\n'
617 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
618 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
619 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
620 <           txt += '            echo "Problems with the registration to LFC" \n'
621 <           txt += '            echo "Try with srm protocol" \n'
622 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
623 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
624 <           txt += '            register_exit_status=$?\n'
625 <           txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
626 <           txt += '            echo "STAGE_OUT = $register_exit_status"\n'
627 <           txt += '            if [ $register_exit_status -ne 0 ]; then \n'
628 <           txt += '               echo "Problems with the registration into LFC" \n'
629 <           txt += '            fi \n'
630 <           txt += '         else \n'
631 <           txt += '            echo "output registered to LFC"\n'
632 <           txt += '         fi \n'
633 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
634 <           txt += '      done\n'
635 <           txt += '   else \n'
636 <           txt += '      echo "Trying to copy output file to CloseSE"\n'
637 < #          txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
638 <           txt += '      CLOSE_SE=`glite-brokerinfo getCloseSEs | head -1`\n'
639 <           txt += '      for out_file in $file_list ; do\n'
640 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
641 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
642 <           txt += '         register_exit_status=$?\n'
643 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
644 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
645 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
646 <           txt += '            echo "Problems with CloseSE or Catalog" \n'
647 <           txt += '         else \n'
648 <           txt += '            echo "The program was successfully executed"\n'
649 <           txt += '            echo "SE = $CLOSE_SE"\n'
650 <           txt += '            echo "LFN for the file is LFN=${LFN}/$out_file"\n'
651 <           txt += '         fi \n'
652 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
653 <           txt += '      done\n'
654 <           txt += '   fi \n'
655 <           txt += '   exit_status=$register_exit_status\n'
656 <           txt += 'fi \n'
501 >            if self.SE:
502 >                txt += 'export SE='+self.SE+'\n'
503 >                txt += 'echo "SE = $SE"\n'
504 >            if self.SE_PATH:
505 >                if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
506 >                SE_PATH=self.SE_PATH
507 >            if int(self.publish_data) == 1:
508 >                txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
509 >                path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
510 >                SE_PATH = SE_PATH + path_add
511 >            txt += 'export SE_PATH='+SE_PATH+'\n'
512 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
513 >            
514 >            txt += 'echo "####################################################"\n'
515 >            txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
516 >            txt += 'echo "####################################################"\n'
517 >            
518 >            txt += 'for out_file in $file_list ; do\n'
519 >            txt += '   echo "Trying to copy output file to $SE using srmcp"\n'
520 >            txt += '   cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
521 >            txt += '   copy_exit_status=$?\n'
522 >            txt += '   echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
523 >            txt += '   echo "STAGE_OUT = $copy_exit_status"\n'
524 >            
525 >            txt += '   if [ $copy_exit_status -ne 0 ]; then\n'
526 >            txt += '       echo "Problem copying $out_file to $SE $SE_PATH"\n'
527 >            txt += '       echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
528 >            txt += '       echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
529 >            txt += '       copy_exit_status=60307\n'
530 >            
531 >            txt += '   else\n'
532 >            txt += '       echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
533 >            txt += '       echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
534 >            txt += '       echo "output copied into $SE/$SE_PATH directory"\n'
535 >            txt += '       echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
536 >            txt += '       echo "srmcp succeeded"\n'
537 >            txt += '    fi\n'
538 >            txt += 'done\n'
539 >            txt += 'if [ $copy_exit_status -ne 0 ]; then\n'
540 >            txt += '      SE=""\n'
541 >            txt += '      echo "SE = $SE"\n'
542 >            txt += '      SE_PATH=""\n'
543 >            txt += '      echo "SE_PATH = $SE_PATH"\n'
544 >            txt += 'fi\n'
545 >            txt += 'exit_status=$copy_exit_status\n'
546 >            pass
547          return txt
548  
549      def loggingInfo(self, id):
# Line 792 | Line 682 | class SchedulerEdg(Scheduler):
682          #xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
683  
684          #xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n')
685 <        xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + os.environ["X509_USER_PROXY"] + '">\n')
685 >        if (self.UseServer== 9999):
686 >            xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + os.environ["X509_USER_PROXY"] + '">\n')
687 >        else:
688 >            xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="NONE">\n')
689 >
690          xml.write(jt_string)
691          
692          if (to_write != ''):

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines