ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.128 by spiga, Sat Jul 14 08:40:15 2007 UTC vs.
Revision 1.133 by fanzago, Wed Aug 15 14:08:18 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   from EdgConfig import *
6 + from BlackWhiteListParser import BlackWhiteListParser
7   import common
8  
9   import os, sys, time
# Line 21 | Line 22 | class SchedulerEdg(Scheduler):
22          return
23  
24      def configure(self, cfg_params):
24 – ##Addedd Daniele
25 – ######################################        
25  
26 <        SEBlackList = []
27 <        try:
29 <            tmpBad = string.split(cfg_params['EDG.se_black_list'],',')
30 <            for tmp in tmpBad:
31 <                tmp=string.strip(tmp)
32 <                SEBlackList.append(tmp)
33 <        except KeyError:
34 <            pass
35 <        common.logger.debug(5,'SEBlackList: '+str(SEBlackList))
36 <        self.reSEBlackList=[]
37 <        for bad in SEBlackList:
38 <            self.reSEBlackList.append(re.compile( string.lower(bad) ))
39 <
40 <        SEWhiteList = []
41 <        try:
42 <            tmpGood = string.split(cfg_params['EDG.se_white_list'],',')
43 <            for tmp in tmpGood:
44 <                tmp=string.strip(tmp)
45 <                SEWhiteList.append(tmp)
46 <        except KeyError:
47 <            pass
48 <        common.logger.debug(5,'SEWhiteList: '+str(SEWhiteList))
49 <        self.reSEWhiteList=[]
50 <        for good in SEWhiteList:
51 <            self.reSEWhiteList.append(re.compile( string.lower(good) ))
26 >        # init BlackWhiteListParser
27 >        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
28  
29 +        self.proxyValid=0
30 +        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
31 +        except KeyError: self.dontCheckProxy = 0
32  
54 – ######################################        
33          try:
34              RB=cfg_params["EDG.rb"]
35              self.rb_param_file=self.rb_configure(RB)
# Line 116 | Line 94 | class SchedulerEdg(Scheduler):
94          ########### FEDE FOR DBS2 ##############################
95          try:
96              self.publish_data = cfg_params["USER.publish_data"]
97 +            self.checkProxy()
98              if int(self.publish_data) == 1:
99                  try:
100                      self.publish_data_name = cfg_params['USER.publish_data_name']
# Line 214 | Line 193 | class SchedulerEdg(Scheduler):
193          libPath=os.path.join(path, "lib", "python")
194          sys.path.append(libPath)
195  
217 –        self.proxyValid=0
218 –
196          try:
197              self._taskId = cfg_params['taskId']
198          except:
# Line 227 | Line 204 | class SchedulerEdg(Scheduler):
204          try: self.schedulerName = cfg_params['CRAB.scheduler']
205          except KeyError: self.scheduler = ''
206  
230 –        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
231 –        except KeyError: self.dontCheckProxy = 0
232 –
207          return
208      
209  
# Line 550 | Line 524 | class SchedulerEdg(Scheduler):
524  
525             txt += '    for out_file in $file_list ; do\n'
526             txt += '        echo "Trying to copy output file to $SE using srmcp"\n'
527 <           txt += '        echo "mkdir -p $HOME/.srmconfig"\n'
528 <           txt += '        mkdir -p $HOME/.srmconfig\n'
527 >           # txt += '        echo "mkdir -p $HOME/.srmconfig"\n'
528 >           # txt += '        mkdir -p $HOME/.srmconfig\n'
529             txt += '        if [ $middleware == LCG ]; then\n'
530             txt += '           echo "srmcp -retry_num 3 -retry_timeout 480000 file:///`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
531             txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 480000 file:///\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
# Line 569 | Line 543 | class SchedulerEdg(Scheduler):
543             txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
544             txt += '            echo "srmcp failed, attempting lcg-cp."\n'
545             if common.logger.debugLevel() >= 5:
572 –               ########### FEDE CHANGES TO WRITE IN SRM LNL.INFN.IT #################
546                 txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
547                 txt += '            exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
548                 #txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
# Line 588 | Line 561 | class SchedulerEdg(Scheduler):
561             txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
562             txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
563             txt += '               echo "srmcp and lcg-cp and failed!"\n'
564 <           txt += '               SE=""\n'
565 <           txt += '               echo "SE = $SE"\n'
566 <           txt += '               SE_PATH=""\n'
567 <           txt += '               echo "SE_PATH = $SE_PATH"\n'
564 >           ################### FEDE moved out of for ##############
565 >           #txt += '               SE=""\n'
566 >           #txt += '               echo "SE = $SE"\n'
567 >           #txt += '               SE_PATH=""\n'
568 >           #txt += '               echo "SE_PATH = $SE_PATH"\n'
569 >           ########################################################
570             txt += '            else\n'
571             txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
572             txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
# Line 607 | Line 582 | class SchedulerEdg(Scheduler):
582             txt += '            echo "srmcp succeeded"\n'
583             txt += '         fi\n'
584             txt += '     done\n'
585 +           txt += '     if [ $copy_exit_status -ne 0 ]; then\n'
586 +           txt += '           SE=""\n'
587 +           txt += '           echo "SE = $SE"\n'
588 +           txt += '           SE_PATH=""\n'
589 +           txt += '           echo "SE_PATH = $SE_PATH"\n'
590 +           txt += '     fi\n'
591             txt += '     exit_status=$copy_exit_status\n'
592          return txt
593  
# Line 690 | Line 671 | class SchedulerEdg(Scheduler):
671          cmd_out = runCommand(cmd)
672          return cmd_out
673  
693 – ##Addedd Daniele
694 – # #######################################################################
695 –    def checkBlackList(self, Sites, fileblocks):
696 –        """
697 –        select sites that are not excluded by the user (via SE black list)
698 –        """
699 –        goodSites = []
700 –        for aSite in Sites:
701 –            common.logger.debug(10,'Site '+aSite)
702 –            good=1
703 –            for re in self.reSEBlackList:
704 –                if re.search(string.lower(aSite)):
705 –                    common.logger.debug(5,'SE in black list, skipping site '+aSite)
706 –                    good=0
707 –                pass
708 –            if good: goodSites.append(aSite)
709 –        if len(goodSites) == 0:
710 –            msg = "No sites hosting the block %s after BlackList" % fileblocks
711 –            common.logger.debug(5,msg)
712 –            common.logger.debug(5,"Proceeding without this block.\n")
713 –        else:
714 –            common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via BlackList are "+str(goodSites)+"\n")
715 –        return goodSites
716 –
717 – # #######################################################################
718 –    def checkWhiteList(self, Sites, fileblocks):
719 –        """
720 –        select sites that are defined by the user (via SE white list)
721 –        """
722 –        if len(self.reSEWhiteList)==0: return Sites
723 –        goodSites = []
724 –        for aSite in Sites:
725 –            good=0
726 –            for re in self.reSEWhiteList:
727 –                if re.search(string.lower(aSite)):
728 –                    common.logger.debug(5,'SE in white list, adding site '+aSite)
729 –                    good=1
730 –                pass
731 –            if good: goodSites.append(aSite)
732 –        
733 –        if len(goodSites) == 0:
734 –            msg = "No sites hosting the block %s after WhiteList" % fileblocks
735 –            common.logger.debug(5,msg)
736 –            common.logger.debug(5,"Proceeding without this block.\n")
737 –        else:
738 –            common.logger.debug(5,"Selected sites for block "+str(fileblocks)+" via WhiteList are "+str(goodSites)+"\n")
739 –      
740 –        return goodSites
741 –
742 – # #######################################################################
674      def findSites_(self, n):
675          itr4 =[]
676  
# Line 751 | Line 682 | class SchedulerEdg(Scheduler):
682          itr = ''
683          if sites != [""]:#CarlosDaniele
684              ##Addedd Daniele
685 <            replicas = self.checkBlackList(sites,n)
685 >            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
686              if len(replicas)!=0:
687 <                replicas = self.checkWhiteList(replicas,n)
687 >                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
688                
689              if len(replicas)==0:
690                  msg = 'No sites remaining that host any part of the requested data! Exiting... '

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines