ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.49 by spiga, Fri Apr 7 12:58:38 2006 UTC vs.
Revision 1.62 by slacapra, Tue Jun 20 15:37:51 2006 UTC

# Line 2 | Line 2 | from Scheduler import Scheduler
2   from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5 + from EdgConfig import *
6   import common
7  
8   import os, sys, time
# Line 30 | Line 31 | class SchedulerEdg(Scheduler):
31              self.edg_config = ''
32              self.edg_config_vo = ''
33  
34 +        try:
35 +            self.proxyServer = cfg_params["EDG.proxy_server"]
36 +        except KeyError:
37 +            self.proxyServer = 'myproxy.cern.ch'
38 +        common.logger.debug(5,'Setting myproxy server to '+self.proxyServer)
39  
40          try: self.LCG_version = cfg_params["EDG.lcg_version"]
41          except KeyError: self.LCG_version = '2'
# Line 146 | Line 152 | class SchedulerEdg(Scheduler):
152          sys.path.append(libPath)
153  
154          self.proxyValid=0
155 +
156 +        try:
157 +            self._taskId = cfg_params['taskId']
158 +        except:
159 +            self._taskId = ''
160 +
161          return
162      
163  
# Line 169 | Line 181 | class SchedulerEdg(Scheduler):
181          Returns part of a job script which does scheduler-specific work.
182          """
183          txt = ''
184 +        txt += "# job number (first parameter for job wrapper)\n"
185 +        txt += "NJob=$1\n"
186 +
187 +        txt += '# job identification to DashBoard \n'
188 +        txt += 'echo "MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo \n'
189 +        txt += 'echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo \n'
190 +        txt += 'echo "MonitorID=`echo ' + self._taskId + '`" | tee -a $RUNTIME_AREA/$repo\n'
191 +
192          txt += 'echo "middleware discovery " \n'
193 <        txt += 'if [ $VO_CMS_SW_DIR ]; then\n'
193 >        txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
194          txt += '    middleware=LCG \n'
195 +        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
196 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
197          txt += '    echo "middleware =$middleware" \n'
198          txt += 'elif [ $GRID3_APP_DIR ]; then\n'
199          txt += '    middleware=OSG \n'
200 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
201 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
202          txt += '    echo "middleware =$middleware" \n'
203          txt += 'elif [ $OSG_APP ]; then \n'
204          txt += '    middleware=OSG \n'
205 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
206 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
207          txt += '    echo "middleware =$middleware" \n'
208          txt += 'else \n'
209 <        txt += '    echo "SET_CMS_ENV 1 ==> middleware not identified" \n'
210 <        txt += '    echo "JOB_EXIT_STATUS = 1"\n'
211 <        txt += '    exit 1\n'
212 <        txt += 'fi\n'
209 >        txt += '    echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
210 >        txt += '    echo "JOB_EXIT_STATUS = 10030" \n'
211 >        txt += '    echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n'
212 >        txt += '    dumpStatus $RUNTIME_AREA/$repo \n'
213 >        txt += '    exit 1 \n'
214 >        txt += 'fi \n'
215  
216 <        txt += '\n\n'
216 >        txt += '# report first time to DashBoard \n'
217 >        txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
218  
219 <        txt += 'if [ $middleware == LCG ]; then \n'
191 <        txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
192 <        txt += 'fi\n'
219 >        txt += '\n\n'
220  
221          if int(self.copy_data) == 1:
222             if self.SE:
# Line 250 | Line 277 | class SchedulerEdg(Scheduler):
277          txt += '    if [ $OSG_JOB_CONTACT ]; then \n'
278          txt += '        CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n'
279          txt += '    else \n'
280 <        txt += '        echo "SET_ENV 1 ==> ERROR in setting CE name - OSG mode -" \n'
280 >        txt += '        echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n'
281 >        txt += '        echo "JOB_EXIT_STATUS = 10099" \n'
282 >        txt += '        echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n'
283 >        txt += '        dumpStatus $RUNTIME_AREA/$repo \n'
284          txt += '        exit 1 \n'
285          txt += '    fi \n'
286          txt += 'fi \n'
# Line 491 | Line 521 | class SchedulerEdg(Scheduler):
521          [sites.append(it) for it in CEs if not sites.count(it)]
522  
523          common.logger.debug(5,"All Sites :"+str(sites))
524 +        common.logger.message("Matched Sites :"+str(sites))
525          return len(sites)
526  
527      def noMatchFound_(self, jdl):
# Line 587 | Line 618 | class SchedulerEdg(Scheduler):
618  
619          # Determine the output directory name
620          dir = common.work_space.resDir()
621 <        dir += os.getlogin()
621 >        dir += os.environ['USER']
622          dir += '_' + os.path.basename(id)
623          return dir
624  
# Line 627 | Line 658 | class SchedulerEdg(Scheduler):
658          jdl.write(jt_string)
659  
660          ### only one .sh  JDL has arguments:
630 –        firstEvent = common.jobDB.firstEvent(nj)
631 –        maxEvents = common.jobDB.maxEvents(nj)
632 –        jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n')
661  
662 +        ### Fabio
663 +        jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n')
664          inp_box = 'InputSandbox = { '
665          inp_box = inp_box + '"' + script + '",'
666  
# Line 688 | Line 718 | class SchedulerEdg(Scheduler):
718  
719  
720          req='Requirements = '
721 +        noreq=req
722          req = req + jbt.getRequirements()
692 – #        ### if at least a CE exists ...
693 – #        if common.analisys_common_info['sites']:
694 – #           if common.analisys_common_info['sw_version']:
695 – #                req='Requirements = '
696 – #                req=req + 'Member("VO-cms-' + \
697 – #                     common.analisys_common_info['sw_version'] + \
698 – #                     '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
699 – #            if len(common.analisys_common_info['sites'])>0:
700 – #                req = req + ' && ('
701 – #                for i in range(len(common.analisys_common_info['sites'])):
702 – #                    req = req + 'other.GlueCEInfoHostName == "' \
703 – #                         + common.analisys_common_info['sites'][i] + '"'
704 – #                    if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
705 – #                        req = req + ' || '
706 – #            req = req + ')'
723          #### and USER REQUIREMENT
724          if self.EDG_requirements:
725 <            if (req == 'Requirement = '):
726 <                req = req + self.EDG_requirements
727 <            else:
712 <                req = req +  ' && ' + self.EDG_requirements
725 >            if (req != noreq):
726 >                req = req +  ' && '
727 >            req = req + self.EDG_requirements
728          #### FEDE #####
729          if self.EDG_ce_white_list:
730              ce_white_list = string.split(self.EDG_ce_white_list,',')
731              #print "req = ", req
732              for i in range(len(ce_white_list)):
733                  if i == 0:
734 <                    if (req == 'Requirement = '):
735 <                        req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
736 <                    else:
722 <                        req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
734 >                    if (req != noreq):
735 >                        req = req +  ' && '
736 >                    req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
737                      pass
738                  else:
739                      req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
# Line 728 | Line 742 | class SchedulerEdg(Scheduler):
742          if self.EDG_ce_black_list:
743              ce_black_list = string.split(self.EDG_ce_black_list,',')
744              for ce in ce_black_list:
745 <                if (req == 'Requirement = '):
746 <                    req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
747 <                else:
734 <                    req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
745 >                if (req != noreq):
746 >                    req = req +  ' && '
747 >                req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
748                  pass
749 +
750          ###############
751 +        clockTime=480
752          if self.EDG_clock_time:
753 <            if (req == 'Requirement = '):
754 <                req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
755 <            else:
756 <                req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
753 >            clockTime= self.EDG_clock_time
754 >        if (req != noreq):
755 >            req = req + ' && '
756 >        req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))'
757  
758 +        cpuTime=1000
759          if self.EDG_cpu_time:
760 <            if (req == 'Requirement = '):
761 <                req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
762 <            else:
763 <                req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
764 <        if (req != 'Requirement = '):
760 >            cpuTime=self.EDG_cpu_time
761 >        if (req != noreq):
762 >            req = req + ' && '
763 >        req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))'
764 >
765 >        if (req != noreq):
766              req = req + ';\n'
767              jdl.write(req)
768                                                                                                                                                              
# Line 755 | Line 772 | class SchedulerEdg(Scheduler):
772              jdl.write('RetryCount = '+self.EDG_retry_count+';\n')
773              pass
774  
775 +        jdl.write('MyProxyServer = "' + self.proxyServer + '";\n')
776 +
777          jdl.close()
778          return
779  
# Line 764 | Line 783 | class SchedulerEdg(Scheduler):
783          """
784          if (self.proxyValid): return
785          timeleft = -999
786 <        minTimeLeft=10 # in hours
787 <        cmd = 'voms-proxy-info -exists -valid '+str(minTimeLeft)+':00'
788 <        # SL Here I have to use os.system since the stupid command exit with >0 if no valid proxy is found
789 <        cmd_out = os.system(cmd)
790 <        if (cmd_out>0):
791 <            common.logger.message( "No valid proxy found or timeleft too short!\n Creating a user proxy with default length of 24h\n")
792 <            cmd = 'voms-proxy-init -voms cms -valid 100:00'
786 >        minTimeLeft=10*3600 # in seconds
787 >
788 >        minTimeLeftServer = 100 # in hours
789 >
790 >        #cmd = 'voms-proxy-info -exists -valid '+str(minTimeLeft)+':00'
791 >        #cmd = 'voms-proxy-info -timeleft'
792 >        mustRenew = 0
793 >        timeLeftLocal = runCommand('voms-proxy-info -timeleft')
794 >        timeLeftServer = -999
795 >        if not timeLeftLocal or int(timeLeftLocal) <= 0 or not isInt(timeLeftLocal):
796 >            mustRenew = 1
797 >        else:
798 >            timeLeftServer = runCommand('voms-proxy-info -actimeleft | head -1')
799 >            if not timeLeftServer or not isInt(timeLeftServer):
800 >                mustRenew = 1
801 >            elif timeLeftLocal<minTimeLeft or timeLeftServer<minTimeLeft:
802 >                mustRenew = 1
803 >            pass
804 >        pass
805 >
806 >        if mustRenew:
807 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
808 >            cmd = 'voms-proxy-init -voms cms -valid 96:00'
809              try:
810                  # SL as above: damn it!
811                  out = os.system(cmd)
# Line 781 | Line 816 | class SchedulerEdg(Scheduler):
816              # cmd = 'grid-proxy-info -timeleft'
817              # cmd_out = runCommand(cmd,0,20)
818              pass
819 +
820 +        ## now I do have a voms proxy valid, and I check the myproxy server
821 +        renewProxy = 0
822 +        cmd = 'myproxy-info -d -s '+self.proxyServer
823 +        cmd_out = runCommand(cmd,0,20)
824 +        if not cmd_out:
825 +            common.logger.message('No credential delegated to myproxy server '+self.proxyServer+' will do now')
826 +            renewProxy = 1
827 +        else:
828 +            # if myproxy exist but not long enough, renew
829 +            reTime = re.compile( r'timeleft: (\d+)' )
830 +            #print "<"+str(reTime.search( cmd_out ).group(1))+">"
831 +            if reTime.match( cmd_out ):
832 +                time = reTime.search( line ).group(1)
833 +                if time < minTimeLeftServer:
834 +                    renewProxy = 1
835 +                    common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
836 +                pass
837 +            pass
838 +        
839 +        # if not, create one.
840 +        if renewProxy:
841 +            cmd = 'myproxy-init -d -n -s '+self.proxyServer
842 +            out = os.system(cmd)
843 +            if (out>0):
844 +                raise CrabException("Unable to delegate the proxy to myproxyserver "+self.proxyServer+" !\n")
845 +            pass
846 +
847 +        # cache proxy validity
848          self.proxyValid=1
849          return
850  
851      def configOpt_(self):
852          edg_ui_cfg_opt = ' '
853          if self.edg_config:
854 <          edg_ui_cfg_opt = ' -c ' + self.edg_config + ' '
854 >            edg_ui_cfg_opt = ' -c ' + self.edg_config + ' '
855          if self.edg_config_vo:
856 <          edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
856 >            edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
857          return edg_ui_cfg_opt

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines