ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.54 by slacapra, Thu Apr 20 16:37:12 2006 UTC vs.
Revision 1.64 by gutsche, Tue Jun 27 02:31:31 2006 UTC

# Line 152 | Line 152 | class SchedulerEdg(Scheduler):
152          sys.path.append(libPath)
153  
154          self.proxyValid=0
155 +
156 +        try:
157 +            self._taskId = cfg_params['taskId']
158 +        except:
159 +            self._taskId = ''
160 +
161          return
162      
163  
# Line 175 | Line 181 | class SchedulerEdg(Scheduler):
181          Returns part of a job script which does scheduler-specific work.
182          """
183          txt = ''
184 +        txt += "# job number (first parameter for job wrapper)\n"
185 +        txt += "NJob=$1\n"
186 +
187 +        txt += '# job identification to DashBoard \n'
188 +        txt += 'MonitorJobID=`echo ${NJob}_$EDG_WL_JOBID`\n'
189 +        txt += 'SyncGridJobId=`echo $EDG_WL_JOBID`\n'
190 +        txt += 'MonitorID=`echo ' + self._taskId + '`\n'
191 +        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
192 +        txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
193 +        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
194 +
195          txt += 'echo "middleware discovery " \n'
196 <        txt += 'if [ $VO_CMS_SW_DIR ]; then\n'
196 >        txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
197          txt += '    middleware=LCG \n'
198 +        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
199 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
200          txt += '    echo "middleware =$middleware" \n'
201          txt += 'elif [ $GRID3_APP_DIR ]; then\n'
202          txt += '    middleware=OSG \n'
203 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
204 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
205          txt += '    echo "middleware =$middleware" \n'
206          txt += 'elif [ $OSG_APP ]; then \n'
207          txt += '    middleware=OSG \n'
208 +        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
209 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
210          txt += '    echo "middleware =$middleware" \n'
211          txt += 'else \n'
212 <        txt += '    echo "SET_CMS_ENV 1 ==> middleware not identified" \n'
213 <        txt += '    echo "JOB_EXIT_STATUS = 1"\n'
214 <        txt += '    exit 1\n'
215 <        txt += 'fi\n'
216 <
212 >        txt += '    echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
213 >        txt += '    echo "JOB_EXIT_STATUS = 10030" \n'
214 >        txt += '    echo "JobExitCode=10030" | tee -a $RUNTIME_AREA/$repo \n'
215 >        txt += '    dumpStatus $RUNTIME_AREA/$repo \n'
216 >        txt += '    rm -f $RUNTIME_AREA/$repo \n'
217 >        txt += '    echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
218 >        txt += '    echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
219 >        txt += '    echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
220 >        txt += '    exit 1 \n'
221 >        txt += 'fi \n'
222 >
223 >        txt += '# report first time to DashBoard \n'
224 >        txt += 'dumpStatus $RUNTIME_AREA/$repo \n'
225 >        txt += 'rm -f $RUNTIME_AREA/$repo \n'
226 >        txt += 'echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
227 >        txt += 'echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
228 >        txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
229 >        
230          txt += '\n\n'
231  
196 –        txt += 'if [ $middleware == LCG ]; then \n'
197 –        txt += '    echo "SyncGridJobId=`echo $EDG_WL_JOBID`" | tee -a $RUNTIME_AREA/$repo\n'
198 –        txt += 'fi\n'
199 –
232          if int(self.copy_data) == 1:
233             if self.SE:
234                txt += 'export SE='+self.SE+'\n'
# Line 256 | Line 288 | class SchedulerEdg(Scheduler):
288          txt += '    if [ $OSG_JOB_CONTACT ]; then \n'
289          txt += '        CE=`echo $OSG_JOB_CONTACT | /usr/bin/awk -F\/ \'{print $1}\'` \n'
290          txt += '    else \n'
291 <        txt += '        echo "SET_ENV 1 ==> ERROR in setting CE name - OSG mode -" \n'
291 >        txt += '        echo "SET_CMS_ENV 10099 ==> OSG mode: ERROR in setting CE name from OSG_JOB_CONTACT" \n'
292 >        txt += '        echo "JOB_EXIT_STATUS = 10099" \n'
293 >        txt += '        echo "JobExitCode=10099" | tee -a $RUNTIME_AREA/$repo \n'
294 >        txt += '        dumpStatus $RUNTIME_AREA/$repo \n'
295 >        txt += '        rm -f $RUNTIME_AREA/$repo \n'
296 >        txt += '        echo "MonitorJobID=`echo $MonitorJobID`" | tee -a $RUNTIME_AREA/$repo \n'
297 >        txt += '        echo "SyncGridJobId=`echo $SyncGridJobId`" | tee -a $RUNTIME_AREA/$repo \n'
298 >        txt += '        echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
299          txt += '        exit 1 \n'
300          txt += '    fi \n'
301          txt += 'fi \n'
# Line 330 | Line 369 | class SchedulerEdg(Scheduler):
369             ## OLI_Daniele globus-* for OSG, lcg-* for LCG
370             txt += '        if [ $middleware == OSG ]; then\n'
371             txt += '           echo "globus-url-copy file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
372 <           txt += '           copy_exit_status=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
373 <           #txt += '           exitstring=`globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
372 >           txt += '           globus-url-copy file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1 \n'
373 >           txt += '           copy_exit_status=$? \n'
374             txt += '        elif [ $middleware == LCG ]; then \n'
375             txt += '           echo "lcg-cp --vo cms -t 1200 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
376 <           txt += '           copy_exit_status=`lcg-cp --vo cms -t 1200 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
377 <           #txt += '           exitstring=`lcg-cp --vo cms -t 30 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
376 >           txt += '           lcg-cp --vo cms -t 1200 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1\n'
377 >           txt += '           copy_exit_status=$? \n'
378             txt += '        fi \n'
340 –           #txt += '        copy_exit_status=$?\n'
379             txt += '        echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
380             txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
381             txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
# Line 497 | Line 535 | class SchedulerEdg(Scheduler):
535          [sites.append(it) for it in CEs if not sites.count(it)]
536  
537          common.logger.debug(5,"All Sites :"+str(sites))
538 +        common.logger.message("Matched Sites :"+str(sites))
539          return len(sites)
540  
541      def noMatchFound_(self, jdl):
# Line 593 | Line 632 | class SchedulerEdg(Scheduler):
632  
633          # Determine the output directory name
634          dir = common.work_space.resDir()
635 <        dir += os.getlogin()
635 >        dir += os.environ['USER']
636          dir += '_' + os.path.basename(id)
637          return dir
638  
# Line 633 | Line 672 | class SchedulerEdg(Scheduler):
672          jdl.write(jt_string)
673  
674          ### only one .sh  JDL has arguments:
636 –        firstEvent = common.jobDB.firstEvent(nj)
637 –        maxEvents = common.jobDB.maxEvents(nj)
638 –        jdl.write('Arguments = "' + str(nj+1)+' '+str(firstEvent)+' '+str(maxEvents)+'";\n')
675  
676 +        ### Fabio
677 +        jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n')
678          inp_box = 'InputSandbox = { '
679          inp_box = inp_box + '"' + script + '",'
680  
# Line 694 | Line 732 | class SchedulerEdg(Scheduler):
732  
733  
734          req='Requirements = '
735 +        noreq=req
736          req = req + jbt.getRequirements()
698 – #        ### if at least a CE exists ...
699 – #        if common.analisys_common_info['sites']:
700 – #           if common.analisys_common_info['sw_version']:
701 – #                req='Requirements = '
702 – #                req=req + 'Member("VO-cms-' + \
703 – #                     common.analisys_common_info['sw_version'] + \
704 – #                     '", other.GlueHostApplicationSoftwareRunTimeEnvironment)'
705 – #            if len(common.analisys_common_info['sites'])>0:
706 – #                req = req + ' && ('
707 – #                for i in range(len(common.analisys_common_info['sites'])):
708 – #                    req = req + 'other.GlueCEInfoHostName == "' \
709 – #                         + common.analisys_common_info['sites'][i] + '"'
710 – #                    if ( i < (int(len(common.analisys_common_info['sites']) - 1)) ):
711 – #                        req = req + ' || '
712 – #            req = req + ')'
737          #### and USER REQUIREMENT
738          if self.EDG_requirements:
739 <            if (req == 'Requirement = '):
740 <                req = req + self.EDG_requirements
741 <            else:
718 <                req = req +  ' && ' + self.EDG_requirements
739 >            if (req != noreq):
740 >                req = req +  ' && '
741 >            req = req + self.EDG_requirements
742          #### FEDE #####
743          if self.EDG_ce_white_list:
744              ce_white_list = string.split(self.EDG_ce_white_list,',')
745              #print "req = ", req
746              for i in range(len(ce_white_list)):
747                  if i == 0:
748 <                    if (req == 'Requirement = '):
749 <                        req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
750 <                    else:
728 <                        req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
748 >                    if (req != noreq):
749 >                        req = req +  ' && '
750 >                    req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
751                      pass
752                  else:
753                      req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
# Line 734 | Line 756 | class SchedulerEdg(Scheduler):
756          if self.EDG_ce_black_list:
757              ce_black_list = string.split(self.EDG_ce_black_list,',')
758              for ce in ce_black_list:
759 <                if (req == 'Requirement = '):
760 <                    req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
761 <                else:
740 <                    req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
759 >                if (req != noreq):
760 >                    req = req +  ' && '
761 >                req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
762                  pass
763 +
764          ###############
765 +        clockTime=480
766          if self.EDG_clock_time:
767 <            if (req == 'Requirement = '):
768 <                req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
769 <            else:
770 <                req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
767 >            clockTime= self.EDG_clock_time
768 >        if (req != noreq):
769 >            req = req + ' && '
770 >        req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))'
771  
772 +        cpuTime=1000
773          if self.EDG_cpu_time:
774 <            if (req == 'Requirement = '):
775 <                req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
776 <            else:
777 <                req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
778 <        if (req != 'Requirement = '):
774 >            cpuTime=self.EDG_cpu_time
775 >        if (req != noreq):
776 >            req = req + ' && '
777 >        req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))'
778 >
779 >        if (req != noreq):
780              req = req + ';\n'
781              jdl.write(req)
782                                                                                                                                                              
# Line 793 | Line 818 | class SchedulerEdg(Scheduler):
818          pass
819  
820          if mustRenew:
821 <            common.logger.message( "No valid proxy found or timeleft too short!\n Creating a user proxy with default length of 24h\n")
822 <            cmd = 'voms-proxy-init -voms cms -valid 24:00'
821 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
822 >            cmd = 'voms-proxy-init -voms cms -valid 96:00'
823              try:
824                  # SL as above: damn it!
825                  out = os.system(cmd)

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines