ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.112 by corvo, Tue Jan 9 18:33:57 2007 UTC vs.
Revision 1.133.2.4 by fanzago, Fri Oct 12 12:56:30 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   from EdgConfig import *
6 + from BlackWhiteListParser import BlackWhiteListParser
7   import common
8  
9 < import os, sys, time, gzip
9 > import os, sys, time
10  
11   class SchedulerEdg(Scheduler):
12      def __init__(self):
# Line 18 | Line 19 | class SchedulerEdg(Scheduler):
19                        "owner","parent_job", "reason","resubmitted","rsl","seed",\
20                        "stateEnterTime","stateEnterTimes","subjob_failed", \
21                        "user tags" , "status" , "status_code","hierarchy"]
21 –        
22          return
23  
24      def configure(self, cfg_params):
25  
26 +        # init BlackWhiteListParser
27 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
28 +
29 +        self.proxyValid=0
30 +        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
31 +        except KeyError: self.dontCheckProxy = 0
32 +
33          try:
34              RB=cfg_params["EDG.rb"]
35              self.rb_param_file=self.rb_configure(RB)
# Line 45 | Line 52 | class SchedulerEdg(Scheduler):
52          except KeyError:
53              self.role = None
54              
55 <        try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 <        except KeyError: self.LCG_version = '2'
55 >        #try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 >        #except KeyError: self.LCG_version = '2'
57  
58          try:
59              self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
# Line 80 | Line 87 | class SchedulerEdg(Scheduler):
87          except KeyError: self.copy_data = 0
88  
89          if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ):
90 <           msg = 'Warning: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n'
90 >           msg = 'Error: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n'
91             msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
92             raise CrabException(msg)
93  
94 <        try:
95 <            self.lfc_host = cfg_params['EDG.lfc_host']
96 <        except KeyError:
97 <            msg = "Error. The [EDG] section does not have 'lfc_host' value"
98 <            msg = msg + " it's necessary to know the LFC host name"
99 <            common.logger.message(msg)
93 <            raise CrabException(msg)
94 <        try:
95 <            self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
96 <        except KeyError:
97 <            msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
98 <            msg = msg + " it's necessary to know the catalog type"
99 <            common.logger.message(msg)
100 <            raise CrabException(msg)
101 <        try:
102 <            self.lfc_home = cfg_params['EDG.lfc_home']
103 <        except KeyError:
104 <            msg = "Error. The [EDG] section does not have 'lfc_home' value"
105 <            msg = msg + " it's necessary to know the home catalog dir"
106 <            common.logger.message(msg)
107 <            raise CrabException(msg)
108 <      
94 >        if ( int(self.return_data) == 1 and int(self.copy_data) == 1 ):
95 >           msg = 'Error: return_data and copy_data cannot be set both to 1\n'
96 >           msg = msg + 'Please modify return_data or copy_data value in your crab.cfg file\n'
97 >           raise CrabException(msg)
98 >
99 >        ########### FEDE FOR DBS2 ##############################
100          try:
101 <            self.register_data = cfg_params["USER.register_data"]
102 <            if int(self.register_data) == 1:
101 >            self.publish_data = cfg_params["USER.publish_data"]
102 >            self.checkProxy()
103 >            if int(self.publish_data) == 1:
104                  try:
105 <                    self.LFN = cfg_params['USER.lfn_dir']
105 >                    self.publish_data_name = cfg_params['USER.publish_data_name']
106                  except KeyError:
107 <                    msg = "Error. The [USER] section does not have 'lfn_dir' value"
108 <                    msg = msg + " it's necessary for LCF registration"
109 <                    common.logger.message(msg)
107 >                    msg = "Error. The [USER] section does not have 'publish_data_name'"
108 >                    raise CrabException(msg)
109 >                try:
110 >                    tmp = runCommand("voms-proxy-info -identity")
111 >                    tmp = string.split(tmp,'/')
112 >                    reCN=re.compile(r'CN=')
113 >                    for t in tmp:
114 >                        if reCN.match(t):
115 >                            self.UserGridName=string.strip((t.replace('CN=','')).replace(' ',''))
116 >                        
117 >                    #self.UserGridName = string.strip(runCommand("voms-proxy-info -identity | awk -F\'CN\' \'{print $2$3$4}\' | tr -d \'=/ \'"))
118 >                except:
119 >                    msg = "Error. Problem with voms-proxy-info -identity command"
120                      raise CrabException(msg)
121 <        except KeyError: self.register_data = 0
121 >        except KeyError: self.publish_data = 0
122  
123 <        if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
124 <           msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
123 >        if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
124 >           msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
125             msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
126             common.logger.message(msg)
127             raise CrabException(msg)
128 +        #################################################
129 +
130 +        #try:
131 +        #    self.lfc_host = cfg_params['EDG.lfc_host']
132 +        #except KeyError:
133 +        #    msg = "Error. The [EDG] section does not have 'lfc_host' value"
134 +        #    msg = msg + " it's necessary to know the LFC host name"
135 +        #    common.logger.message(msg)
136 +        #    raise CrabException(msg)
137 +        #try:
138 +        #    self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
139 +        #except KeyError:
140 +        #    msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
141 +        #    msg = msg + " it's necessary to know the catalog type"
142 +        #    common.logger.message(msg)
143 +        #    raise CrabException(msg)
144 +        #try:
145 +        #    self.lfc_home = cfg_params['EDG.lfc_home']
146 +        #except KeyError:
147 +        #    msg = "Error. The [EDG] section does not have 'lfc_home' value"
148 +        #    msg = msg + " it's necessary to know the home catalog dir"
149 +        #    common.logger.message(msg)
150 +        #    raise CrabException(msg)
151 +      
152 +        #try:
153 +        #    self.register_data = cfg_params["USER.register_data"]
154 +        #    if int(self.register_data) == 1:
155 +        #        try:
156 +        #            self.LFN = cfg_params['USER.lfn_dir']
157 +        #        except KeyError:
158 +        #            msg = "Error. The [USER] section does not have 'lfn_dir' value"
159 +        #            msg = msg + " it's necessary for LCF registration"
160 +        #            common.logger.message(msg)
161 +        #            raise CrabException(msg)
162 +        #except KeyError: self.register_data = 0
163 +
164 +        #if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
165 +        #   msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
166 +        #   msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
167 +        #   common.logger.message(msg)
168 +        #   raise CrabException(msg)
169  
170          try: self.EDG_requirements = cfg_params['EDG.requirements']
171          except KeyError: self.EDG_requirements = ''
# Line 155 | Line 198 | class SchedulerEdg(Scheduler):
198          libPath=os.path.join(path, "lib", "python")
199          sys.path.append(libPath)
200  
158 –        self.proxyValid=0
159 –
201          try:
202              self._taskId = cfg_params['taskId']
203          except:
# Line 168 | Line 209 | class SchedulerEdg(Scheduler):
209          try: self.schedulerName = cfg_params['CRAB.scheduler']
210          except KeyError: self.scheduler = ''
211  
171 –        try: self.dontCheckProxy=cfg_params["EDG.dont_check_proxy"]
172 –        except KeyError: self.dontCheckProxy = 0
173 –
212          return
213      
214  
# Line 184 | Line 222 | class SchedulerEdg(Scheduler):
222          self.edg_config_vo = edgConfig.configVO()
223  
224          if (self.edg_config and self.edg_config_vo != ''):
225 <            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";'
225 >            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";\n'
226              #print "rb_param_file = ", self.rb_param_file
227          return self.rb_param_file
228        
# Line 292 | Line 330 | class SchedulerEdg(Scheduler):
330          txt += 'echo "middleware discovery " \n'
331          txt += 'if [ $GRID3_APP_DIR ]; then\n'
332          txt += '    middleware=OSG \n'
333 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
333 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
334 >        txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
335 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
336 >        txt += '    else\n'
337 >        txt += '        echo "not reporting SyncCE";\n'
338 >        txt += '    fi\n';
339          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
340          txt += '    echo "middleware =$middleware" \n'
341          txt += 'elif [ $OSG_APP ]; then \n'
342          txt += '    middleware=OSG \n'
343 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
343 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
344 >        txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
345 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
346 >        txt += '    else\n'
347 >        txt += '        echo "not reporting SyncCE";\n'
348 >        txt += '    fi\n';
349          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
350          txt += '    echo "middleware =$middleware" \n'
351          txt += 'elif [ $VO_CMS_SW_DIR ]; then \n'
352          txt += '    middleware=LCG \n'
353 <        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
353 >   #     txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
354 >        txt += '    echo "SyncCE=`glite-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
355          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
356          txt += '    echo "middleware =$middleware" \n'
357          txt += 'else \n'
# Line 324 | Line 373 | class SchedulerEdg(Scheduler):
373          
374          txt += '\n\n'
375  
376 <        if int(self.copy_data) == 1:
377 <           if self.SE:
378 <              txt += 'export SE='+self.SE+'\n'
379 <              txt += 'echo "SE = $SE"\n'
380 <           if self.SE_PATH:
381 <              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
382 <              txt += 'export SE_PATH='+self.SE_PATH+'\n'
383 <              txt += 'echo "SE_PATH = $SE_PATH"\n'
376 > #        if int(self.copy_data) == 1:
377 > #           if self.SE:
378 > #              txt += 'export SE='+self.SE+'\n'
379 > #              txt += 'echo "SE = $SE"\n'
380 > #           if self.SE_PATH:
381 > #              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
382 > #              txt += 'export SE_PATH='+self.SE_PATH+'\n'
383 > #              txt += 'echo "SE_PATH = $SE_PATH"\n'
384  
385          txt += 'export VO='+self.VO+'\n'
386          ### add some line for LFC catalog setting
387 <        txt += 'if [ $middleware == LCG ]; then \n'
388 <        txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
389 <        txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
390 <        txt += '    fi\n'
391 <        txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
392 <        txt += '        export LFC_HOST='+self.lfc_host+'\n'
393 <        txt += '    fi\n'
394 <        txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
395 <        txt += '        export LFC_HOME='+self.lfc_home+'\n'
396 <        txt += '    fi\n'
397 <        txt += 'elif [ $middleware == OSG ]; then\n'
398 <        txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
399 <        txt += 'fi\n'
387 >        #txt += 'if [ $middleware == LCG ]; then \n'
388 >        #txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
389 >        #txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
390 >        #txt += '    fi\n'
391 >        #txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
392 >        #txt += '        export LFC_HOST='+self.lfc_host+'\n'
393 >        #txt += '    fi\n'
394 >        #txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
395 >        #txt += '        export LFC_HOME='+self.lfc_home+'\n'
396 >        #txt += '    fi\n'
397 >        #txt += 'elif [ $middleware == OSG ]; then\n'
398 >        #txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
399 >        #txt += 'fi\n'
400          #####
401 <        if int(self.register_data) == 1:
402 <           txt += 'if [ $middleware == LCG ]; then \n'
403 <           txt += '    export LFN='+self.LFN+'\n'
404 <           txt += '    lfc-ls $LFN\n'
405 <           txt += '    result=$?\n'
406 <           txt += '    echo $result\n'
407 <           ### creation of LFN dir in LFC catalog, under /grid/cms dir  
408 <           txt += '    if [ $result != 0 ]; then\n'
409 <           txt += '       lfc-mkdir $LFN\n'
410 <           txt += '       result=$?\n'
411 <           txt += '       echo $result\n'
412 <           txt += '    fi\n'
413 <           txt += 'elif [ $middleware == OSG ]; then\n'
414 <           txt += '    echo " Files registration to be implemented for OSG"\n'
415 <           txt += 'fi\n'
416 <           txt += '\n'
417 <
418 <           if self.VO:
419 <              txt += 'export VO='+self.VO+'\n'
420 <           if self.LFN:
421 <              txt += 'if [ $middleware == LCG ]; then \n'
422 <              txt += '    export LFN='+self.LFN+'\n'
423 <              txt += 'fi\n'
375 <              txt += '\n'
401 >        #if int(self.register_data) == 1:
402 >        #   txt += 'if [ $middleware == LCG ]; then \n'
403 >        #   txt += '    export LFN='+self.LFN+'\n'
404 >        #   txt += '    lfc-ls $LFN\n'
405 >        #   txt += '    result=$?\n'
406 >        #   txt += '    echo $result\n'
407 >        #   ### creation of LFN dir in LFC catalog, under /grid/cms dir  
408 >        #   txt += '    if [ $result != 0 ]; then\n'
409 >        #   txt += '       lfc-mkdir $LFN\n'
410 >        #   txt += '       result=$?\n'
411 >        #   txt += '       echo $result\n'
412 >        #   txt += '    fi\n'
413 >        #   txt += 'elif [ $middleware == OSG ]; then\n'
414 >        #   txt += '    echo " Files registration to be implemented for OSG"\n'
415 >        #   txt += 'fi\n'
416 >        #   txt += '\n'
417 >        #   if self.VO:
418 >        #      txt += 'export VO='+self.VO+'\n'
419 >        #   if self.LFN:
420 >        #      txt += 'if [ $middleware == LCG ]; then \n'
421 >        #      txt += '    export LFN='+self.LFN+'\n'
422 >        #      txt += 'fi\n'
423 >        #      txt += '\n'
424  
425          txt += 'if [ $middleware == LCG ]; then\n'
426 <        txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
426 >    #    txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
427 >        txt += '    CloseCEs=`glite-brokerinfo getCE`\n'
428          txt += '    echo "CloseCEs = $CloseCEs"\n'
429          txt += '    CE=`echo $CloseCEs | sed -e "s/:.*//"`\n'
430          txt += '    echo "CE = $CE"\n'
# Line 449 | Line 498 | class SchedulerEdg(Scheduler):
498          Write a CopyResults part of a job script, e.g.
499          to copy produced output into a storage element.
500          """
501 <        txt = ''
453 <        if int(self.copy_data) == 1:
454 <           txt += '#\n'
455 <           txt += '#   Copy output to SE = $SE\n'
456 <           txt += '#\n'
457 <           txt += '    if [ $middleware == OSG ]; then\n'
458 <           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
459 <           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
460 <           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
461 <           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
462 <           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
463 <           txt += '    fi \n'
464 <
465 <           txt += '    for out_file in $file_list ; do\n'
466 <           txt += '        echo "Trying to copy output file to $SE using srmcp"\n'
467 <           txt += '        echo "mkdir -p $HOME/.srmconfig"\n'
468 <           txt += '        mkdir -p $HOME/.srmconfig\n'
469 <           txt += '        if [ $middleware == LCG ]; then\n'
470 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 480000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
471 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 480000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
472 <           txt += '        elif [ $middleware == OSG ]; then\n'
473 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
474 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
475 <           txt += '        fi \n'
476 <           txt += '        copy_exit_status=$?\n'
477 <           txt += '        echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
478 <           txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
479 <
480 <           txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
481 <           txt += '            echo "Possible problem with SE = $SE"\n'
482 <           txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
483 <           txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
484 <           txt += '            echo "srmcp failed, attempting lcg-cp."\n'
485 <           if common.logger.debugLevel() >= 5:
486 <               txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
487 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
488 <           else:              
489 <               txt += '            echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
490 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
491 <           txt += '            copy_exit_status=$?\n'
492 <           txt += '            echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
493 <           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
494 <
495 <           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
496 <           txt += '               echo "Problems with SE = $SE"\n'
497 <           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
498 <           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
499 <           txt += '               echo "srmcp and lcg-cp and failed!"\n'
500 <           txt += '            else\n'
501 <           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
502 <           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
503 <           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
504 <           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
505 <           txt += '               echo "lcg-cp succeeded"\n'
506 <           txt += '            fi\n'
507 <           txt += '        else\n'
508 <           txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
509 <           txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
510 <           txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
511 <           txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
512 <           txt += '            echo "srmcp succeeded"\n'
513 <           txt += '         fi\n'
514 <           txt += '     done\n'
515 <        return txt
501 >        txt = '\n'
502  
503 <    def wsRegisterOutput(self):
504 <        """
505 <        Returns part of a job script which does scheduler-specific work.
520 <        """
503 >        txt += '#\n'
504 >        txt += '# COPY OUTPUT FILE TO SE\n'
505 >        txt += '#\n\n'
506  
507 <        txt = ''
508 <        if int(self.register_data) == 1:
509 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
510 <           txt += 'if [ $middleware == OSG ]; then\n'
511 <           txt += '   #\n'
512 <           txt += '   #   Register output to LFC deactivated in OSG mode\n'
513 <           txt += '   #\n'
514 <           txt += '   echo "Register output to LFC deactivated in OSG mode"\n'
515 <           txt += 'elif [ $middleware == LCG ]; then \n'
516 <           txt += '#\n'
517 <           txt += '#  Register output to LFC\n'
518 <           txt += '#\n'
519 <           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
520 <           txt += '      for out_file in $file_list ; do\n'
521 <           txt += '         echo "Trying to register the output file into LFC"\n'
522 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
523 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
524 <           txt += '         register_exit_status=$?\n'
525 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
526 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
527 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
528 <           txt += '            echo "Problems with the registration to LFC" \n'
529 <           txt += '            echo "Try with srm protocol" \n'
530 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
531 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
532 <           txt += '            register_exit_status=$?\n'
533 <           txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
534 <           txt += '            echo "STAGE_OUT = $register_exit_status"\n'
535 <           txt += '            if [ $register_exit_status -ne 0 ]; then \n'
536 <           txt += '               echo "Problems with the registration into LFC" \n'
537 <           txt += '            fi \n'
538 <           txt += '         else \n'
539 <           txt += '            echo "output registered to LFC"\n'
540 <           txt += '         fi \n'
541 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
542 <           txt += '      done\n'
543 <           txt += '   else \n'
544 <           txt += '      echo "Trying to copy output file to CloseSE"\n'
545 <           txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
546 <           txt += '      for out_file in $file_list ; do\n'
547 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
548 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
549 <           txt += '         register_exit_status=$?\n'
550 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
551 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
552 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
553 <           txt += '            echo "Problems with CloseSE or Catalog" \n'
554 <           txt += '         else \n'
555 <           txt += '            echo "The program was successfully executed"\n'
556 <           txt += '            echo "SE = $CLOSE_SE"\n'
572 <           txt += '            echo "LFN for the file is LFN=${LFN}/$out_file"\n'
573 <           txt += '         fi \n'
574 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
575 <           txt += '      done\n'
576 <           txt += '   fi \n'
577 <           txt += '   exit_status=$register_exit_status\n'
578 <           txt += 'fi \n'
507 >        SE_PATH=''
508 >        if int(self.copy_data) == 1:
509 >            if self.SE:
510 >                txt += 'export SE='+self.SE+'\n'
511 >                txt += 'echo "SE = $SE"\n'
512 >            if self.SE_PATH:
513 >                if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
514 >                SE_PATH=self.SE_PATH
515 >            if int(self.publish_data) == 1:
516 >                txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
517 >                path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
518 >                SE_PATH = SE_PATH + path_add
519 >            txt += 'export SE_PATH='+SE_PATH+'\n'
520 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
521 >            
522 >            txt += 'echo "####################################################"\n'
523 >            txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
524 >            txt += 'echo "####################################################"\n'
525 >            
526 >            txt += 'if [ $output_exit_status -eq 60302 ]; then\n'
527 >            txt += '    echo "--> No output file to copy to $SE"\n'
528 >            txt += '    copy_exit_status=$output_exit_status\n'
529 >            txt += '    echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
530 >            txt += 'else\n'
531 >            txt += '    for out_file in $file_list ; do\n'
532 >            txt += '        echo "Trying to copy output file to $SE"\n'
533 >            txt += '        cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
534 >            txt += '        copy_exit_status=$?\n'
535 >            txt += '        echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
536 >            txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
537 >            txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
538 >            txt += '            echo "Problem copying $out_file to $SE $SE_PATH"\n'
539 >            txt += '            echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
540 >            txt += '            copy_exit_status=60307\n'
541 >            txt += '        else\n'
542 >            txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
543 >            txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
544 >            txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
545 >            txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
546 >            txt += '        fi\n'
547 >            txt += '    done\n'
548 >            txt += '    if [ $copy_exit_status -ne 0 ]; then\n'
549 >            txt += '        SE=""\n'
550 >            txt += '        echo "SE = $SE"\n'
551 >            txt += '        SE_PATH=""\n'
552 >            txt += '        echo "SE_PATH = $SE_PATH"\n'
553 >            txt += '    fi\n'
554 >            txt += 'fi\n'
555 >            txt += 'exit_status=$copy_exit_status\n'
556 >            pass
557          return txt
558  
559      def loggingInfo(self, id):
# Line 593 | Line 571 | class SchedulerEdg(Scheduler):
571          cmd_out = runCommand(cmd)
572          return cmd_out
573  
596 –    ##### FEDE ######        
574      def findSites_(self, n):
575          itr4 =[]
576 +
577          sites = common.jobDB.destination(n)
578 <        if len(sites)>0 and sites[0]=="Any":
578 >
579 >        if len(sites)>0 and sites[0]=="":
580              return itr4
581 +
582          itr = ''
583          if sites != [""]:#CarlosDaniele
584 <            for site in sites:
584 >            ##Addedd Daniele
585 >            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
586 >            if len(replicas)!=0:
587 >                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
588 >              
589 >            if len(replicas)==0:
590 >                itr = itr + 'target.GlueSEUniqueID=="NONE" '
591 >                #msg = 'No sites remaining that host any part of the requested data! Exiting... '
592 >                #raise CrabException(msg)
593 >            #####        
594 >           # for site in sites:
595 >            for site in replicas:
596                  #itr = itr + 'target.GlueSEUniqueID==&quot;'+site+'&quot; || '
597                  itr = itr + 'target.GlueSEUniqueID=="'+site+'" || '
598              itr = itr[0:-4]
# Line 624 | Line 615 | class SchedulerEdg(Scheduler):
615          jbt = job.type()
616          
617          inp_sandbox = jbt.inputSandbox(index)
618 <        out_sandbox = jbt.outputSandbox(index)
618 >        #out_sandbox = jbt.outputSandbox(index)
619          """
620          [end] FIX-ME
621          """
# Line 640 | Line 631 | class SchedulerEdg(Scheduler):
631          dir = string.split(common.work_space.topDir(), '/')
632          taskName = dir[len(dir)-2]
633    
643 –        to_writeReq = ''
634          to_write = ''
635  
636          req=' '
# Line 700 | Line 690 | class SchedulerEdg(Scheduler):
690          taskName = dir[len(dir)-2]
691  
692          xml.write(str(title))
693 <        xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
693 >        #xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
694 >
695 >        #xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n')
696 >        x509_cmd = 'ls /tmp/x509up_u`id -u`'
697 >        x509=runCommand(x509_cmd).strip()
698 >        xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + str(x509) + '">\n')
699          xml.write(jt_string)
700          
701          if (to_write != ''):
# Line 727 | Line 722 | class SchedulerEdg(Scheduler):
722          indy: here itr4
723          '''
724          
725 <
726 <        xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
725 >        xml.write('<chain name="' +str(taskName)+'__ITR1_" scheduler="'+str(self.schedulerName)+'">\n')
726 >       # xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
727          xml.write(jt_string)
728  
729          #executable
# Line 752 | Line 747 | class SchedulerEdg(Scheduler):
747                  pass
748              pass
749  
750 <        if (not jbt.additional_inbox_files == []):
750 > #        if (not jbt.additional_inbox_files == []):
751   #            inp_box = inp_box + ','
752 <            for addFile in jbt.additional_inbox_files:
753 <                #addFile = os.path.abspath(addFile)
754 <                inp_box = inp_box+''+addFile+','
755 <                pass
752 > #            for addFile in jbt.additional_inbox_files:
753 > #                #addFile = os.path.abspath(addFile)
754 > #                inp_box = inp_box+''+addFile+','
755 > #                pass
756  
757          if inp_box[-1] == ',' : inp_box = inp_box[:-1]
758          inp_box = '<infiles> <![CDATA[\n' + inp_box + '\n]]> </infiles>\n'
# Line 825 | Line 820 | class SchedulerEdg(Scheduler):
820          if (self.proxyValid): return
821  
822          ### Just return if asked to do so
823 <        if (self.dontCheckProxy):
823 >        if (self.dontCheckProxy==1):
824              self.proxyValid=1
825              return
826  
832 –        timeleft = -999
827          minTimeLeft=10*3600 # in seconds
828  
829          minTimeLeftServer = 100 # in hours
# Line 874 | Line 868 | class SchedulerEdg(Scheduler):
868              common.logger.message('No credential delegated to myproxy server '+self.proxyServer+' will do now')
869              renewProxy = 1
870          else:
871 <            # if myproxy exist but not long enough, renew
872 <            reTime = re.compile( r'timeleft: (\d+)' )
873 <            #print "<"+str(reTime.search( cmd_out ).group(1))+">"
874 <            if reTime.match( cmd_out ):
875 <                time = reTime.search( line ).group(1)
876 <                if time < minTimeLeftServer:
877 <                    renewProxy = 1
878 <                    common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
871 >            ## minimum time: 5 days
872 >            minTime = 4 * 24 * 3600
873 >            ## regex to extract the right information
874 >            myproxyRE = re.compile("timeleft: (?P<hours>[\\d]*):(?P<minutes>[\\d]*):(?P<seconds>[\\d]*)")
875 >            for row in cmd_out.split("\n"):
876 >                g = myproxyRE.search(row)
877 >                if g:
878 >                    hours = g.group("hours")
879 >                    minutes = g.group("minutes")
880 >                    seconds = g.group("seconds")
881 >                    timeleft = int(hours)*3600 + int(minutes)*60 + int(seconds)
882 >                    if timeleft < minTime:
883 >                        renewProxy = 1
884 >                        common.logger.message('Your proxy will expire in:\n\t'+hours+' hours '+minutes+' minutes '+seconds+' seconds\n')
885 >                        common.logger.message('Need to renew it:')
886 >                    pass
887                  pass
888              pass
889          
# Line 904 | Line 906 | class SchedulerEdg(Scheduler):
906          if self.edg_config_vo:
907              edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
908          return edg_ui_cfg_opt
909 +
910 +    def submitTout(self, list):
911 +        return 120
912 +
913 +

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines