ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.108 by slacapra, Wed Dec 6 10:27:30 2006 UTC vs.
Revision 1.143 by mcinquil, Mon Oct 8 19:01:00 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   from EdgConfig import *
6 + from BlackWhiteListParser import BlackWhiteListParser
7   import common
8  
9 < import os, sys, time, gzip
9 > import os, sys, time
10  
11   class SchedulerEdg(Scheduler):
12      def __init__(self):
# Line 18 | Line 19 | class SchedulerEdg(Scheduler):
19                        "owner","parent_job", "reason","resubmitted","rsl","seed",\
20                        "stateEnterTime","stateEnterTimes","subjob_failed", \
21                        "user tags" , "status" , "status_code","hierarchy"]
21 –        
22          return
23  
24      def configure(self, cfg_params):
25  
26 +        # init BlackWhiteListParser
27 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
28 +
29 +        self.proxyValid=0
30 +        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
31 +        except KeyError: self.dontCheckProxy = 0
32 +
33          try:
34              RB=cfg_params["EDG.rb"]
35              self.rb_param_file=self.rb_configure(RB)
# Line 45 | Line 52 | class SchedulerEdg(Scheduler):
52          except KeyError:
53              self.role = None
54              
55 <        try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 <        except KeyError: self.LCG_version = '2'
55 >        #try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 >        #except KeyError: self.LCG_version = '2'
57  
58          try:
59              self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
# Line 84 | Line 91 | class SchedulerEdg(Scheduler):
91             msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
92             raise CrabException(msg)
93  
94 <        try:
88 <            self.lfc_host = cfg_params['EDG.lfc_host']
89 <        except KeyError:
90 <            msg = "Error. The [EDG] section does not have 'lfc_host' value"
91 <            msg = msg + " it's necessary to know the LFC host name"
92 <            common.logger.message(msg)
93 <            raise CrabException(msg)
94 <        try:
95 <            self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
96 <        except KeyError:
97 <            msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
98 <            msg = msg + " it's necessary to know the catalog type"
99 <            common.logger.message(msg)
100 <            raise CrabException(msg)
101 <        try:
102 <            self.lfc_home = cfg_params['EDG.lfc_home']
103 <        except KeyError:
104 <            msg = "Error. The [EDG] section does not have 'lfc_home' value"
105 <            msg = msg + " it's necessary to know the home catalog dir"
106 <            common.logger.message(msg)
107 <            raise CrabException(msg)
108 <      
94 >        ########### FEDE FOR DBS2 ##############################
95          try:
96 <            self.register_data = cfg_params["USER.register_data"]
97 <            if int(self.register_data) == 1:
96 >            self.publish_data = cfg_params["USER.publish_data"]
97 >            self.checkProxy()
98 >            if int(self.publish_data) == 1:
99                  try:
100 <                    self.LFN = cfg_params['USER.lfn_dir']
100 >                    self.publish_data_name = cfg_params['USER.publish_data_name']
101                  except KeyError:
102 <                    msg = "Error. The [USER] section does not have 'lfn_dir' value"
116 <                    msg = msg + " it's necessary for LCF registration"
117 <                    common.logger.message(msg)
102 >                    msg = "Error. The [USER] section does not have 'publish_data_name'"
103                      raise CrabException(msg)
104 <        except KeyError: self.register_data = 0
104 >                try:
105 >                    tmp = runCommand("voms-proxy-info -identity")
106 >                    tmp = string.split(tmp,'/')
107 >                    reCN=re.compile(r'CN=')
108 >                    for t in tmp:
109 >                        if reCN.match(t):
110 >                            self.UserGridName=string.strip((t.replace('CN=','')).replace(' ',''))
111 >                        
112 >                    #self.UserGridName = string.strip(runCommand("voms-proxy-info -identity | awk -F\'CN\' \'{print $2$3$4}\' | tr -d \'=/ \'"))
113 >                except:
114 >                    msg = "Error. Problem with voms-proxy-info -identity command"
115 >                    raise CrabException(msg)
116 >        except KeyError: self.publish_data = 0
117  
118 <        if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
119 <           msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
118 >        if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
119 >           msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
120             msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
121             common.logger.message(msg)
122             raise CrabException(msg)
123 +        #################################################
124 +
125 +        #try:
126 +        #    self.lfc_host = cfg_params['EDG.lfc_host']
127 +        #except KeyError:
128 +        #    msg = "Error. The [EDG] section does not have 'lfc_host' value"
129 +        #    msg = msg + " it's necessary to know the LFC host name"
130 +        #    common.logger.message(msg)
131 +        #    raise CrabException(msg)
132 +        #try:
133 +        #    self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
134 +        #except KeyError:
135 +        #    msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
136 +        #    msg = msg + " it's necessary to know the catalog type"
137 +        #    common.logger.message(msg)
138 +        #    raise CrabException(msg)
139 +        #try:
140 +        #    self.lfc_home = cfg_params['EDG.lfc_home']
141 +        #except KeyError:
142 +        #    msg = "Error. The [EDG] section does not have 'lfc_home' value"
143 +        #    msg = msg + " it's necessary to know the home catalog dir"
144 +        #    common.logger.message(msg)
145 +        #    raise CrabException(msg)
146 +      
147 +        #try:
148 +        #    self.register_data = cfg_params["USER.register_data"]
149 +        #    if int(self.register_data) == 1:
150 +        #        try:
151 +        #            self.LFN = cfg_params['USER.lfn_dir']
152 +        #        except KeyError:
153 +        #            msg = "Error. The [USER] section does not have 'lfn_dir' value"
154 +        #            msg = msg + " it's necessary for LCF registration"
155 +        #            common.logger.message(msg)
156 +        #            raise CrabException(msg)
157 +        #except KeyError: self.register_data = 0
158 +
159 +        #if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
160 +        #   msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
161 +        #   msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
162 +        #   common.logger.message(msg)
163 +        #   raise CrabException(msg)
164  
165          try: self.EDG_requirements = cfg_params['EDG.requirements']
166          except KeyError: self.EDG_requirements = ''
# Line 155 | Line 193 | class SchedulerEdg(Scheduler):
193          libPath=os.path.join(path, "lib", "python")
194          sys.path.append(libPath)
195  
158 –        self.proxyValid=0
159 –
196          try:
197              self._taskId = cfg_params['taskId']
198          except:
# Line 168 | Line 204 | class SchedulerEdg(Scheduler):
204          try: self.schedulerName = cfg_params['CRAB.scheduler']
205          except KeyError: self.scheduler = ''
206  
171 –        try: self.dontCheckProxy=cfg_params["EDG.dont_check_proxy"]
172 –        except KeyError: self.dontCheckProxy = 0
173 –
207          return
208      
209  
# Line 184 | Line 217 | class SchedulerEdg(Scheduler):
217          self.edg_config_vo = edgConfig.configVO()
218  
219          if (self.edg_config and self.edg_config_vo != ''):
220 <            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";'
220 >            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";\n'
221              #print "rb_param_file = ", self.rb_param_file
222          return self.rb_param_file
223        
# Line 292 | Line 325 | class SchedulerEdg(Scheduler):
325          txt += 'echo "middleware discovery " \n'
326          txt += 'if [ $GRID3_APP_DIR ]; then\n'
327          txt += '    middleware=OSG \n'
328 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
328 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
329 >        txt += '        SyncCE=`echo "https://cmsgrid02.hep.wisc.edu:59387/11075/1191845762/" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
330 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
331 >        txt += '    else\n'
332 >        txt += '        echo "not reporting SyncCE";\n'
333 >        txt += '    fi\n';
334          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
335          txt += '    echo "middleware =$middleware" \n'
336          txt += 'elif [ $OSG_APP ]; then \n'
337          txt += '    middleware=OSG \n'
338 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
338 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
339 >        txt += '        SyncCE=`echo "https://cmsgrid02.hep.wisc.edu:59387/11075/1191845762/" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
340 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
341 >        txt += '    else\n'
342 >        txt += '        echo "not reporting SyncCE";\n'
343 >        txt += '    fi\n';
344          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
345          txt += '    echo "middleware =$middleware" \n'
346          txt += 'elif [ $VO_CMS_SW_DIR ]; then \n'
347          txt += '    middleware=LCG \n'
348 <        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
348 >   #     txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
349 >        txt += '    echo "SyncCE=`glite-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
350          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
351          txt += '    echo "middleware =$middleware" \n'
352          txt += 'else \n'
# Line 324 | Line 368 | class SchedulerEdg(Scheduler):
368          
369          txt += '\n\n'
370  
371 <        if int(self.copy_data) == 1:
372 <           if self.SE:
373 <              txt += 'export SE='+self.SE+'\n'
374 <              txt += 'echo "SE = $SE"\n'
375 <           if self.SE_PATH:
376 <              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
377 <              txt += 'export SE_PATH='+self.SE_PATH+'\n'
378 <              txt += 'echo "SE_PATH = $SE_PATH"\n'
371 > #        if int(self.copy_data) == 1:
372 > #           if self.SE:
373 > #              txt += 'export SE='+self.SE+'\n'
374 > #              txt += 'echo "SE = $SE"\n'
375 > #           if self.SE_PATH:
376 > #              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
377 > #              txt += 'export SE_PATH='+self.SE_PATH+'\n'
378 > #              txt += 'echo "SE_PATH = $SE_PATH"\n'
379  
380          txt += 'export VO='+self.VO+'\n'
381          ### add some line for LFC catalog setting
382 <        txt += 'if [ $middleware == LCG ]; then \n'
383 <        txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
384 <        txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
385 <        txt += '    fi\n'
386 <        txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
387 <        txt += '        export LFC_HOST='+self.lfc_host+'\n'
388 <        txt += '    fi\n'
389 <        txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
390 <        txt += '        export LFC_HOME='+self.lfc_home+'\n'
391 <        txt += '    fi\n'
392 <        txt += 'elif [ $middleware == OSG ]; then\n'
393 <        txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
394 <        txt += 'fi\n'
382 >        #txt += 'if [ $middleware == LCG ]; then \n'
383 >        #txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
384 >        #txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
385 >        #txt += '    fi\n'
386 >        #txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
387 >        #txt += '        export LFC_HOST='+self.lfc_host+'\n'
388 >        #txt += '    fi\n'
389 >        #txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
390 >        #txt += '        export LFC_HOME='+self.lfc_home+'\n'
391 >        #txt += '    fi\n'
392 >        #txt += 'elif [ $middleware == OSG ]; then\n'
393 >        #txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
394 >        #txt += 'fi\n'
395          #####
396 <        if int(self.register_data) == 1:
397 <           txt += 'if [ $middleware == LCG ]; then \n'
398 <           txt += '    export LFN='+self.LFN+'\n'
399 <           txt += '    lfc-ls $LFN\n'
400 <           txt += '    result=$?\n'
401 <           txt += '    echo $result\n'
402 <           ### creation of LFN dir in LFC catalog, under /grid/cms dir  
403 <           txt += '    if [ $result != 0 ]; then\n'
404 <           txt += '       lfc-mkdir $LFN\n'
405 <           txt += '       result=$?\n'
406 <           txt += '       echo $result\n'
407 <           txt += '    fi\n'
408 <           txt += 'elif [ $middleware == OSG ]; then\n'
409 <           txt += '    echo " Files registration to be implemented for OSG"\n'
410 <           txt += 'fi\n'
411 <           txt += '\n'
412 <
413 <           if self.VO:
414 <              txt += 'export VO='+self.VO+'\n'
415 <           if self.LFN:
416 <              txt += 'if [ $middleware == LCG ]; then \n'
417 <              txt += '    export LFN='+self.LFN+'\n'
418 <              txt += 'fi\n'
375 <              txt += '\n'
396 >        #if int(self.register_data) == 1:
397 >        #   txt += 'if [ $middleware == LCG ]; then \n'
398 >        #   txt += '    export LFN='+self.LFN+'\n'
399 >        #   txt += '    lfc-ls $LFN\n'
400 >        #   txt += '    result=$?\n'
401 >        #   txt += '    echo $result\n'
402 >        #   ### creation of LFN dir in LFC catalog, under /grid/cms dir  
403 >        #   txt += '    if [ $result != 0 ]; then\n'
404 >        #   txt += '       lfc-mkdir $LFN\n'
405 >        #   txt += '       result=$?\n'
406 >        #   txt += '       echo $result\n'
407 >        #   txt += '    fi\n'
408 >        #   txt += 'elif [ $middleware == OSG ]; then\n'
409 >        #   txt += '    echo " Files registration to be implemented for OSG"\n'
410 >        #   txt += 'fi\n'
411 >        #   txt += '\n'
412 >        #   if self.VO:
413 >        #      txt += 'export VO='+self.VO+'\n'
414 >        #   if self.LFN:
415 >        #      txt += 'if [ $middleware == LCG ]; then \n'
416 >        #      txt += '    export LFN='+self.LFN+'\n'
417 >        #      txt += 'fi\n'
418 >        #      txt += '\n'
419  
420          txt += 'if [ $middleware == LCG ]; then\n'
421 <        txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
421 >    #    txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
422 >        txt += '    CloseCEs=`glite-brokerinfo getCE`\n'
423          txt += '    echo "CloseCEs = $CloseCEs"\n'
424          txt += '    CE=`echo $CloseCEs | sed -e "s/:.*//"`\n'
425          txt += '    echo "CE = $CE"\n'
# Line 449 | Line 493 | class SchedulerEdg(Scheduler):
493          Write a CopyResults part of a job script, e.g.
494          to copy produced output into a storage element.
495          """
496 <        txt = ''
453 <        if int(self.copy_data) == 1:
454 <           txt += '#\n'
455 <           txt += '#   Copy output to SE = $SE\n'
456 <           txt += '#\n'
457 <           txt += '    if [ $middleware == OSG ]; then\n'
458 <           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
459 <           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
460 <           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
461 <           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
462 <           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
463 <           txt += '    fi \n'
464 <
465 <           txt += '    for out_file in $file_list ; do\n'
466 <           txt += '        echo "Trying to copy output file to $SE using srmcp"\n'
467 <           txt += '        echo "mkdir -p $HOME/.srmconfig"\n'
468 <           txt += '        mkdir -p $HOME/.srmconfig\n'
469 <           txt += '        if [ $middleware == LCG ]; then\n'
470 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 480000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
471 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 480000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
472 <           txt += '        elif [ $middleware == OSG ]; then\n'
473 <           txt += '           echo "srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
474 <           txt += '           exitstring=`srmcp -retry_num 3 -retry_timeout 240000 -x509_user_trusted_certificates $X509_CERT_DIR file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
475 <           txt += '        fi \n'
476 <           txt += '        copy_exit_status=$?\n'
477 <           txt += '        echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
478 <           txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
479 <
480 <           txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
481 <           txt += '            echo "Possible problem with SE = $SE"\n'
482 <           txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
483 <           txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
484 <           txt += '            echo "srmcp failed, attempting lcg-cp."\n'
485 <           if common.logger.debugLevel() >= 5:
486 <               txt += '            echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
487 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
488 <           else:              
489 <               txt += '            echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
490 <               txt += '            exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
491 <           txt += '            copy_exit_status=$?\n'
492 <           txt += '            echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
493 <           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
494 <
495 <           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
496 <           txt += '               echo "Problems with SE = $SE"\n'
497 <           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
498 <           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
499 <           txt += '               echo "srmcp and lcg-cp and failed!"\n'
500 <           txt += '            else\n'
501 <           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
502 <           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
503 <           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
504 <           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
505 <           txt += '               echo "lcg-cp succeeded"\n'
506 <           txt += '            fi\n'
507 <           txt += '        else\n'
508 <           txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
509 <           txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
510 <           txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
511 <           txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
512 <           txt += '            echo "srmcp succeeded"\n'
513 <           txt += '         fi\n'
514 <           txt += '     done\n'
515 <        return txt
496 >        txt = '\n'
497  
498 <    def wsRegisterOutput(self):
499 <        """
500 <        Returns part of a job script which does scheduler-specific work.
520 <        """
498 >        txt += '#\n'
499 >        txt += '# COPY OUTPUT FILE TO SE\n'
500 >        txt += '#\n\n'
501  
502 <        txt = ''
503 <        if int(self.register_data) == 1:
504 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
505 <           txt += 'if [ $middleware == OSG ]; then\n'
506 <           txt += '   #\n'
507 <           txt += '   #   Register output to LFC deactivated in OSG mode\n'
508 <           txt += '   #\n'
509 <           txt += '   echo "Register output to LFC deactivated in OSG mode"\n'
510 <           txt += 'elif [ $middleware == LCG ]; then \n'
511 <           txt += '#\n'
512 <           txt += '#  Register output to LFC\n'
513 <           txt += '#\n'
514 <           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
515 <           txt += '      for out_file in $file_list ; do\n'
516 <           txt += '         echo "Trying to register the output file into LFC"\n'
517 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
518 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
519 <           txt += '         register_exit_status=$?\n'
520 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
521 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
522 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
523 <           txt += '            echo "Problems with the registration to LFC" \n'
524 <           txt += '            echo "Try with srm protocol" \n'
525 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
526 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
527 <           txt += '            register_exit_status=$?\n'
528 <           txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
529 <           txt += '            echo "STAGE_OUT = $register_exit_status"\n'
530 <           txt += '            if [ $register_exit_status -ne 0 ]; then \n'
531 <           txt += '               echo "Problems with the registration into LFC" \n'
532 <           txt += '            fi \n'
533 <           txt += '         else \n'
534 <           txt += '            echo "output registered to LFC"\n'
535 <           txt += '         fi \n'
536 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
537 <           txt += '      done\n'
538 <           txt += '   else \n'
539 <           txt += '      echo "Trying to copy output file to CloseSE"\n'
540 <           txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
541 <           txt += '      for out_file in $file_list ; do\n'
542 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
543 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
544 <           txt += '         register_exit_status=$?\n'
545 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
546 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
547 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
568 <           txt += '            echo "Problems with CloseSE or Catalog" \n'
569 <           txt += '         else \n'
570 <           txt += '            echo "The program was successfully executed"\n'
571 <           txt += '            echo "SE = $CLOSE_SE"\n'
572 <           txt += '            echo "LFN for the file is LFN=${LFN}/$out_file"\n'
573 <           txt += '         fi \n'
574 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
575 <           txt += '      done\n'
576 <           txt += '   fi \n'
577 <           txt += '   exit_status=$register_exit_status\n'
578 <           txt += 'fi \n'
502 >        SE_PATH=''
503 >        if int(self.copy_data) == 1:
504 >            if self.SE:
505 >                txt += 'export SE='+self.SE+'\n'
506 >                txt += 'echo "SE = $SE"\n'
507 >            if self.SE_PATH:
508 >                if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
509 >                SE_PATH=self.SE_PATH
510 >            if int(self.publish_data) == 1:
511 >                txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
512 >                path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
513 >                SE_PATH = SE_PATH + path_add
514 >            txt += 'export SE_PATH='+SE_PATH+'\n'
515 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
516 >            
517 >            txt += 'echo "####################################################"\n'
518 >            txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
519 >            txt += 'echo "####################################################"\n'
520 >            
521 >            txt += 'for out_file in $file_list ; do\n'
522 >            txt += '   echo "Trying to copy output file to $SE"\n'
523 >            txt += '   cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
524 >            txt += '   copy_exit_status=$?\n'
525 >            txt += '   echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
526 >            txt += '   echo "STAGE_OUT = $copy_exit_status"\n'
527 >            
528 >            txt += '   if [ $copy_exit_status -ne 0 ]; then\n'
529 >            txt += '       echo "Problem copying $out_file to $SE $SE_PATH"\n'
530 >            txt += '       echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
531 >            txt += '       copy_exit_status=60307\n'
532 >            
533 >            txt += '   else\n'
534 >            txt += '       echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
535 >            txt += '       echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
536 >            txt += '       echo "output copied into $SE/$SE_PATH directory"\n'
537 >            txt += '       echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
538 >            txt += '    fi\n'
539 >            txt += 'done\n'
540 >            txt += 'if [ $copy_exit_status -ne 0 ]; then\n'
541 >            txt += '      SE=""\n'
542 >            txt += '      echo "SE = $SE"\n'
543 >            txt += '      SE_PATH=""\n'
544 >            txt += '      echo "SE_PATH = $SE_PATH"\n'
545 >            txt += 'fi\n'
546 >            txt += 'exit_status=$copy_exit_status\n'
547 >            pass
548          return txt
549  
550      def loggingInfo(self, id):
# Line 593 | Line 562 | class SchedulerEdg(Scheduler):
562          cmd_out = runCommand(cmd)
563          return cmd_out
564  
596 –    ##### FEDE ######        
565      def findSites_(self, n):
566          itr4 =[]
567 +
568          sites = common.jobDB.destination(n)
569 <        if len(sites)>0 and sites[0]=="Any":
569 >
570 >        if len(sites)>0 and sites[0]=="":
571              return itr4
572 +
573          itr = ''
574          if sites != [""]:#CarlosDaniele
575 <            for site in sites:
575 >            ##Addedd Daniele
576 >            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
577 >            if len(replicas)!=0:
578 >                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
579 >              
580 >            if len(replicas)==0:
581 >                msg = 'No sites remaining that host any part of the requested data! Exiting... '
582 >                raise CrabException(msg)
583 >            #####        
584 >           # for site in sites:
585 >            for site in replicas:
586                  #itr = itr + 'target.GlueSEUniqueID==&quot;'+site+'&quot; || '
587                  itr = itr + 'target.GlueSEUniqueID=="'+site+'" || '
588              itr = itr[0:-4]
# Line 624 | Line 605 | class SchedulerEdg(Scheduler):
605          jbt = job.type()
606          
607          inp_sandbox = jbt.inputSandbox(index)
608 <        out_sandbox = jbt.outputSandbox(index)
608 >        #out_sandbox = jbt.outputSandbox(index)
609          """
610          [end] FIX-ME
611          """
# Line 640 | Line 621 | class SchedulerEdg(Scheduler):
621          dir = string.split(common.work_space.topDir(), '/')
622          taskName = dir[len(dir)-2]
623    
643 –        to_writeReq = ''
624          to_write = ''
625  
626          req=' '
# Line 700 | Line 680 | class SchedulerEdg(Scheduler):
680          taskName = dir[len(dir)-2]
681  
682          xml.write(str(title))
683 <        xml.write('<task name="' +str(taskName)+'" sub_path="' + common.work_space.bossCache() + '">\n')
683 >        #xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
684 >
685 >        #xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n')
686 >        x509_cmd = 'ls /tmp/x509up_u`id -u`'
687 >        x509=runCommand(x509_cmd).strip()
688 >        xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + str(x509) + '">\n')
689          xml.write(jt_string)
690          
691          if (to_write != ''):
# Line 727 | Line 712 | class SchedulerEdg(Scheduler):
712          indy: here itr4
713          '''
714          
715 <
716 <        xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
715 >        xml.write('<chain name="' +str(taskName)+'__ITR1_" scheduler="'+str(self.schedulerName)+'">\n')
716 >       # xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
717          xml.write(jt_string)
718  
719          #executable
# Line 752 | Line 737 | class SchedulerEdg(Scheduler):
737                  pass
738              pass
739  
740 <        if (not jbt.additional_inbox_files == []):
741 <            inp_box = inp_box + ','
742 <            for addFile in jbt.additional_inbox_files:
743 <                addFile = os.path.abspath(addFile)
744 <                inp_box = inp_box+''+addFile+','
745 <                pass
740 > #        if (not jbt.additional_inbox_files == []):
741 > #            inp_box = inp_box + ','
742 > #            for addFile in jbt.additional_inbox_files:
743 > #                #addFile = os.path.abspath(addFile)
744 > #                inp_box = inp_box+''+addFile+','
745 > #                pass
746  
747          if inp_box[-1] == ',' : inp_box = inp_box[:-1]
748          inp_box = '<infiles> <![CDATA[\n' + inp_box + '\n]]> </infiles>\n'
# Line 825 | Line 810 | class SchedulerEdg(Scheduler):
810          if (self.proxyValid): return
811  
812          ### Just return if asked to do so
813 <        if (self.dontCheckProxy):
813 >        if (self.dontCheckProxy==1):
814              self.proxyValid=1
815              return
816  
832 –        timeleft = -999
817          minTimeLeft=10*3600 # in seconds
818  
819          minTimeLeftServer = 100 # in hours
# Line 878 | Line 862 | class SchedulerEdg(Scheduler):
862              reTime = re.compile( r'timeleft: (\d+)' )
863              #print "<"+str(reTime.search( cmd_out ).group(1))+">"
864              if reTime.match( cmd_out ):
865 <                time = reTime.search( line ).group(1)
865 >                time = reTime.search( cmd_out ).group(1)
866                  if time < minTimeLeftServer:
867                      renewProxy = 1
868                      common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
# Line 904 | Line 888 | class SchedulerEdg(Scheduler):
888          if self.edg_config_vo:
889              edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
890          return edg_ui_cfg_opt
891 +
892 +    def submitTout(self, list):
893 +        return 120
894 +
895 +

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines