ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.90 by slacapra, Fri Oct 6 15:50:17 2006 UTC vs.
Revision 1.144 by mcinquil, Mon Oct 8 20:57:13 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   from EdgConfig import *
6 + from BlackWhiteListParser import BlackWhiteListParser
7   import common
8  
9   import os, sys, time
# Line 22 | Line 23 | class SchedulerEdg(Scheduler):
23  
24      def configure(self, cfg_params):
25  
26 +        # init BlackWhiteListParser
27 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
28 +
29 +        self.proxyValid=0
30 +        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
31 +        except KeyError: self.dontCheckProxy = 0
32 +
33          try:
34 <            RB = cfg_params["EDG.rb"]
35 <            edgConfig = EdgConfig(RB)
28 <            self.edg_config = edgConfig.config()
29 <            self.edg_config_vo = edgConfig.configVO()
34 >            RB=cfg_params["EDG.rb"]
35 >            self.rb_param_file=self.rb_configure(RB)
36          except KeyError:
37 <            self.edg_config = ''
38 <            self.edg_config_vo = ''
33 <
37 >            self.rb_param_file=''
38 >            pass
39          try:
40              self.proxyServer = cfg_params["EDG.proxy_server"]
41          except KeyError:
# Line 38 | Line 43 | class SchedulerEdg(Scheduler):
43          common.logger.debug(5,'Setting myproxy server to '+self.proxyServer)
44  
45          try:
46 +            self.group = cfg_params["EDG.group"]
47 +        except KeyError:
48 +            self.group = None
49 +            
50 +        try:
51              self.role = cfg_params["EDG.role"]
52          except KeyError:
53              self.role = None
54              
55 <        try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 <        except KeyError: self.LCG_version = '2'
47 <
48 <        try: self.EDG_requirements = cfg_params['EDG.requirements']
49 <        except KeyError: self.EDG_requirements = ''
50 <
51 <        try: self.EDG_retry_count = cfg_params['EDG.retry_count']
52 <        except KeyError: self.EDG_retry_count = ''
55 >        #try: self.LCG_version = cfg_params["EDG.lcg_version"]
56 >        #except KeyError: self.LCG_version = '2'
57  
58          try:
59              self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
56            #print "self.EDG_ce_black_list = ", self.EDG_ce_black_list
60          except KeyError:
61              self.EDG_ce_black_list  = ''
62  
63          try:
64              self.EDG_ce_white_list = cfg_params['EDG.ce_white_list']
62            #print "self.EDG_ce_white_list = ", self.EDG_ce_white_list
65          except KeyError: self.EDG_ce_white_list = ''
66  
67          try: self.VO = cfg_params['EDG.virtual_organization']
68          except KeyError: self.VO = 'cms'
69  
70 +        try: self.copy_input_data = cfg_params["USER.copy_input_data"]
71 +        except KeyError: self.copy_input_data = 0
72 +
73          try: self.return_data = cfg_params['USER.return_data']
74 <        except KeyError: self.return_data = 1
74 >        except KeyError: self.return_data = 0
75  
76          try:
77              self.copy_data = cfg_params["USER.copy_data"]
# Line 86 | Line 91 | class SchedulerEdg(Scheduler):
91             msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
92             raise CrabException(msg)
93  
94 <        try:
90 <            self.lfc_host = cfg_params['EDG.lfc_host']
91 <        except KeyError:
92 <            msg = "Error. The [EDG] section does not have 'lfc_host' value"
93 <            msg = msg + " it's necessary to know the LFC host name"
94 <            common.logger.message(msg)
95 <            raise CrabException(msg)
96 <        try:
97 <            self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
98 <        except KeyError:
99 <            msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
100 <            msg = msg + " it's necessary to know the catalog type"
101 <            common.logger.message(msg)
102 <            raise CrabException(msg)
103 <        try:
104 <            self.lfc_home = cfg_params['EDG.lfc_home']
105 <        except KeyError:
106 <            msg = "Error. The [EDG] section does not have 'lfc_home' value"
107 <            msg = msg + " it's necessary to know the home catalog dir"
108 <            common.logger.message(msg)
109 <            raise CrabException(msg)
110 <      
94 >        ########### FEDE FOR DBS2 ##############################
95          try:
96 <            self.register_data = cfg_params["USER.register_data"]
97 <            if int(self.register_data) == 1:
96 >            self.publish_data = cfg_params["USER.publish_data"]
97 >            self.checkProxy()
98 >            if int(self.publish_data) == 1:
99                  try:
100 <                    self.LFN = cfg_params['USER.lfn_dir']
100 >                    self.publish_data_name = cfg_params['USER.publish_data_name']
101                  except KeyError:
102 <                    msg = "Error. The [USER] section does not have 'lfn_dir' value"
118 <                    msg = msg + " it's necessary for LCF registration"
119 <                    common.logger.message(msg)
102 >                    msg = "Error. The [USER] section does not have 'publish_data_name'"
103                      raise CrabException(msg)
104 <        except KeyError: self.register_data = 0
104 >                try:
105 >                    tmp = runCommand("voms-proxy-info -identity")
106 >                    tmp = string.split(tmp,'/')
107 >                    reCN=re.compile(r'CN=')
108 >                    for t in tmp:
109 >                        if reCN.match(t):
110 >                            self.UserGridName=string.strip((t.replace('CN=','')).replace(' ',''))
111 >                        
112 >                    #self.UserGridName = string.strip(runCommand("voms-proxy-info -identity | awk -F\'CN\' \'{print $2$3$4}\' | tr -d \'=/ \'"))
113 >                except:
114 >                    msg = "Error. Problem with voms-proxy-info -identity command"
115 >                    raise CrabException(msg)
116 >        except KeyError: self.publish_data = 0
117  
118 <        if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
119 <           msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
118 >        if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
119 >           msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
120             msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
121             common.logger.message(msg)
122             raise CrabException(msg)
123 +        #################################################
124 +
125 +        #try:
126 +        #    self.lfc_host = cfg_params['EDG.lfc_host']
127 +        #except KeyError:
128 +        #    msg = "Error. The [EDG] section does not have 'lfc_host' value"
129 +        #    msg = msg + " it's necessary to know the LFC host name"
130 +        #    common.logger.message(msg)
131 +        #    raise CrabException(msg)
132 +        #try:
133 +        #    self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
134 +        #except KeyError:
135 +        #    msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
136 +        #    msg = msg + " it's necessary to know the catalog type"
137 +        #    common.logger.message(msg)
138 +        #    raise CrabException(msg)
139 +        #try:
140 +        #    self.lfc_home = cfg_params['EDG.lfc_home']
141 +        #except KeyError:
142 +        #    msg = "Error. The [EDG] section does not have 'lfc_home' value"
143 +        #    msg = msg + " it's necessary to know the home catalog dir"
144 +        #    common.logger.message(msg)
145 +        #    raise CrabException(msg)
146 +      
147 +        #try:
148 +        #    self.register_data = cfg_params["USER.register_data"]
149 +        #    if int(self.register_data) == 1:
150 +        #        try:
151 +        #            self.LFN = cfg_params['USER.lfn_dir']
152 +        #        except KeyError:
153 +        #            msg = "Error. The [USER] section does not have 'lfn_dir' value"
154 +        #            msg = msg + " it's necessary for LCF registration"
155 +        #            common.logger.message(msg)
156 +        #            raise CrabException(msg)
157 +        #except KeyError: self.register_data = 0
158 +
159 +        #if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
160 +        #   msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
161 +        #   msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
162 +        #   common.logger.message(msg)
163 +        #   raise CrabException(msg)
164  
165          try: self.EDG_requirements = cfg_params['EDG.requirements']
166          except KeyError: self.EDG_requirements = ''
167 <                                                                                                                                                            
167 >
168 >        try: self.EDG_addJdlParam = string.split(cfg_params['EDG.additional_jdl_parameters'],',')
169 >        except KeyError: self.EDG_addJdlParam = []
170 >
171          try: self.EDG_retry_count = cfg_params['EDG.retry_count']
172          except KeyError: self.EDG_retry_count = ''
173 <                                                                                                                                                            
173 >
174 >        try: self.EDG_shallow_retry_count= cfg_params['EDG.shallow_retry_count']
175 >        except KeyError: self.EDG_shallow_retry_count = ''
176 >
177          try: self.EDG_clock_time = cfg_params['EDG.max_wall_clock_time']
178          except KeyError: self.EDG_clock_time= ''
179 <                                                                                                                                                            
179 >
180          try: self.EDG_cpu_time = cfg_params['EDG.max_cpu_time']
181          except KeyError: self.EDG_cpu_time = ''
182  
# Line 151 | Line 193 | class SchedulerEdg(Scheduler):
193          libPath=os.path.join(path, "lib", "python")
194          sys.path.append(libPath)
195  
154        self.proxyValid=0
155
196          try:
197              self._taskId = cfg_params['taskId']
198          except:
# Line 167 | Line 207 | class SchedulerEdg(Scheduler):
207          return
208      
209  
210 +    def rb_configure(self, RB):
211 +        self.edg_config = ''
212 +        self.edg_config_vo = ''
213 +        self.rb_param_file = ''
214 +
215 +        edgConfig = EdgConfig(RB)
216 +        self.edg_config = edgConfig.config()
217 +        self.edg_config_vo = edgConfig.configVO()
218 +
219 +        if (self.edg_config and self.edg_config_vo != ''):
220 +            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";\n'
221 +            #print "rb_param_file = ", self.rb_param_file
222 +        return self.rb_param_file
223 +      
224 +
225      def sched_parameter(self):
226          """
227          Returns file with requirements and scheduler-specific parameters
# Line 175 | Line 230 | class SchedulerEdg(Scheduler):
230          job = common.job_list[index]
231          jbt = job.type()
232          
233 <        lastDest=''
233 >        lastBlock=-1
234          first = []
180        last  = []
235          for n in range(common.jobDB.nJobs()):
236 <            currDest=common.jobDB.destination(n)
237 <            if (currDest!=lastDest):
238 <                lastDest = currDest
236 >            currBlock=common.jobDB.block(n)
237 >            if (currBlock!=lastBlock):
238 >                lastBlock = currBlock
239                  first.append(n)
186                if n != 0:last.append(n-1)
187        if len(first)>len(last) :last.append(common.jobDB.nJobs())
240    
241          req = ''
242          req = req + jbt.getRequirements()
# Line 194 | Line 246 | class SchedulerEdg(Scheduler):
246                  req = req + self.EDG_requirements
247              else:
248                  req = req +  ' && ' + self.EDG_requirements
249 +
250          if self.EDG_ce_white_list:
251              ce_white_list = string.split(self.EDG_ce_white_list,',')
252              for i in range(len(ce_white_list)):
253                  if i == 0:
254                      if (req == ' '):
255 <                        req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
255 >                        req = req + '((RegExp("' + string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
256                      else:
257 <                        req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
257 >                        req = req +  ' && ((RegExp("' +  string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
258                      pass
259                  else:
260 <                    req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
260 >                    req = req +  ' || (RegExp("' +  string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
261              req = req + ')'
262          
263          if self.EDG_ce_black_list:
264              ce_black_list = string.split(self.EDG_ce_black_list,',')
265              for ce in ce_black_list:
266                  if (req == ' '):
267 <                    req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
267 >                    req = req + '(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))'
268                  else:
269 <                    req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
269 >                    req = req +  ' && (!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))'
270                  pass
271          if self.EDG_clock_time:
272              if (req == ' '):
# Line 228 | Line 281 | class SchedulerEdg(Scheduler):
281                  req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
282                  
283          for i in range(len(first)): # Add loop DS
284 +            groupReq = req
285              self.param='sched_param_'+str(i)+'.clad'
286              param_file = open(common.work_space.shareDir()+'/'+self.param, 'w')
287  
288              itr4=self.findSites_(first[i])
289 <            if (itr4 != []):
290 <                req1=''  
291 <                for arg in itr4:
238 <                    req1 = req + ' && anyMatch(other.storage.CloseSEs, ('+str(arg)+'))'
239 <            param_file.write('Requirements = '+req1 +';\n')  
289 >            for arg in itr4:
290 >                groupReq = groupReq + ' && anyMatch(other.storage.CloseSEs, ('+str(arg)+'))'
291 >            param_file.write('Requirements = '+groupReq +';\n')  
292    
293 <            if (self.edg_config and self.edg_config_vo != ''):
294 <                param_file.write('RBconfig = "'+self.edg_config+'";\n')  
295 <                param_file.write('RBconfigVO = "'+self.edg_config_vo+'";')
293 >            if (self.rb_param_file != ''):
294 >                param_file.write(self.rb_param_file)  
295 >
296 >            if len(self.EDG_addJdlParam):
297 >                for p in self.EDG_addJdlParam:
298 >                    param_file.write(p)
299  
300              param_file.close()  
301  
# Line 270 | Line 325 | class SchedulerEdg(Scheduler):
325          txt += 'echo "middleware discovery " \n'
326          txt += 'if [ $GRID3_APP_DIR ]; then\n'
327          txt += '    middleware=OSG \n'
328 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
328 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
329 >        txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
330 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
331 >        txt += '    else\n'
332 >        txt += '        echo "not reporting SyncCE";\n'
333 >        txt += '    fi\n';
334          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
335          txt += '    echo "middleware =$middleware" \n'
336          txt += 'elif [ $OSG_APP ]; then \n'
337          txt += '    middleware=OSG \n'
338 <        txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
338 >        txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
339 >        txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
340 >        txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
341 >        txt += '    else\n'
342 >        txt += '        echo "not reporting SyncCE";\n'
343 >        txt += '    fi\n';
344          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
345          txt += '    echo "middleware =$middleware" \n'
346          txt += 'elif [ $VO_CMS_SW_DIR ]; then \n'
347          txt += '    middleware=LCG \n'
348 <        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
348 >   #     txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
349 >        txt += '    echo "SyncCE=`glite-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
350          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
351          txt += '    echo "middleware =$middleware" \n'
352          txt += 'else \n'
# Line 302 | Line 368 | class SchedulerEdg(Scheduler):
368          
369          txt += '\n\n'
370  
371 <        if int(self.copy_data) == 1:
372 <           if self.SE:
373 <              txt += 'export SE='+self.SE+'\n'
374 <              txt += 'echo "SE = $SE"\n'
375 <           if self.SE_PATH:
376 <              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
377 <              txt += 'export SE_PATH='+self.SE_PATH+'\n'
378 <              txt += 'echo "SE_PATH = $SE_PATH"\n'
371 > #        if int(self.copy_data) == 1:
372 > #           if self.SE:
373 > #              txt += 'export SE='+self.SE+'\n'
374 > #              txt += 'echo "SE = $SE"\n'
375 > #           if self.SE_PATH:
376 > #              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
377 > #              txt += 'export SE_PATH='+self.SE_PATH+'\n'
378 > #              txt += 'echo "SE_PATH = $SE_PATH"\n'
379  
380          txt += 'export VO='+self.VO+'\n'
381          ### add some line for LFC catalog setting
382 <        txt += 'if [ $middleware == LCG ]; then \n'
383 <        txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
384 <        txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
385 <        txt += '    fi\n'
386 <        txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
387 <        txt += '        export LFC_HOST='+self.lfc_host+'\n'
388 <        txt += '    fi\n'
389 <        txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
390 <        txt += '        export LFC_HOME='+self.lfc_home+'\n'
391 <        txt += '    fi\n'
392 <        txt += 'elif [ $middleware == OSG ]; then\n'
393 <        txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
394 <        txt += 'fi\n'
382 >        #txt += 'if [ $middleware == LCG ]; then \n'
383 >        #txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
384 >        #txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
385 >        #txt += '    fi\n'
386 >        #txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
387 >        #txt += '        export LFC_HOST='+self.lfc_host+'\n'
388 >        #txt += '    fi\n'
389 >        #txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
390 >        #txt += '        export LFC_HOME='+self.lfc_home+'\n'
391 >        #txt += '    fi\n'
392 >        #txt += 'elif [ $middleware == OSG ]; then\n'
393 >        #txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
394 >        #txt += 'fi\n'
395          #####
396 <        if int(self.register_data) == 1:
397 <           txt += 'if [ $middleware == LCG ]; then \n'
398 <           txt += '    export LFN='+self.LFN+'\n'
399 <           txt += '    lfc-ls $LFN\n'
400 <           txt += '    result=$?\n'
401 <           txt += '    echo $result\n'
402 <           ### creation of LFN dir in LFC catalog, under /grid/cms dir  
403 <           txt += '    if [ $result != 0 ]; then\n'
404 <           txt += '       lfc-mkdir $LFN\n'
405 <           txt += '       result=$?\n'
406 <           txt += '       echo $result\n'
407 <           txt += '    fi\n'
408 <           txt += 'elif [ $middleware == OSG ]; then\n'
409 <           txt += '    echo " Files registration to be implemented for OSG"\n'
410 <           txt += 'fi\n'
411 <           txt += '\n'
412 <
413 <           if self.VO:
414 <              txt += 'export VO='+self.VO+'\n'
415 <           if self.LFN:
416 <              txt += 'if [ $middleware == LCG ]; then \n'
417 <              txt += '    export LFN='+self.LFN+'\n'
418 <              txt += 'fi\n'
353 <              txt += '\n'
396 >        #if int(self.register_data) == 1:
397 >        #   txt += 'if [ $middleware == LCG ]; then \n'
398 >        #   txt += '    export LFN='+self.LFN+'\n'
399 >        #   txt += '    lfc-ls $LFN\n'
400 >        #   txt += '    result=$?\n'
401 >        #   txt += '    echo $result\n'
402 >        #   ### creation of LFN dir in LFC catalog, under /grid/cms dir  
403 >        #   txt += '    if [ $result != 0 ]; then\n'
404 >        #   txt += '       lfc-mkdir $LFN\n'
405 >        #   txt += '       result=$?\n'
406 >        #   txt += '       echo $result\n'
407 >        #   txt += '    fi\n'
408 >        #   txt += 'elif [ $middleware == OSG ]; then\n'
409 >        #   txt += '    echo " Files registration to be implemented for OSG"\n'
410 >        #   txt += 'fi\n'
411 >        #   txt += '\n'
412 >        #   if self.VO:
413 >        #      txt += 'export VO='+self.VO+'\n'
414 >        #   if self.LFN:
415 >        #      txt += 'if [ $middleware == LCG ]; then \n'
416 >        #      txt += '    export LFN='+self.LFN+'\n'
417 >        #      txt += 'fi\n'
418 >        #      txt += '\n'
419  
420          txt += 'if [ $middleware == LCG ]; then\n'
421 <        txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
421 >    #    txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
422 >        txt += '    CloseCEs=`glite-brokerinfo getCE`\n'
423          txt += '    echo "CloseCEs = $CloseCEs"\n'
424          txt += '    CE=`echo $CloseCEs | sed -e "s/:.*//"`\n'
425          txt += '    echo "CE = $CE"\n'
# Line 379 | Line 445 | class SchedulerEdg(Scheduler):
445          Copy input data from SE to WN    
446          """
447          txt = ''
448 +        if not self.copy_input_data: return txt
449  
450          ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
451          txt += 'if [ $middleware == OSG ]; then\n'
# Line 426 | Line 493 | class SchedulerEdg(Scheduler):
493          Write a CopyResults part of a job script, e.g.
494          to copy produced output into a storage element.
495          """
496 <        txt = ''
430 <        if int(self.copy_data) == 1:
431 <           txt += '#\n'
432 <           txt += '#   Copy output to SE = $SE\n'
433 <           txt += '#\n'
434 <           txt += '    if [ $middleware == OSG ]; then\n'
435 <           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
436 <           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
437 <           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
438 <           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
439 <           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
440 <           txt += '    fi \n'
441 <           txt += '    for out_file in $file_list ; do\n'
442 <           txt += '        echo "Trying to copy output file to $SE using lcg-cp"\n'
443 <           if common.logger.debugLevel() >= 5:
444 <               txt += '        echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
445 <               txt += '        exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
446 <           else:
447 <               txt += '        echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
448 <               txt += '        exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
449 <           txt += '        copy_exit_status=$?\n'
450 <           txt += '        echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
451 <           txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
452 <           txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
453 <           txt += '            echo "Possible problem with SE = $SE"\n'
454 <           txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
455 <           txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
456 <           txt += '            echo "lcg-cp failed.  For verbose lcg-cp output, use command line option -debug 5."\n'
457 <           txt += '            echo "lcg-cp failed, attempting srmcp"\n'
458 <           txt += '            echo "mkdir -p $HOME/.srmconfig"\n'
459 <           txt += '            mkdir -p $HOME/.srmconfig\n'
460 <           txt += '            if [ $middleware == LCG ]; then\n'
461 <           txt += '               echo "srmcp -retry_num 5 -retry_timeout 480000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
462 <           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 480000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
463 <           txt += '            elif [ $middleware == OSG ]; then\n'
464 <           txt += '               echo "srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
465 <           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
466 <           txt += '            fi \n'
467 <           txt += '            copy_exit_status=$?\n'
468 <           txt += '            echo "COPY_EXIT_STATUS for srm = $copy_exit_status"\n'
469 <           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
470 <           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
471 <           txt += '               echo "Problems with SE = $SE"\n'
472 <           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
473 <           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
474 <           txt += '               echo "lcg-cp and srm failed"\n'
475 <           txt += '               echo "If storage_path in your config file contains a ? you may need a \? instead."\n'
476 <           txt += '            else\n'
477 <           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
478 <           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
479 <           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
480 <           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
481 <           txt += '               echo "srmcp succeeded"\n'
482 <           txt += '            fi\n'
483 <           txt += '        else\n'
484 <           txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
485 <           txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
486 <           txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
487 <           txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
488 <           txt += '            echo "lcg-cp succeeded"\n'
489 <           txt += '         fi\n'
490 <           txt += '     done\n'
491 <        return txt
496 >        txt = '\n'
497  
498 <    def wsRegisterOutput(self):
499 <        """
500 <        Returns part of a job script which does scheduler-specific work.
496 <        """
498 >        txt += '#\n'
499 >        txt += '# COPY OUTPUT FILE TO SE\n'
500 >        txt += '#\n\n'
501  
502 <        txt = ''
503 <        if int(self.register_data) == 1:
504 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
505 <           txt += 'if [ $middleware == OSG ]; then\n'
506 <           txt += '   #\n'
507 <           txt += '   #   Register output to LFC deactivated in OSG mode\n'
508 <           txt += '   #\n'
509 <           txt += '   echo "Register output to LFC deactivated in OSG mode"\n'
510 <           txt += 'elif [ $middleware == LCG ]; then \n'
511 <           txt += '#\n'
512 <           txt += '#  Register output to LFC\n'
513 <           txt += '#\n'
514 <           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
515 <           txt += '      for out_file in $file_list ; do\n'
516 <           txt += '         echo "Trying to register the output file into LFC"\n'
517 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
518 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
519 <           txt += '         register_exit_status=$?\n'
520 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
521 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
522 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
523 <           txt += '            echo "Problems with the registration to LFC" \n'
524 <           txt += '            echo "Try with srm protocol" \n'
525 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
526 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
527 <           txt += '            register_exit_status=$?\n'
528 <           txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
529 <           txt += '            echo "STAGE_OUT = $register_exit_status"\n'
530 <           txt += '            if [ $register_exit_status -ne 0 ]; then \n'
531 <           txt += '               echo "Problems with the registration into LFC" \n'
532 <           txt += '            fi \n'
533 <           txt += '         else \n'
534 <           txt += '            echo "output registered to LFC"\n'
535 <           txt += '         fi \n'
536 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
537 <           txt += '      done\n'
538 <           txt += '   else \n'
539 <           txt += '      echo "Trying to copy output file to CloseSE"\n'
540 <           txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
541 <           txt += '      for out_file in $file_list ; do\n'
542 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
543 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
544 <           txt += '         register_exit_status=$?\n'
545 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
546 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
547 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
544 <           txt += '            echo "Problems with CloseSE or Catalog" \n'
545 <           txt += '         else \n'
546 <           txt += '            echo "The program was successfully executed"\n'
547 <           txt += '            echo "SE = $CLOSE_SE"\n'
548 <           txt += '            echo "LFN for the file is LFN=${LFN}/$out_file"\n'
549 <           txt += '         fi \n'
550 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
551 <           txt += '      done\n'
552 <           txt += '   fi \n'
553 <           txt += '   exit_status=$register_exit_status\n'
554 <           txt += 'fi \n'
502 >        SE_PATH=''
503 >        if int(self.copy_data) == 1:
504 >            if self.SE:
505 >                txt += 'export SE='+self.SE+'\n'
506 >                txt += 'echo "SE = $SE"\n'
507 >            if self.SE_PATH:
508 >                if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
509 >                SE_PATH=self.SE_PATH
510 >            if int(self.publish_data) == 1:
511 >                txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
512 >                path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
513 >                SE_PATH = SE_PATH + path_add
514 >            txt += 'export SE_PATH='+SE_PATH+'\n'
515 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
516 >            
517 >            txt += 'echo "####################################################"\n'
518 >            txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
519 >            txt += 'echo "####################################################"\n'
520 >            
521 >            txt += 'for out_file in $file_list ; do\n'
522 >            txt += '   echo "Trying to copy output file to $SE"\n'
523 >            txt += '   cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
524 >            txt += '   copy_exit_status=$?\n'
525 >            txt += '   echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
526 >            txt += '   echo "STAGE_OUT = $copy_exit_status"\n'
527 >            
528 >            txt += '   if [ $copy_exit_status -ne 0 ]; then\n'
529 >            txt += '       echo "Problem copying $out_file to $SE $SE_PATH"\n'
530 >            txt += '       echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
531 >            txt += '       copy_exit_status=60307\n'
532 >            
533 >            txt += '   else\n'
534 >            txt += '       echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
535 >            txt += '       echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
536 >            txt += '       echo "output copied into $SE/$SE_PATH directory"\n'
537 >            txt += '       echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
538 >            txt += '    fi\n'
539 >            txt += 'done\n'
540 >            txt += 'if [ $copy_exit_status -ne 0 ]; then\n'
541 >            txt += '      SE=""\n'
542 >            txt += '      echo "SE = $SE"\n'
543 >            txt += '      SE_PATH=""\n'
544 >            txt += '      echo "SE_PATH = $SE_PATH"\n'
545 >            txt += 'fi\n'
546 >            txt += 'exit_status=$copy_exit_status\n'
547 >            pass
548          return txt
549  
550      def loggingInfo(self, id):
# Line 560 | Line 553 | class SchedulerEdg(Scheduler):
553          """
554          self.checkProxy()
555          cmd = 'edg-job-get-logging-info -v 2 ' + id
563        #cmd_out = os.popen(cmd)
556          cmd_out = runCommand(cmd)
557          return cmd_out
558  
567    def getExitStatus(self, id):
568        return self.getStatusAttribute_(id, 'exit_code')
569
570    def queryStatus(self, id):
571        return self.getStatusAttribute_(id, 'status')
572
573    def queryDest(self, id):  
574        return self.getStatusAttribute_(id, 'destination')
575
576
577    def getStatusAttribute_(self, id, attr):
578        """ Query a status of the job with id """
579
580        self.checkProxy()
581        hstates = {}
582        Status = importName('edg_wl_userinterface_common_LbWrapper', 'Status')
583        # Bypass edg-job-status interfacing directly to C++ API
584        # Job attribute vector to retrieve status without edg-job-status
585        level = 0
586        # Instance of the Status class provided by LB API
587        jobStat = Status()
588        st = 0
589        jobStat.getStatus(id, level)
590        err, apiMsg = jobStat.get_error()
591        if err:
592            common.logger.debug(5,'Error caught' + apiMsg)
593            return None
594        else:
595            for i in range(len(self.states)):
596                # Fill an hash table with all information retrieved from LB API
597                hstates[ self.states[i] ] = jobStat.loadStatus(st)[i]
598            result = jobStat.loadStatus(st)[self.states.index(attr)]
599            return result
600
559      def queryDetailedStatus(self, id):
560          """ Query a detailed status of the job with id """
561          cmd = 'edg-job-status '+id
562          cmd_out = runCommand(cmd)
563          return cmd_out
564  
607    ##### FEDE ######        
565      def findSites_(self, n):
566          itr4 =[]
567 +
568          sites = common.jobDB.destination(n)
569 <        if len(sites)>0 and sites[0]=="Any":
569 >
570 >        if len(sites)>0 and sites[0]=="":
571              return itr4
572 +
573          itr = ''
574          if sites != [""]:#CarlosDaniele
575 <            for site in sites:
575 >            ##Addedd Daniele
576 >            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
577 >            if len(replicas)!=0:
578 >                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
579 >              
580 >            if len(replicas)==0:
581 >                msg = 'No sites remaining that host any part of the requested data! Exiting... '
582 >                raise CrabException(msg)
583 >            #####        
584 >           # for site in sites:
585 >            for site in replicas:
586                  #itr = itr + 'target.GlueSEUniqueID==&quot;'+site+'&quot; || '
587                  itr = itr + 'target.GlueSEUniqueID=="'+site+'" || '
588              itr = itr[0:-4]
# Line 620 | Line 590 | class SchedulerEdg(Scheduler):
590          return itr4
591  
592      def createXMLSchScript(self, nj, argsList):
623   # def createXMLSchScript(self, nj):
593        
594          """
595          Create a XML-file for BOSS4.
# Line 636 | Line 605 | class SchedulerEdg(Scheduler):
605          jbt = job.type()
606          
607          inp_sandbox = jbt.inputSandbox(index)
608 <        out_sandbox = jbt.outputSandbox(index)
608 >        #out_sandbox = jbt.outputSandbox(index)
609          """
610          [end] FIX-ME
611          """
# Line 652 | Line 621 | class SchedulerEdg(Scheduler):
621          dir = string.split(common.work_space.topDir(), '/')
622          taskName = dir[len(dir)-2]
623    
655        to_writeReq = ''
624          to_write = ''
625  
626          req=' '
627          req = req + jbt.getRequirements()
628  
661
662        #sites = common.jobDB.destination(nj)
663        #if len(sites)>0 and sites[0]!="Any":
664        #    req = req + ' && anyMatch(other.storage.CloseSEs, (_ITR4_))'
665        #req = req    
666    
629          if self.EDG_requirements:
630              if (req == ' '):
631                  req = req + self.EDG_requirements
# Line 706 | Line 668 | class SchedulerEdg(Scheduler):
668              to_write = to_write + 'RetryCount = "'+self.EDG_retry_count+'"\n'
669              pass
670  
671 +        if ( self.EDG_shallow_retry_count ):              
672 +            to_write = to_write + 'ShallowRetryCount = "'+self.EDG_shallow_retry_count+'"\n'
673 +            pass
674 +
675          to_write = to_write + 'MyProxyServer = "&quot;' + self.proxyServer + '&quot;"\n'
676          to_write = to_write + 'VirtualOrganisation = "&quot;' + self.VO + '&quot;"\n'
677  
678 <                #TaskName  
678 >        #TaskName  
679          dir = string.split(common.work_space.topDir(), '/')
680          taskName = dir[len(dir)-2]
681  
682          xml.write(str(title))
683 <        xml.write('<task name="' +str(taskName)+'">\n')
683 >        #xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
684 >
685 >        #xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n')
686 >        x509_cmd = 'ls /tmp/x509up_u`id -u`'
687 >        x509=runCommand(x509_cmd).strip()
688 >        xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + str(x509) + '">\n')
689          xml.write(jt_string)
690          
691          if (to_write != ''):
# Line 741 | Line 712 | class SchedulerEdg(Scheduler):
712          indy: here itr4
713          '''
714          
715 <
716 <        xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
715 >        xml.write('<chain name="' +str(taskName)+'__ITR1_" scheduler="'+str(self.schedulerName)+'">\n')
716 >       # xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
717          xml.write(jt_string)
718  
719          #executable
# Line 755 | Line 726 | class SchedulerEdg(Scheduler):
726          xml.write('<program>\n')
727          xml.write('<exec> ' + os.path.basename(script) +' </exec>\n')
728          xml.write(jt_string)
729 <    
759 <          
760 <        ### only one .sh  JDL has arguments:
761 <        ### Fabio
762 < #        xml.write('args = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'"\n')
729 >
730          xml.write('<args> <![CDATA[\n _ITR2_ \n]]> </args>\n')
731          xml.write('<program_types> crabjob </program_types>\n')
732 <        inp_box = script + ','
732 >        inp_box = common.work_space.pathForTgz() + 'job/' + jbt.scriptName + ','
733  
734          if inp_sandbox != None:
735              for fl in inp_sandbox:
# Line 770 | Line 737 | class SchedulerEdg(Scheduler):
737                  pass
738              pass
739  
740 <        inp_box = inp_box + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + ',' +\
741 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + ','+\
742 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + ','+\
743 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + ','+\
744 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py')
745 <
779 <        if (not jbt.additional_inbox_files == []):
780 <            inp_box = inp_box + ','
781 <            for addFile in jbt.additional_inbox_files:
782 <                addFile = os.path.abspath(addFile)
783 <                inp_box = inp_box+''+addFile+','
784 <                pass
740 > #        if (not jbt.additional_inbox_files == []):
741 > #            inp_box = inp_box + ','
742 > #            for addFile in jbt.additional_inbox_files:
743 > #                #addFile = os.path.abspath(addFile)
744 > #                inp_box = inp_box+''+addFile+','
745 > #                pass
746  
747          if inp_box[-1] == ',' : inp_box = inp_box[:-1]
748          inp_box = '<infiles> <![CDATA[\n' + inp_box + '\n]]> </infiles>\n'
# Line 812 | Line 773 | class SchedulerEdg(Scheduler):
773          INDY
774          something similar should be also done for infiles (if it makes sense!)
775          """
776 +        # Stuff to be returned _always_ via sandbox
777 +        for fl in jbt.output_file_sandbox:
778 +            out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ','
779 +            pass
780 +        pass
781 +
782 +        # via sandbox iif required return_data
783          if int(self.return_data) == 1:
784              for fl in jbt.output_file:
785                  out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ','
# Line 840 | Line 808 | class SchedulerEdg(Scheduler):
808          Function to check the Globus proxy.
809          """
810          if (self.proxyValid): return
811 <        timeleft = -999
811 >
812 >        ### Just return if asked to do so
813 >        if (self.dontCheckProxy==1):
814 >            self.proxyValid=1
815 >            return
816 >
817          minTimeLeft=10*3600 # in seconds
818  
819          minTimeLeftServer = 100 # in hours
# Line 860 | Line 833 | class SchedulerEdg(Scheduler):
833          pass
834  
835          if mustRenew:
836 <            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
837 <            cmd = 'voms-proxy-init -voms '+self.VO+' -valid 96:00'
836 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 192h\n")
837 >            cmd = 'voms-proxy-init -voms '+self.VO
838 >            if self.group:
839 >                cmd += ':/'+self.VO+'/'+self.group
840              if self.role:
841 <                cmd = 'voms-proxy-init -voms '+self.VO+':/'+self.VO+'/role='+self.role+' -valid 96:00'
841 >                cmd += '/role='+self.role
842 >            cmd += ' -valid 192:00'
843              try:
844                  # SL as above: damn it!
845 +                common.logger.debug(10,cmd)
846                  out = os.system(cmd)
847                  if (out>0): raise CrabException("Unable to create a valid proxy!\n")
848              except:
# Line 885 | Line 862 | class SchedulerEdg(Scheduler):
862              reTime = re.compile( r'timeleft: (\d+)' )
863              #print "<"+str(reTime.search( cmd_out ).group(1))+">"
864              if reTime.match( cmd_out ):
865 <                time = reTime.search( line ).group(1)
865 >                time = reTime.search( cmd_out ).group(1)
866                  if time < minTimeLeftServer:
867                      renewProxy = 1
868                      common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
# Line 911 | Line 888 | class SchedulerEdg(Scheduler):
888          if self.edg_config_vo:
889              edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
890          return edg_ui_cfg_opt
891 +
892 +    def submitTout(self, list):
893 +        return 120
894 +
895 +

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines