ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/SchedulerEdg.py
(Generate patch)

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.82 by spiga, Thu Sep 21 18:39:36 2006 UTC vs.
Revision 1.137 by spiga, Tue Sep 18 15:17:55 2007 UTC

# Line 3 | Line 3 | from crab_logger import Logger
3   from crab_exceptions import *
4   from crab_util import *
5   from EdgConfig import *
6 + from BlackWhiteListParser import BlackWhiteListParser
7   import common
8  
9   import os, sys, time
# Line 22 | Line 23 | class SchedulerEdg(Scheduler):
23  
24      def configure(self, cfg_params):
25  
26 +        # init BlackWhiteListParser
27 +        self.blackWhiteListParser = BlackWhiteListParser(cfg_params)
28 +
29 +        self.proxyValid=0
30 +        try: self.dontCheckProxy=int(cfg_params["EDG.dont_check_proxy"])
31 +        except KeyError: self.dontCheckProxy = 0
32 +
33          try:
34 <            RB = cfg_params["EDG.rb"]
35 <            edgConfig = EdgConfig(RB)
28 <            self.edg_config = edgConfig.config()
29 <            self.edg_config_vo = edgConfig.configVO()
34 >            RB=cfg_params["EDG.rb"]
35 >            self.rb_param_file=self.rb_configure(RB)
36          except KeyError:
37 <            self.edg_config = ''
38 <            self.edg_config_vo = ''
33 <
37 >            self.rb_param_file=''
38 >            pass
39          try:
40              self.proxyServer = cfg_params["EDG.proxy_server"]
41          except KeyError:
# Line 38 | Line 43 | class SchedulerEdg(Scheduler):
43          common.logger.debug(5,'Setting myproxy server to '+self.proxyServer)
44  
45          try:
46 +            self.group = cfg_params["EDG.group"]
47 +        except KeyError:
48 +            self.group = None
49 +            
50 +        try:
51              self.role = cfg_params["EDG.role"]
52          except KeyError:
53              self.role = None
# Line 45 | Line 55 | class SchedulerEdg(Scheduler):
55          try: self.LCG_version = cfg_params["EDG.lcg_version"]
56          except KeyError: self.LCG_version = '2'
57  
48        try: self.EDG_requirements = cfg_params['EDG.requirements']
49        except KeyError: self.EDG_requirements = ''
50
51        try: self.EDG_retry_count = cfg_params['EDG.retry_count']
52        except KeyError: self.EDG_retry_count = ''
53
58          try:
59              self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
56            #print "self.EDG_ce_black_list = ", self.EDG_ce_black_list
60          except KeyError:
61              self.EDG_ce_black_list  = ''
62  
63          try:
64              self.EDG_ce_white_list = cfg_params['EDG.ce_white_list']
62            #print "self.EDG_ce_white_list = ", self.EDG_ce_white_list
65          except KeyError: self.EDG_ce_white_list = ''
66  
67          try: self.VO = cfg_params['EDG.virtual_organization']
68          except KeyError: self.VO = 'cms'
69  
70 <        try: self.return_data = cfg_params['USER.return_data']
69 <        except KeyError: self.return_data = 1
70 <
71 <        try:
72 <             self.copy_input_data = common.analisys_common_info['copy_input_data']
73 <             #print "self.copy_input_data = ", self.copy_input_data
70 >        try: self.copy_input_data = cfg_params["USER.copy_input_data"]
71          except KeyError: self.copy_input_data = 0
72  
73 +        try: self.return_data = cfg_params['USER.return_data']
74 +        except KeyError: self.return_data = 0
75 +
76          try:
77              self.copy_data = cfg_params["USER.copy_data"]
78              if int(self.copy_data) == 1:
# Line 91 | Line 91 | class SchedulerEdg(Scheduler):
91             msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
92             raise CrabException(msg)
93  
94 +        ########### FEDE FOR DBS2 ##############################
95 +        try:
96 +            self.publish_data = cfg_params["USER.publish_data"]
97 +            self.checkProxy()
98 +            if int(self.publish_data) == 1:
99 +                try:
100 +                    self.publish_data_name = cfg_params['USER.publish_data_name']
101 +                except KeyError:
102 +                    msg = "Error. The [USER] section does not have 'publish_data_name'"
103 +                    raise CrabException(msg)
104 +                try:
105 +                    tmp = runCommand("voms-proxy-info -identity")
106 +                    tmp = string.split(tmp,'/')
107 +                    reCN=re.compile(r'CN=')
108 +                    for t in tmp:
109 +                        if reCN.match(t):
110 +                            self.UserGridName=string.strip((t.replace('CN=','')).replace(' ',''))
111 +                        
112 +                    #self.UserGridName = string.strip(runCommand("voms-proxy-info -identity | awk -F\'CN\' \'{print $2$3$4}\' | tr -d \'=/ \'"))
113 +                except:
114 +                    msg = "Error. Problem with voms-proxy-info -identity command"
115 +                    raise CrabException(msg)
116 +        except KeyError: self.publish_data = 0
117 +
118 +        if ( int(self.copy_data) == 0 and int(self.publish_data) == 1 ):
119 +           msg = 'Warning: publish_data = 1 must be used with copy_data = 1\n'
120 +           msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
121 +           common.logger.message(msg)
122 +           raise CrabException(msg)
123 +        #################################################
124 +
125          try:
126              self.lfc_host = cfg_params['EDG.lfc_host']
127          except KeyError:
# Line 133 | Line 164 | class SchedulerEdg(Scheduler):
164  
165          try: self.EDG_requirements = cfg_params['EDG.requirements']
166          except KeyError: self.EDG_requirements = ''
167 <                                                                                                                                                            
167 >
168 >        try: self.EDG_addJdlParam = string.split(cfg_params['EDG.additional_jdl_parameters'],',')
169 >        except KeyError: self.EDG_addJdlParam = []
170 >
171          try: self.EDG_retry_count = cfg_params['EDG.retry_count']
172          except KeyError: self.EDG_retry_count = ''
173 <                                                                                                                                                            
173 >
174 >        try: self.EDG_shallow_retry_count= cfg_params['EDG.shallow_retry_count']
175 >        except KeyError: self.EDG_shallow_retry_count = ''
176 >
177          try: self.EDG_clock_time = cfg_params['EDG.max_wall_clock_time']
178          except KeyError: self.EDG_clock_time= ''
179 <                                                                                                                                                            
179 >
180          try: self.EDG_cpu_time = cfg_params['EDG.max_cpu_time']
181          except KeyError: self.EDG_cpu_time = ''
182  
# Line 156 | Line 193 | class SchedulerEdg(Scheduler):
193          libPath=os.path.join(path, "lib", "python")
194          sys.path.append(libPath)
195  
159        self.proxyValid=0
160
196          try:
197              self._taskId = cfg_params['taskId']
198          except:
# Line 172 | Line 207 | class SchedulerEdg(Scheduler):
207          return
208      
209  
210 +    def rb_configure(self, RB):
211 +        self.edg_config = ''
212 +        self.edg_config_vo = ''
213 +        self.rb_param_file = ''
214 +
215 +        edgConfig = EdgConfig(RB)
216 +        self.edg_config = edgConfig.config()
217 +        self.edg_config_vo = edgConfig.configVO()
218 +
219 +        if (self.edg_config and self.edg_config_vo != ''):
220 +            self.rb_param_file = 'RBconfig = "'+self.edg_config+'";\nRBconfigVO = "'+self.edg_config_vo+'";\n'
221 +            #print "rb_param_file = ", self.rb_param_file
222 +        return self.rb_param_file
223 +      
224 +
225      def sched_parameter(self):
226          """
227 <        Returns file with scheduler-specific parameters
227 >        Returns file with requirements and scheduler-specific parameters
228          """
229 <      
230 <        if (self.edg_config and self.edg_config_vo != ''):
231 <            self.param='sched_param.clad'
229 >        index = int(common.jobDB.nJobs()) - 1
230 >        job = common.job_list[index]
231 >        jbt = job.type()
232 >        
233 >        lastBlock=-1
234 >        first = []
235 >        for n in range(common.jobDB.nJobs()):
236 >            currBlock=common.jobDB.block(n)
237 >            if (currBlock!=lastBlock):
238 >                lastBlock = currBlock
239 >                first.append(n)
240 >  
241 >        req = ''
242 >        req = req + jbt.getRequirements()
243 >    
244 >        if self.EDG_requirements:
245 >            if (req == ' '):
246 >                req = req + self.EDG_requirements
247 >            else:
248 >                req = req +  ' && ' + self.EDG_requirements
249 >
250 >        if self.EDG_ce_white_list:
251 >            ce_white_list = string.split(self.EDG_ce_white_list,',')
252 >            for i in range(len(ce_white_list)):
253 >                if i == 0:
254 >                    if (req == ' '):
255 >                        req = req + '((RegExp("' + string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
256 >                    else:
257 >                        req = req +  ' && ((RegExp("' +  string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
258 >                    pass
259 >                else:
260 >                    req = req +  ' || (RegExp("' +  string.strip(ce_white_list[i]) + '", other.GlueCEUniqueId))'
261 >            req = req + ')'
262 >        
263 >        if self.EDG_ce_black_list:
264 >            ce_black_list = string.split(self.EDG_ce_black_list,',')
265 >            for ce in ce_black_list:
266 >                if (req == ' '):
267 >                    req = req + '(!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))'
268 >                else:
269 >                    req = req +  ' && (!RegExp("' + string.strip(ce) + '", other.GlueCEUniqueId))'
270 >                pass
271 >        if self.EDG_clock_time:
272 >            if (req == ' '):
273 >                req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
274 >            else:
275 >                req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
276 >
277 >        if self.EDG_cpu_time:
278 >            if (req == ' '):
279 >                req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
280 >            else:
281 >                req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
282 >                
283 >        for i in range(len(first)): # Add loop DS
284 >            groupReq = req
285 >            self.param='sched_param_'+str(i)+'.clad'
286              param_file = open(common.work_space.shareDir()+'/'+self.param, 'w')
287 <            param_file.write('RBconfig = "'+self.edg_config+'";\n')  
288 <            param_file.write('RBconfigVO = "'+self.edg_config_vo+'";')
287 >
288 >            itr4=self.findSites_(first[i])
289 >            for arg in itr4:
290 >                groupReq = groupReq + ' && anyMatch(other.storage.CloseSEs, ('+str(arg)+'))'
291 >            param_file.write('Requirements = '+groupReq +';\n')  
292 >  
293 >            if (self.rb_param_file != ''):
294 >                param_file.write(self.rb_param_file)  
295 >
296 >            if len(self.EDG_addJdlParam):
297 >                for p in self.EDG_addJdlParam:
298 >                    param_file.write(p)
299 >
300              param_file.close()  
301 <            return 1
187 <        else:
188 <            return 0
301 >
302  
303      def wsSetupEnvironment(self):
304          """
# Line 210 | Line 323 | class SchedulerEdg(Scheduler):
323          txt += 'echo "MonitorID=`echo $MonitorID`" | tee -a $RUNTIME_AREA/$repo\n'
324  
325          txt += 'echo "middleware discovery " \n'
326 <        txt += 'if [ $VO_CMS_SW_DIR ]; then \n'
214 <        txt += '    middleware=LCG \n'
215 <        txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
216 <        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
217 <        txt += '    echo "middleware =$middleware" \n'
218 <        txt += 'elif [ $GRID3_APP_DIR ]; then\n'
326 >        txt += 'if [ $GRID3_APP_DIR ]; then\n'
327          txt += '    middleware=OSG \n'
328          txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
329          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
# Line 225 | Line 333 | class SchedulerEdg(Scheduler):
333          txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
334          txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
335          txt += '    echo "middleware =$middleware" \n'
336 +        txt += 'elif [ $VO_CMS_SW_DIR ]; then \n'
337 +        txt += '    middleware=LCG \n'
338 +   #     txt += '    echo "SyncCE=`edg-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
339 +        txt += '    echo "SyncCE=`glite-brokerinfo getCE`" | tee -a $RUNTIME_AREA/$repo \n'
340 +        txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
341 +        txt += '    echo "middleware =$middleware" \n'
342          txt += 'else \n'
343          txt += '    echo "SET_CMS_ENV 10030 ==> middleware not identified" \n'
344          txt += '    echo "JOB_EXIT_STATUS = 10030" \n'
# Line 244 | Line 358 | class SchedulerEdg(Scheduler):
358          
359          txt += '\n\n'
360  
361 <        if int(self.copy_data) == 1:
362 <           if self.SE:
363 <              txt += 'export SE='+self.SE+'\n'
364 <              txt += 'echo "SE = $SE"\n'
365 <           if self.SE_PATH:
366 <              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
367 <              txt += 'export SE_PATH='+self.SE_PATH+'\n'
368 <              txt += 'echo "SE_PATH = $SE_PATH"\n'
361 > #        if int(self.copy_data) == 1:
362 > #           if self.SE:
363 > #              txt += 'export SE='+self.SE+'\n'
364 > #              txt += 'echo "SE = $SE"\n'
365 > #           if self.SE_PATH:
366 > #              if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
367 > #              txt += 'export SE_PATH='+self.SE_PATH+'\n'
368 > #              txt += 'echo "SE_PATH = $SE_PATH"\n'
369  
370          txt += 'export VO='+self.VO+'\n'
371 <        ### FEDE: add some line for LFC catalog setting
371 >        ### add some line for LFC catalog setting
372          txt += 'if [ $middleware == LCG ]; then \n'
373          txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
374          txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
# Line 295 | Line 409 | class SchedulerEdg(Scheduler):
409                txt += '\n'
410  
411          txt += 'if [ $middleware == LCG ]; then\n'
412 <        txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
412 >    #    txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
413 >        txt += '    CloseCEs=`glite-brokerinfo getCE`\n'
414          txt += '    echo "CloseCEs = $CloseCEs"\n'
415          txt += '    CE=`echo $CloseCEs | sed -e "s/:.*//"`\n'
416          txt += '    echo "CE = $CE"\n'
# Line 321 | Line 436 | class SchedulerEdg(Scheduler):
436          Copy input data from SE to WN    
437          """
438          txt = ''
439 <        try:
440 <            self.copy_input_data = common.analisys_common_info['copy_input_data']
326 <        except KeyError: self.copy_input_data = 0
327 <        if int(self.copy_input_data) == 1:
439 >        if not self.copy_input_data: return txt
440 >
441          ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
442 <           txt += 'if [ $middleware == OSG ]; then\n'
443 <           txt += '   #\n'
444 <           txt += '   #   Copy Input Data from SE to this WN deactivated in OSG mode\n'
445 <           txt += '   #\n'
446 <           txt += '   echo "Copy Input Data from SE to this WN deactivated in OSG mode"\n'
447 <           txt += 'elif [ $middleware == LCG ]; then \n'
448 <           txt += '   #\n'
449 <           txt += '   #   Copy Input Data from SE to this WN\n'
450 <           txt += '   #\n'
451 <           ### changed by georgia (put a loop copying more than one input files per jobs)          
452 <           txt += '   for input_file in $cur_file_list \n'
453 <           txt += '   do \n'
454 <           txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
455 <           txt += '      copy_input_exit_status=$?\n'
456 <           txt += '      echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n'
457 <           txt += '      if [ $copy_input_exit_status -ne 0 ]; then \n'
458 <           txt += '         echo "Problems with copying to WN" \n'
459 <           txt += '      else \n'
460 <           txt += '         echo "input copied into WN" \n'
461 <           txt += '      fi \n'
462 <           txt += '   done \n'
463 <           ### copy a set of PU ntuples (same for each jobs -- but accessed randomly)
464 <           txt += '   for file in $cur_pu_list \n'
465 <           txt += '   do \n'
466 <           txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
467 <           txt += '      copy_input_pu_exit_status=$?\n'
468 <           txt += '      echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n'
469 <           txt += '      if [ $copy_input_pu_exit_status -ne 0 ]; then \n'
470 <           txt += '         echo "Problems with copying pu to WN" \n'
471 <           txt += '      else \n'
472 <           txt += '         echo "input pu files copied into WN" \n'
473 <           txt += '      fi \n'
474 <           txt += '   done \n'
475 <           txt += '   \n'
476 <           txt += '   ### Check SCRATCH space available on WN : \n'
477 <           txt += '   df -h \n'
478 <           txt += 'fi \n'
442 >        txt += 'if [ $middleware == OSG ]; then\n'
443 >        txt += '   #\n'
444 >        txt += '   #   Copy Input Data from SE to this WN deactivated in OSG mode\n'
445 >        txt += '   #\n'
446 >        txt += '   echo "Copy Input Data from SE to this WN deactivated in OSG mode"\n'
447 >        txt += 'elif [ $middleware == LCG ]; then \n'
448 >        txt += '   #\n'
449 >        txt += '   #   Copy Input Data from SE to this WN\n'
450 >        txt += '   #\n'
451 >        ### changed by georgia (put a loop copying more than one input files per jobs)          
452 >        txt += '   for input_file in $cur_file_list \n'
453 >        txt += '   do \n'
454 >        txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$input_lfn/$input_file file:`pwd`/$input_file 2>&1\n'
455 >        txt += '      copy_input_exit_status=$?\n'
456 >        txt += '      echo "COPY_INPUT_EXIT_STATUS = $copy_input_exit_status"\n'
457 >        txt += '      if [ $copy_input_exit_status -ne 0 ]; then \n'
458 >        txt += '         echo "Problems with copying to WN" \n'
459 >        txt += '      else \n'
460 >        txt += '         echo "input copied into WN" \n'
461 >        txt += '      fi \n'
462 >        txt += '   done \n'
463 >        ### copy a set of PU ntuples (same for each jobs -- but accessed randomly)
464 >        txt += '   for file in $cur_pu_list \n'
465 >        txt += '   do \n'
466 >        txt += '      lcg-cp --vo $VO --verbose -t 1200 lfn:$pu_lfn/$file file:`pwd`/$file 2>&1\n'
467 >        txt += '      copy_input_pu_exit_status=$?\n'
468 >        txt += '      echo "COPY_INPUT_PU_EXIT_STATUS = $copy_input_pu_exit_status"\n'
469 >        txt += '      if [ $copy_input_pu_exit_status -ne 0 ]; then \n'
470 >        txt += '         echo "Problems with copying pu to WN" \n'
471 >        txt += '      else \n'
472 >        txt += '         echo "input pu files copied into WN" \n'
473 >        txt += '      fi \n'
474 >        txt += '   done \n'
475 >        txt += '   \n'
476 >        txt += '   ### Check SCRATCH space available on WN : \n'
477 >        txt += '   df -h \n'
478 >        txt += 'fi \n'
479            
480          return txt
481  
# Line 371 | Line 484 | class SchedulerEdg(Scheduler):
484          Write a CopyResults part of a job script, e.g.
485          to copy produced output into a storage element.
486          """
487 <        txt = ''
375 <        if int(self.copy_data) == 1:
376 <           txt += '#\n'
377 <           txt += '#   Copy output to SE = $SE\n'
378 <           txt += '#\n'
379 <           txt += '    if [ $middleware == OSG ]; then\n'
380 <           txt += '        echo "X509_USER_PROXY = $X509_USER_PROXY"\n'
381 <           txt += '        echo "source $OSG_APP/glite/setup_glite_ui.sh"\n'
382 <           txt += '        source $OSG_APP/glite/setup_glite_ui.sh\n'
383 <           txt += '        export X509_CERT_DIR=$OSG_APP/glite/etc/grid-security/certificates\n'
384 <           txt += '        echo "export X509_CERT_DIR=$X509_CERT_DIR"\n'
385 <           txt += '    fi \n'
386 <           txt += '    for out_file in $file_list ; do\n'
387 <           txt += '        echo "Trying to copy output file to $SE using lcg-cp"\n'
388 <           if common.logger.debugLevel() >= 5:
389 <               txt += '        echo "lcg-cp --vo $VO -t 2400 --verbose file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
390 <               txt += '        exitstring=`lcg-cp --vo $VO -t 2400 --verbose file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
391 <           else:
392 <               txt += '        echo "lcg-cp --vo $VO -t 2400 file://`pwd`/$out_file gsiftp://${SE}${SE_PATH}$out_file"\n'
393 <               txt += '        exitstring=`lcg-cp --vo $VO -t 2400 file://\`pwd\`/$out_file gsiftp://${SE}${SE_PATH}$out_file 2>&1`\n'
394 <           txt += '        copy_exit_status=$?\n'
395 <           txt += '        echo "COPY_EXIT_STATUS for lcg-cp = $copy_exit_status"\n'
396 <           txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
397 <           txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
398 <           txt += '            echo "Possible problem with SE = $SE"\n'
399 <           txt += '            echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
400 <           txt += '            echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
401 <           txt += '            echo "lcg-cp failed.  For verbose lcg-cp output, use command line option -debug 5."\n'
402 <           txt += '            echo "lcg-cp failed, attempting srmcp"\n'
403 <           txt += '            echo "mkdir -p $HOME/.srmconfig"\n'
404 <           txt += '            mkdir -p $HOME/.srmconfig\n'
405 <           txt += '            if [ $middleware == LCG ]; then\n'
406 <           txt += '               echo "srmcp -retry_num 5 -retry_timeout 480000 file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
407 <           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 480000 file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
408 <           txt += '            elif [ $middleware == OSG ]; then\n'
409 <           txt += '               echo "srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////`pwd`/$out_file srm://${SE}:8443${SE_PATH}$out_file"\n'
410 <           txt += '               exitstring=`srmcp -retry_num 5 -retry_timeout 240000 -x509_user_trusted_certificates $OSG_APP/glite/etc/grid-security/certificates file:////\`pwd\`/$out_file srm://${SE}:8443${SE_PATH}$out_file 2>&1`\n'
411 <           txt += '            fi \n'
412 <           txt += '            copy_exit_status=$?\n'
413 <           txt += '            echo "COPY_EXIT_STATUS for srm = $copy_exit_status"\n'
414 <           txt += '            echo "STAGE_OUT = $copy_exit_status"\n'
415 <           txt += '            if [ $copy_exit_status -ne 0 ]; then\n'
416 <           txt += '               echo "Problems with SE = $SE"\n'
417 <           txt += '               echo "StageOutExitStatus = 198" | tee -a $RUNTIME_AREA/$repo\n'
418 <           txt += '               echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
419 <           txt += '               echo "lcg-cp and srm failed"\n'
420 <           txt += '               echo "If storage_path in your config file contains a ? you may need a \? instead."\n'
421 <           txt += '            else\n'
422 <           txt += '               echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
423 <           txt += '               echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
424 <           txt += '               echo "output copied into $SE/$SE_PATH directory"\n'
425 <           txt += '               echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
426 <           txt += '               echo "srmcp succeeded"\n'
427 <           txt += '            fi\n'
428 <           txt += '        else\n'
429 <           txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
430 <           txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
431 <           txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
432 <           txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
433 <           txt += '            echo "lcg-cp succeeded"\n'
434 <           txt += '         fi\n'
435 <           txt += '     done\n'
436 <        return txt
487 >        txt = '\n'
488  
489 <    def wsRegisterOutput(self):
490 <        """
491 <        Returns part of a job script which does scheduler-specific work.
441 <        """
489 >        txt += '#\n'
490 >        txt += '# COPY OUTPUT FILE TO SE\n'
491 >        txt += '#\n\n'
492  
493 <        txt = ''
494 <        if int(self.register_data) == 1:
495 <        ## OLI_Daniele deactivate for OSG (wait for LCG UI installed on OSG)
496 <           txt += 'if [ $middleware == OSG ]; then\n'
497 <           txt += '   #\n'
498 <           txt += '   #   Register output to LFC deactivated in OSG mode\n'
499 <           txt += '   #\n'
500 <           txt += '   echo "Register output to LFC deactivated in OSG mode"\n'
501 <           txt += 'elif [ $middleware == LCG ]; then \n'
502 <           txt += '#\n'
503 <           txt += '#  Register output to LFC\n'
504 <           txt += '#\n'
505 <           txt += '   if [ $copy_exit_status -eq 0 ]; then\n'
506 <           txt += '      for out_file in $file_list ; do\n'
507 <           txt += '         echo "Trying to register the output file into LFC"\n'
508 <           txt += '         echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1"\n'
509 <           txt += '         lcg-rf -l $LFN/$out_file --vo $VO -t 1200 sfn://$SE$SE_PATH/$out_file 2>&1 \n'
510 <           txt += '         register_exit_status=$?\n'
511 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
512 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
513 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
514 <           txt += '            echo "Problems with the registration to LFC" \n'
515 <           txt += '            echo "Try with srm protocol" \n'
516 <           txt += '            echo "lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1"\n'
517 <           txt += '            lcg-rf -l $LFN/$out_file --vo $VO -t 1200 srm://$SE$SE_PATH/$out_file 2>&1 \n'
518 <           txt += '            register_exit_status=$?\n'
519 <           txt += '            echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
520 <           txt += '            echo "STAGE_OUT = $register_exit_status"\n'
521 <           txt += '            if [ $register_exit_status -ne 0 ]; then \n'
522 <           txt += '               echo "Problems with the registration into LFC" \n'
523 <           txt += '            fi \n'
524 <           txt += '         else \n'
525 <           txt += '            echo "output registered to LFC"\n'
526 <           txt += '         fi \n'
527 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
528 <           txt += '      done\n'
529 <           txt += '   else \n'
530 <           txt += '      echo "Trying to copy output file to CloseSE"\n'
531 <           txt += '      CLOSE_SE=`edg-brokerinfo getCloseSEs | head -1`\n'
532 <           txt += '      for out_file in $file_list ; do\n'
533 <           txt += '         echo "lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1" \n'
534 <           txt += '         lcg-cr -v -l lfn:${LFN}/$out_file -d $CLOSE_SE -P $LFN/$out_file --vo $VO file://$RUNTIME_AREA/$out_file 2>&1 \n'
535 <           txt += '         register_exit_status=$?\n'
536 <           txt += '         echo "REGISTER_EXIT_STATUS = $register_exit_status"\n'
537 <           txt += '         echo "STAGE_OUT = $register_exit_status"\n'
538 <           txt += '         if [ $register_exit_status -ne 0 ]; then \n'
539 <           txt += '            echo "Problems with CloseSE or Catalog" \n'
540 <           txt += '         else \n'
491 <           txt += '            echo "The program was successfully executed"\n'
492 <           txt += '            echo "SE = $CLOSE_SE"\n'
493 <           txt += '            echo "LFN for the file is LFN=${LFN}/$out_file"\n'
494 <           txt += '         fi \n'
495 <           txt += '         echo "StageOutExitStatus = $register_exit_status" | tee -a $RUNTIME_AREA/$repo\n'
496 <           txt += '      done\n'
497 <           txt += '   fi \n'
498 <           txt += '   exit_status=$register_exit_status\n'
499 <           txt += 'fi \n'
493 >        SE_PATH=''
494 >        if int(self.copy_data) == 1:
495 >            if self.SE:
496 >                txt += 'export SE='+self.SE+'\n'
497 >                txt += 'echo "SE = $SE"\n'
498 >            if self.SE_PATH:
499 >                if ( self.SE_PATH[-1] != '/' ) : self.SE_PATH = self.SE_PATH + '/'
500 >                SE_PATH=self.SE_PATH
501 >            if int(self.publish_data) == 1:
502 >                txt += '### publish_data = 1 so the SE path where to copy the output is: \n'
503 >                path_add = self.UserGridName + '/' + self.publish_data_name +'_${PSETHASH}/'
504 >                SE_PATH = SE_PATH + path_add
505 >            txt += 'export SE_PATH='+SE_PATH+'\n'
506 >            txt += 'echo "SE_PATH = $SE_PATH"\n'
507 >            
508 >            txt += 'echo "####################################################"\n'
509 >            txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
510 >            txt += 'echo "####################################################"\n'
511 >            
512 >            txt += 'for out_file in $file_list ; do\n'
513 >            txt += '   echo "Trying to copy output file to $SE using srmcp"\n'
514 >            txt += '   cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
515 >            txt += '   copy_exit_status=$?\n'
516 >            txt += '   echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
517 >            txt += '   echo "STAGE_OUT = $copy_exit_status"\n'
518 >            
519 >            txt += '   if [ $copy_exit_status -ne 0 ]; then\n'
520 >            txt += '       echo "Problem copying $out_file to $SE $SE_PATH"\n'
521 >            txt += '       echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
522 >            txt += '       echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
523 >            txt += '       copy_exit_status=60307\n'
524 >            
525 >            txt += '   else\n'
526 >            txt += '       echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
527 >            txt += '       echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
528 >            txt += '       echo "output copied into $SE/$SE_PATH directory"\n'
529 >            txt += '       echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
530 >            txt += '       echo "srmcp succeeded"\n'
531 >            txt += '    fi\n'
532 >            txt += 'done\n'
533 >            txt += 'if [ $copy_exit_status -ne 0 ]; then\n'
534 >            txt += '      SE=""\n'
535 >            txt += '      echo "SE = $SE"\n'
536 >            txt += '      SE_PATH=""\n'
537 >            txt += '      echo "SE_PATH = $SE_PATH"\n'
538 >            txt += 'fi\n'
539 >            txt += 'exit_status=$copy_exit_status\n'
540 >            pass
541          return txt
542  
543      def loggingInfo(self, id):
# Line 505 | Line 546 | class SchedulerEdg(Scheduler):
546          """
547          self.checkProxy()
548          cmd = 'edg-job-get-logging-info -v 2 ' + id
508        #cmd_out = os.popen(cmd)
549          cmd_out = runCommand(cmd)
550          return cmd_out
551  
512    def getExitStatus(self, id):
513        return self.getStatusAttribute_(id, 'exit_code')
514
515    def queryStatus(self, id):
516        return self.getStatusAttribute_(id, 'status')
517
518    def queryDest(self, id):  
519        return self.getStatusAttribute_(id, 'destination')
520
521
522    def getStatusAttribute_(self, id, attr):
523        """ Query a status of the job with id """
524
525        self.checkProxy()
526        hstates = {}
527        Status = importName('edg_wl_userinterface_common_LbWrapper', 'Status')
528        # Bypass edg-job-status interfacing directly to C++ API
529        # Job attribute vector to retrieve status without edg-job-status
530        level = 0
531        # Instance of the Status class provided by LB API
532        jobStat = Status()
533        st = 0
534        jobStat.getStatus(id, level)
535        err, apiMsg = jobStat.get_error()
536        if err:
537            common.logger.debug(5,'Error caught' + apiMsg)
538            return None
539        else:
540            for i in range(len(self.states)):
541                # Fill an hash table with all information retrieved from LB API
542                hstates[ self.states[i] ] = jobStat.loadStatus(st)[i]
543            result = jobStat.loadStatus(st)[self.states.index(attr)]
544            return result
545
552      def queryDetailedStatus(self, id):
553          """ Query a detailed status of the job with id """
554          cmd = 'edg-job-status '+id
555          cmd_out = runCommand(cmd)
556          return cmd_out
557  
558 <    ##### FEDE ######        
559 <    def findSites_(self, n_tot_job):
560 <        itr4 = []
561 <       # print "n_tot_job = ", n_tot_job
562 <        for n in range(n_tot_job):
563 <            sites = common.jobDB.destination(n)
564 <            if len(sites)>0 and sites[0]=="Any": continue
565 <
566 <            #job = common.job_list[n]
567 <            #jbt = job.type()
568 <           # print "common.jobDB.destination(n) = ", common.jobDB.destination(n)
569 <           # print "sites = ", sites
570 <            itr = ''
571 <            for site in sites:
558 >    def findSites_(self, n):
559 >        itr4 =[]
560 >
561 >        sites = common.jobDB.destination(n)
562 >
563 >        if len(sites)>0 and sites[0]=="":
564 >            return itr4
565 >
566 >        itr = ''
567 >        if sites != [""]:#CarlosDaniele
568 >            ##Addedd Daniele
569 >            replicas = self.blackWhiteListParser.checkBlackList(sites,n)
570 >            if len(replicas)!=0:
571 >                replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
572 >              
573 >            if len(replicas)==0:
574 >                msg = 'No sites remaining that host any part of the requested data! Exiting... '
575 >                raise CrabException(msg)
576 >            #####        
577 >           # for site in sites:
578 >            for site in replicas:
579                  #itr = itr + 'target.GlueSEUniqueID==&quot;'+site+'&quot; || '
580                  itr = itr + 'target.GlueSEUniqueID=="'+site+'" || '
568                pass
569            # remove last ||
581              itr = itr[0:-4]
582              itr4.append( itr )
572        # remove last ,
573       # print "itr4 = ", itr4
583          return itr4
584  
585      def createXMLSchScript(self, nj, argsList):
586 <   # def createXMLSchScript(self, nj):
586 >      
587          """
588          Create a XML-file for BOSS4.
589          """
590    #      job = common.job_list[nj]
591          """
592          INDY
593 <        [begin] da rivedere:
594 <        in particolare passerei il jobType ed eliminerei le dipendenze da job
593 >        [begin] FIX-ME:
594 >        I would pass jobType instead of job
595          """
596          index = nj - 1
597          job = common.job_list[index]
598          jbt = job.type()
599          
600          inp_sandbox = jbt.inputSandbox(index)
601 <        out_sandbox = jbt.outputSandbox(index)
601 >        #out_sandbox = jbt.outputSandbox(index)
602          """
603 <        [end] da rivedere
603 >        [end] FIX-ME
604          """
605  
606          
# Line 605 | Line 614 | class SchedulerEdg(Scheduler):
614          dir = string.split(common.work_space.topDir(), '/')
615          taskName = dir[len(dir)-2]
616    
608        to_writeReq = ''
617          to_write = ''
618  
619          req=' '
620          req = req + jbt.getRequirements()
621  
614
615        #sites = common.jobDB.destination(nj)
616        #if len(sites)>0 and sites[0]!="Any":
617        #    req = req + ' && anyMatch(other.storage.CloseSEs, (_ITR4_))'
618        #req = req    
619    
622          if self.EDG_requirements:
623              if (req == ' '):
624                  req = req + self.EDG_requirements
# Line 659 | Line 661 | class SchedulerEdg(Scheduler):
661              to_write = to_write + 'RetryCount = "'+self.EDG_retry_count+'"\n'
662              pass
663  
664 +        if ( self.EDG_shallow_retry_count ):              
665 +            to_write = to_write + 'ShallowRetryCount = "'+self.EDG_shallow_retry_count+'"\n'
666 +            pass
667 +
668          to_write = to_write + 'MyProxyServer = "&quot;' + self.proxyServer + '&quot;"\n'
669          to_write = to_write + 'VirtualOrganisation = "&quot;' + self.VO + '&quot;"\n'
670  
665
671          #TaskName  
672          dir = string.split(common.work_space.topDir(), '/')
673          taskName = dir[len(dir)-2]
674  
675          xml.write(str(title))
676 <        xml.write('<task name="' +str(taskName)+'">\n')
676 >        #xml.write('<task name="' +str(taskName)+'" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache">\n')
677 >
678 >        #xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + '" task_info="' + os.path.expandvars('X509_USER_PROXY') + '">\n')
679 >        x509_cmd = 'ls /tmp/x509up_u`id -u`'
680 >        x509=runCommand(x509_cmd).strip()
681 >        xml.write('<task name="' +str(taskName)+ '" sub_path="' +common.work_space.pathForTgz() + 'share/.boss_cache"' + ' task_info="' + str(x509) + '">\n')
682          xml.write(jt_string)
683 +        
684 +        if (to_write != ''):
685 +            xml.write('<extraTags\n')
686 +            xml.write(to_write)
687 +            xml.write('/>\n')
688 +            pass
689  
690          xml.write('<iterator>\n')
691          xml.write('\t<iteratorRule name="ITR1">\n')
# Line 686 | Line 702 | class SchedulerEdg(Scheduler):
702          xml.write('\t</iteratorRule>\n')
703  
704          '''
705 <        indy: qui sotto ci sta itr4
705 >        indy: here itr4
706          '''
707          
708 <        itr4=self.findSites_(nj)
709 <        #print "--->>> itr4 = ", itr4
694 <        if (itr4 != ''):
695 <            xml.write('\t<iteratorRule name="ITR4">\n')
696 <        #print argsList
697 <            for arg in itr4:
698 <                xml.write('\t\t<ruleElement> <![CDATA[\n'+ arg + '\n\t\t]]> </ruleElement>\n')
699 <                pass
700 <            xml.write('\t</iteratorRule>\n')
701 <            req = req + ' && anyMatch(other.storage.CloseSEs, (_ITR4_))'
702 <            pass
703 <        #    print "--->>> req= ", req        
704 <        
705 <        if (to_write != ''):
706 <            xml.write('<extraTags\n')
707 <            xml.write(to_write)
708 <            xml.write('/>\n')
709 <            pass
710 <
711 <        xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
708 >        xml.write('<chain name="' +str(taskName)+'__ITR1_" scheduler="'+str(self.schedulerName)+'">\n')
709 >       # xml.write('<chain scheduler="'+str(self.schedulerName)+'">\n')
710          xml.write(jt_string)
711  
714        if (req != ' '):
715            req = req + '\n'
716            xml.write('<extraTags>\n')
717            xml.write('<Requirements>\n')
718            xml.write('<![CDATA[\n')
719            xml.write(req)
720            xml.write(']]>\n')
721            xml.write('</Requirements>\n')
722            xml.write('</extraTags>\n')
723            pass
724
712          #executable
713  
714          """
715          INDY
716 <        script dipende dal jobType: dovrebbe essere semplice tirarlo fuori in altro modo
716 >        script depends on jobType: it should be probably get in a different way
717          """        
718          script = job.scriptFilename()
719          xml.write('<program>\n')
720          xml.write('<exec> ' + os.path.basename(script) +' </exec>\n')
721          xml.write(jt_string)
722 <    
736 <          
737 <        ### only one .sh  JDL has arguments:
738 <        ### Fabio
739 < #        xml.write('args = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'"\n')
722 >
723          xml.write('<args> <![CDATA[\n _ITR2_ \n]]> </args>\n')
724          xml.write('<program_types> crabjob </program_types>\n')
725 <        inp_box = script + ','
725 >        inp_box = common.work_space.pathForTgz() + 'job/' + jbt.scriptName + ','
726  
727          if inp_sandbox != None:
728              for fl in inp_sandbox:
# Line 747 | Line 730 | class SchedulerEdg(Scheduler):
730                  pass
731              pass
732  
733 <        inp_box = inp_box + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + ',' +\
734 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + ','+\
735 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + ','+\
736 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + ','+\
737 <                  os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py')
738 <
756 <        if (not jbt.additional_inbox_files == []):
757 <            inp_box = inp_box + ','
758 <            for addFile in jbt.additional_inbox_files:
759 <                addFile = os.path.abspath(addFile)
760 <                inp_box = inp_box+''+addFile+','
761 <                pass
733 > #        if (not jbt.additional_inbox_files == []):
734 > #            inp_box = inp_box + ','
735 > #            for addFile in jbt.additional_inbox_files:
736 > #                #addFile = os.path.abspath(addFile)
737 > #                inp_box = inp_box+''+addFile+','
738 > #                pass
739  
740          if inp_box[-1] == ',' : inp_box = inp_box[:-1]
741          inp_box = '<infiles> <![CDATA[\n' + inp_box + '\n]]> </infiles>\n'
# Line 787 | Line 764 | class SchedulerEdg(Scheduler):
764  
765          """
766          INDY
767 <        qualcosa del genere andrebbe fatta per gli infiles
768 <        """        
767 >        something similar should be also done for infiles (if it makes sense!)
768 >        """
769 >        # Stuff to be returned _always_ via sandbox
770 >        for fl in jbt.output_file_sandbox:
771 >            out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ','
772 >            pass
773 >        pass
774 >
775 >        # via sandbox iif required return_data
776          if int(self.return_data) == 1:
777              for fl in jbt.output_file:
778                  out_box = out_box + '' + jbt.numberFile_(fl, '_ITR1_') + ','
# Line 808 | Line 792 | class SchedulerEdg(Scheduler):
792          xml.write('</task>\n')
793  
794          xml.close()
795 +      
796 +
797          return
798  
799      def checkProxy(self):
# Line 815 | Line 801 | class SchedulerEdg(Scheduler):
801          Function to check the Globus proxy.
802          """
803          if (self.proxyValid): return
804 <        timeleft = -999
804 >
805 >        ### Just return if asked to do so
806 >        if (self.dontCheckProxy==1):
807 >            self.proxyValid=1
808 >            return
809 >
810          minTimeLeft=10*3600 # in seconds
811  
812          minTimeLeftServer = 100 # in hours
# Line 835 | Line 826 | class SchedulerEdg(Scheduler):
826          pass
827  
828          if mustRenew:
829 <            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 96h\n")
830 <            cmd = 'voms-proxy-init -voms '+self.VO+' -valid 96:00'
829 >            common.logger.message( "No valid proxy found or remaining time of validity of already existing proxy shorter than 10 hours!\n Creating a user proxy with default length of 192h\n")
830 >            cmd = 'voms-proxy-init -voms '+self.VO
831 >            if self.group:
832 >                cmd += ':/'+self.VO+'/'+self.group
833              if self.role:
834 <                cmd = 'voms-proxy-init -voms '+self.VO+':/'+self.VO+'/role='+self.role+' -valid 96:00'
834 >                cmd += '/role='+self.role
835 >            cmd += ' -valid 192:00'
836              try:
837                  # SL as above: damn it!
838 +                common.logger.debug(10,cmd)
839                  out = os.system(cmd)
840                  if (out>0): raise CrabException("Unable to create a valid proxy!\n")
841              except:
# Line 860 | Line 855 | class SchedulerEdg(Scheduler):
855              reTime = re.compile( r'timeleft: (\d+)' )
856              #print "<"+str(reTime.search( cmd_out ).group(1))+">"
857              if reTime.match( cmd_out ):
858 <                time = reTime.search( line ).group(1)
858 >                time = reTime.search( cmd_out ).group(1)
859                  if time < minTimeLeftServer:
860                      renewProxy = 1
861                      common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
# Line 886 | Line 881 | class SchedulerEdg(Scheduler):
881          if self.edg_config_vo:
882              edg_ui_cfg_opt += ' --config-vo ' + self.edg_config_vo + ' '
883          return edg_ui_cfg_opt
884 +
885 +    def submitTout(self, list):
886 +        return 120
887 +
888 +

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines