ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/PhEDExDatasvcInfo.py
(Generate patch)

Comparing COMP/CRAB/python/PhEDExDatasvcInfo.py (file contents):
Revision 1.9 by spiga, Fri Oct 10 14:38:00 2008 UTC vs.
Revision 1.29 by spiga, Sat Sep 19 11:11:01 2009 UTC

# Line 2 | Line 2 | from Actor import *
2   import urllib
3   from xml.dom.minidom import parse
4   from crab_exceptions import *
5 – from crab_logger import Logger
5   from WorkSpace import *
6   from urlparse import urlparse
7   from LFNBaseName import *
8  
9   class PhEDExDatasvcInfo:
10 <    def __init__( self , cfg_params ):
10 >    def __init__( self , cfg_params=None, config=None ):
11  
12          ## PhEDEx Data Service URL
13 <        url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
15 <        self.datasvc_url = cfg_params.get("USER.datasvc_url",url)
13 >        self.datasvc_url="https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
14  
15          self.FacOps_savannah = 'https://savannah.cern.ch/support/?func=additem&group=cmscompinfrasup'
16 +        stage_out_faq='https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabHowTo#Stageout_and_publication'
17 +        self.dataPub_faq = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabForPublication'
18  
19 +        self.usePhedex = True
20 +        self.sched = common.scheduler.name().upper()
21 +
22 +        if config!=None:
23 +            self.checkConfig(config)  
24 +        else:
25 +            self.checkCfgConfig(cfg_params)  
26 +
27 +        self.protocol = self.srm_version
28 +
29  
30 +    def checkConfig(self,config):
31 +        """
32 +        """
33 +        self.srm_version = config.get("srm_version",'srmv2')
34 +        self.node = config.get('storage_element',None)
35 +        self.lfn='/store/'
36 +
37 +    def checkCfgConfig(self,cfg_params):
38 +        """
39 +        """
40 +        self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
41          self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
42          self.node = cfg_params.get('USER.storage_element',None)
43 <        
43 >
44          self.publish_data = cfg_params.get("USER.publish_data",0)
45          self.usenamespace = cfg_params.get("USER.usenamespace",0)
46 <        self.user_remote_dir = cfg_params.get("USER.remote_dir",'')
46 >        self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
47          if self.user_remote_dir:
48              if ( self.user_remote_dir[-1] != '/' ) : self.user_remote_dir = self.user_remote_dir + '/'
49 <            
49 >          
50          self.datasetpath = cfg_params.get("CMSSW.datasetpath")
51          self.publish_data_name = cfg_params.get('USER.publish_data_name','')
52  
32 –        self.user_lfn = cfg_params.get("USER.lfn",'')
33 –        if self.user_lfn:
34 –            if ( self.user_lfn[-1] != '/' ) : self.user_lfn = self.user_lfn + '/'
35 –            
53          self.user_port = cfg_params.get("USER.storage_port",'8443')
54          self.user_se_path = cfg_params.get("USER.storage_path",'')
55          if self.user_se_path:
56              if ( self.user_se_path[-1] != '/' ) : self.user_se_path = self.user_se_path + '/'
57                                                      
41 –      
58          #check if using "private" Storage
59 <        self.usePhedex = True
59 >        if not self.node :
60 >            msg = 'Please specify the storage_element name in your crab.cfg section [USER].\n'
61 >            msg +='\tFor further information please visit : %s'%stage_out_faq
62 >            raise CrabException(msg)
63          if (self.node.find('T1_') + self.node.find('T2_')+self.node.find('T3_')) == -3: self.usePhedex = False
64 <        if not self.usePhedex and ( self.user_lfn == '' or self.user_se_path == '' ):
64 >
65 >        if not self.usePhedex and ( self.user_remote_dir == '' or self.user_se_path == '' ):
66              msg = 'You are asking to stage out without using CMS Storage Name convention. In this case you \n'
67 <            msg += '      must specify both lfn and storage_path in the crab.cfg section [USER].\n '
68 <            msg += '      For further information please visit: ADD_TWIKI_LINK'
67 >            msg += '\t must specify both user_remote_dir and storage_path in the crab.cfg section [USER].\n '
68 >            msg += '\t For further information please visit : \n\t%s'%stage_out_faq
69              raise CrabException(msg)
50 –        self.sched = common.scheduler.name().upper()
51 –        self.protocol = self.srm_version
52 –        if self.sched in ['CAF','LSF']:self.protocol = 'direct'
70  
71 +        self.forced_path = '/store/user/'
72 +        if self.sched in ['CAF','LSF']:
73 +            self.protocol = 'direct'
74 +            self.SE = {'CAF':'caf.cern.ch', 'LSF':''}
75 +            if self.sched == 'CAF': self.forced_path = '/store/caf/user/'
76 +            
77 +        if not self.usePhedex:
78 +            self.forced_path = self.user_remote_dir
79          return
80  
81      def getEndpoint(self):  
# Line 61 | Line 86 | class PhEDExDatasvcInfo:
86  
87          #extract the PFN for the given node,LFN,protocol
88          endpoint = self.getStageoutPFN()
89 +        ### FEDE added a check for the final /
90 +        if ( endpoint[-1] != '/' ) : endpoint = endpoint + '/'
91 +        ######################################
92    
93          #extract SE name an SE_PATH (needed for publication)
94          SE, SE_PATH, User = self.splitEndpoint(endpoint)
# Line 78 | Line 106 | class PhEDExDatasvcInfo:
106              if self.protocol == 'direct':
107                  query=endpoint
108                  SE_PATH = endpoint
109 <                ### FEDE added SE ###
82 <                SE = self.sched
109 >                SE = self.SE[self.sched]
110              else:
111                  url = 'http://'+endpoint.split('://')[1]
112                  # python > 2.4
# Line 89 | Line 116 | class PhEDExDatasvcInfo:
116                  SE_PATH = endpoint.split(host)[1]
117              USER = (query.split('user')[1]).split('/')[1]
118          else:
119 +            #### to test #####
120 +           # url = 'http://'+endpoint.split('://')[1]
121 +           # scheme, host, path, params, query, fragment = urlparse(url)
122 +           # SE = host.split(':')[0]
123 +           # SE_PATH = endpoint.split(host)[1]
124              SE = self.node
125 <            SE_PATH = self.user_se_path + self.user_lfn
125 >            SE_PATH = self.user_se_path + self.user_remote_dir
126              try:
127                  USER = (self.lfn.split('user')[1]).split('/')[1]
128              except:
# Line 110 | Line 142 | class PhEDExDatasvcInfo:
142              ## check if storage_name is a T2 (siteDB query)
143              ## if yes :match self.user_lfn with LFNBaseName...
144              ##     if NOT : raise (you are using a T2. It's not allowed stage out into self.user_path+self.user_lfn)  
145 <            lfn = self.user_lfn
145 >            lfn = self.user_remote_dir
146              return lfn
147          if self.publish_data_name == '' and int(self.publish_data) == 1:
148 <            msg = "Eeror. The [USER] section does not have 'publish_data_name'"
148 >            msg = "Error. The [USER] section does not have 'publish_data_name'\n"
149 >            msg += '\tFor further information please visit : \n\t%s'%self.dataPub_faq
150              raise CrabException(msg)
151          if self.publish_data_name == '' and int(self.usenamespace) == 1:
152             self.publish_data_name = "DefaultDataset"
153 <        if int(self.publish_data) == 1 or int(self.usenamespace) == 1:
153 >        if int(self.publish_data) == 1:
154 >            if self.sched in ['CAF']: l_User=True
155 >            primaryDataset = self.computePrimaryDataset()
156 >            ### added the case lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)
157 >            ### for the publication in order to be able to check the lfn length  
158 >            lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name, publish=True)  + '/${PSETHASH}/'    
159 >        elif int(self.usenamespace) == 1:
160              if self.sched in ['CAF']: l_User=True
161              primaryDataset = self.computePrimaryDataset()
162 <            lfn = LFNBase(primaryDataset,self.publish_data_name,LocalUser=l_User)  + '/${PSETHASH}/'    
162 >            lfn = LFNBase(self.forced_path, primaryDataset, self.publish_data_name)  + '/${PSETHASH}/'    
163          else:
164              if self.sched in ['CAF','LSF']: l_User=True
165 <            lfn = LFNBase(self.user_remote_dir,LocalUser=l_User)
165 >            lfn = LFNBase(self.forced_path,self.user_remote_dir)
166          return lfn
167  
168      def computePrimaryDataset(self):
# Line 146 | Line 185 | class PhEDExDatasvcInfo:
185          params = {'node' : self.node , 'lfn': self.lfn , 'protocol': self.protocol}
186          params = urllib.urlencode(params)
187          datasvc_lfn2pfn="%s/lfn2pfn"%self.datasvc_url
149 –        urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
188          try:
189 +            urlresults = urllib.urlopen(datasvc_lfn2pfn, params)
190              urlresults = parse(urlresults)
191 +        except IOError:
192 +            msg="Unable to access PhEDEx Data Service at %s"%datasvc_lfn2pfn
193 +            raise CrabException(msg)
194          except:
195              urlresults = None
196  
# Line 211 | Line 253 | class PhEDExDatasvcInfo:
253                  msg+='       OriginalSubmission: stageout path is not retrieved from %s \n'%fullurl
254                  raise CrabException(msg)
255          else:
256 <            stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
256 >            if self.sched in ['CAF','LSF'] :
257 >                stageoutpfn = self.user_se_path+self.lfn
258 >            else:
259 >                stageoutpfn = 'srm://'+self.node+':'+self.user_port+self.user_se_path+self.lfn
260  
261          return stageoutpfn
262  

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines