ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/DataDiscovery.py
(Generate patch)

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.6 by gutsche, Sun May 28 02:20:13 2006 UTC vs.
Revision 1.7.2.1 by spiga, Thu Jul 20 12:03:48 2006 UTC

# Line 5 | Line 5 | from DBSInfo import *
5  
6   # ####################################
7   class DataDiscoveryError(exceptions.Exception):
8 <  def __init__(self, errorMessage):
9 <   args=errorMessage
10 <   exceptions.Exception.__init__(self, args)
11 <   pass
12 <
13 <  def getErrorMessage(self):
14 <   """ Return exception error """
15 <   return "%s" % (self.args)
8 >    def __init__(self, errorMessage):
9 >        args=errorMessage
10 >        exceptions.Exception.__init__(self, args)
11 >        pass
12 >
13 >    def getErrorMessage(self):
14 >        """ Return exception error """
15 >        return "%s" % (self.args)
16  
17   # ####################################
18   class NotExistingDatasetError(exceptions.Exception):
19 <  def __init__(self, errorMessage):
20 <   args=errorMessage
21 <   exceptions.Exception.__init__(self, args)
22 <   pass
23 <
24 <  def getErrorMessage(self):
25 <   """ Return exception error """
26 <   return "%s" % (self.args)
19 >    def __init__(self, errorMessage):
20 >        args=errorMessage
21 >        exceptions.Exception.__init__(self, args)
22 >        pass
23 >
24 >    def getErrorMessage(self):
25 >        """ Return exception error """
26 >        return "%s" % (self.args)
27  
28   # ####################################
29   class NoDataTierinProvenanceError(exceptions.Exception):
30 <  def __init__(self, errorMessage):
31 <   args=errorMessage
32 <   exceptions.Exception.__init__(self, args)
33 <   pass
34 <
35 <  def getErrorMessage(self):
36 <   """ Return exception error """
37 <   return "%s" % (self.args)
30 >    def __init__(self, errorMessage):
31 >        args=errorMessage
32 >        exceptions.Exception.__init__(self, args)
33 >        pass
34 >
35 >    def getErrorMessage(self):
36 >        """ Return exception error """
37 >        return "%s" % (self.args)
38  
39   # ####################################
40   # class to find and extact info from published data
# Line 60 | Line 60 | class DataDiscovery:
60  
61          ## add the PU among the required data tiers if the Digi are requested
62          if (self.dataTiers.count('Digi')>0) & (self.dataTiers.count('PU')<=0) :
63 <          self.dataTiers.append('PU')
63 >            self.dataTiers.append('PU')
64  
65          ## get info about the requested dataset
66          dbs=DBSInfo()
67          try:
68 <         self.datasets = dbs.getMatchingDatasets(self.owner, self.dataset)
68 >            self.datasets = dbs.getMatchingDatasets(self.owner, self.dataset)
69          except DBSError, ex:
70 <          raise DataDiscoveryError(ex.getErrorMessage())
70 >            raise DataDiscoveryError(ex.getErrorMessage())
71          if len(self.datasets) == 0:
72 <          raise DataDiscoveryError("Owner=%s, Dataset=%s unknown to DBS" % (self.owner, self.dataset))
72 >            raise DataDiscoveryError("Owner=%s, Dataset=%s unknown to DBS" % (self.owner, self.dataset))
73          if len(self.datasets) > 1:
74 <          raise DataDiscoveryError("Owner=%s, Dataset=%s is ambiguous" % (self.owner, self.dataset))
74 >            raise DataDiscoveryError("Owner=%s, Dataset=%s is ambiguous" % (self.owner, self.dataset))
75          try:
76 <          self.dbsdataset = self.datasets[0].get('datasetPathName')
77 <          self.blocksinfo = dbs.getDatasetContents(self.dbsdataset)
78 <          self.allblocks.append (self.blocksinfo.keys ()) # add also the current fileblocksinfo
79 <          self.dbspaths.append(self.dbsdataset)
76 >            self.dbsdataset = self.datasets[0].get('datasetPathName')
77 >            self.blocksinfo = dbs.getDatasetContents(self.dbsdataset)
78 >            self.allblocks.append (self.blocksinfo.keys ()) # add also the current fileblocksinfo
79 >            self.dbspaths.append(self.dbsdataset)
80          except DBSError, ex:
81 <          raise DataDiscoveryError(ex.getErrorMessage())
81 >            raise DataDiscoveryError(ex.getErrorMessage())
82          
83          if len(self.blocksinfo)<=0:
84 <         msg="\nERROR Data for %s do not exist in DBS! \n Check the dataset/owner variables in crab.cfg !"%self.dbsdataset
85 <         raise NotExistingDatasetError(msg)
84 >            msg="\nERROR Data for %s do not exist in DBS! \n Check the dataset/owner variables in crab.cfg !"%self.dbsdataset
85 >            raise NotExistingDatasetError(msg)
86  
87  
88          ## get info about the parents
89          try:
90 <          parents=dbs.getDatasetProvenance(self.dbsdataset, self.dataTiers)
90 >            parents=dbs.getDatasetProvenance(self.dbsdataset, self.dataTiers)
91          except DBSInvalidDataTierError, ex:
92 <          msg=ex.getErrorMessage()+' \n Check the data_tier variable in crab.cfg !\n'
93 <          raise DataDiscoveryError(msg)
92 >            msg=ex.getErrorMessage()+' \n Check the data_tier variable in crab.cfg !\n'
93 >            raise DataDiscoveryError(msg)
94          except DBSError, ex:
95 <          raise DataDiscoveryError(ex.getErrorMessage())
95 >            raise DataDiscoveryError(ex.getErrorMessage())
96  
97          ## check that the user asks for parent Data Tier really existing in the DBS provenance
98          self.checkParentDataTier(parents, self.dataTiers)
99  
100          ## for each parent get the corresponding fileblocks
101          try:
102 <          for p in parents:
103 <            ## fill a list of dbspaths
104 <            parentPath = p.get('parent').get('datasetPathName')
105 <            self.dbspaths.append (parentPath)
106 <            parentBlocks = dbs.getDatasetContents (parentPath)
107 <            self.allblocks.append (parentBlocks.keys ())  # add parent fileblocksinfo
102 >            for p in parents:
103 >                ## fill a list of dbspaths
104 >                parentPath = p.get('parent').get('datasetPathName')
105 >                self.dbspaths.append (parentPath)
106 >                parentBlocks = dbs.getDatasetContents (parentPath)
107 >                self.allblocks.append (parentBlocks.keys ())  # add parent fileblocksinfo
108          except DBSError, ex:
109              raise DataDiscoveryError(ex.getErrorMessage())
110  
111   # #################################################
112      def checkParentDataTier(self, parents, dataTiers):
113          """
114 <         check that the data tiers requested by the user really exists in the provenance of the given dataset
114 >        check that the data tiers requested by the user really exists in the provenance of the given dataset
115          """
116          startType = string.split(self.dbsdataset,'/')[2]
117          # for example 'type' is PU and 'dataTier' is Hit
118          parentTypes = map(lambda p: p.get('type'), parents)
119          for tier in dataTiers:
120 <          if parentTypes.count(tier) <= 0 and tier != startType:
121 <            msg="\nERROR Data %s not published in DBS with asked data tiers : the data tier not found is %s !\n  Check the data_tier variable in crab.cfg !"%(self.dbsdataset,tier)
122 <            raise  NoDataTierinProvenanceError(msg)
120 >            if parentTypes.count(tier) <= 0 and tier != startType:
121 >                msg="\nERROR Data %s not published in DBS with asked data tiers : the data tier not found is %s !\n  Check the data_tier variable in crab.cfg !"%(self.dbsdataset,tier)
122 >                raise  NoDataTierinProvenanceError(msg)
123  
124  
125   # #################################################
126      def getMaxEvents(self):
127          """
128 <         max events of the primary dataset-owner
128 >        max events of the primary dataset-owner
129          """
130          ## loop over the fileblocks of the primary dataset-owner
131          nevts=0      
132          for blockevts in self.blocksinfo.values():
133 <          nevts=nevts+blockevts
133 >            nevts=nevts+blockevts
134  
135          return nevts
136  
137   # #################################################
138      def getDBSPaths(self):
139          """
140 <         list the DBSpaths for all required data
140 >        list the DBSpaths for all required data
141          """
142          return self.dbspaths
143  
144   # #################################################
145      def getEVC(self):
146          """
147 <         list the event collections structure by fileblock
147 >        list the event collections structure by fileblock
148          """
149          print "To be used by a more complex job splitting... TODO later... "
150          print "it requires changes in what's returned by DBSInfo.getDatasetContents and then fetchDBSInfo"
# Line 152 | Line 152 | class DataDiscovery:
152   # #################################################
153      def getFileBlocks(self):
154          """
155 <         fileblocks for all required dataset-owners
155 >        fileblocks for all required dataset-owners
156          """
157          return self.allblocks        
158  
159   ########################################################################
160 –
161 –

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines