ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DataDiscovery.py

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.24.2.1 by spiga, Thu Jul 3 23:05:05 2008 UTC vs.
Revision 1.29 by edelmann, Tue Mar 17 14:36:44 2009 UTC

#	Line 99 | Line 99 | class DataDiscovery:
99		'glidein':  global_url,\
100		'lsf':      global_url,\
101		'caf':      caf_url,\
102	<	'sge':      global_url
102	>	'sge':      global_url,
103	>	'arc':      global_url
104		}
105
106		dbs_url_default = dbs_url_map[(common.scheduler.name()).lower()]
#	Line 111 | Line 112 | class DataDiscovery:
112		if (self.cfg_params.has_key('CMSSW.runselection')):
113		runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
114
115	+
116	+	self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
117	+
118	+	self.ads = int(self.cfg_params.get('CMSSW.ads', 0))
119	+
120		common.logger.debug(6,"runselection is: %s"%runselection)
121		## service API
122		args = {}
#	Line 118 | Line 124 | class DataDiscovery:
124		args['level']   = 'CRITICAL'
125
126		## check if has been requested to use the parent info
127	<	useParent = self.cfg_params.get('CMSSW.use_parent',False)
127	>	useparent = int(self.cfg_params.get('CMSSW.use_parent',0))
128
129		## check if has been asked for a non default file to store/read analyzed fileBlocks
130		defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'
131		fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))
132
133		api = DBSAPI.dbsApi.DbsApi(args)
128	–	allowedRetriveValue = ['retrive_parent',
129	–	'retrive_block',
130	–	'retrive_lumi',
131	–	'retrive_run'
132	–	]
133	–	try:
134	–	if len(runselection) <= 0 :
135	–	if useParent:
136	–	files = api.listFiles(path=self.datasetPath, retriveList=allowedRetriveValue)
137	–	common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
138	–	common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
139	–	else:
140	–	files = api.listDatasetFiles(self.datasetPath)
141	–	else :
142	–	files=[]
143	–	for arun in runselection:
144	–	try:
145	–	filesinrun = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue,runNumber=arun)
146	–	files.extend(filesinrun)
147	–	except:
148	–	msg="WARNING: problem extracting info from DBS for run %s "%arun
149	–	common.logger.message(msg)
150	–	pass
134
135	<	except DbsBadRequest, msg:
153	<	raise DataDiscoveryError(msg)
154	<	except DBSError, msg:
155	<	raise DataDiscoveryError(msg)
135	>	self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)
136
137		anFileBlocks = []
138		if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)
139
140		# parse files and fill arrays
141	<	for file in files :
141	>	for file in self.files :
142		parList = []
143		# skip already analyzed blocks
144		fileblock = file['Block']['Name']
145		if fileblock not in anFileBlocks :
146		filename = file['LogicalFileName']
147		# asked retry the list of parent for the given child
148	<	if useParent: parList = [x['LogicalFileName'] for x in file['ParentList']]
148	>	if useparent==1: parList = [x['LogicalFileName'] for x in file['ParentList']]
149		self.parent[filename] = parList
150		if filename.find('.dat') < 0 :
151		events    = file['NumberOfEvents']
#	Line 201 | Line 181 | class DataDiscovery:
181		% self.datasetPath)
182
183
184	+	###########################
185	+
186	+	def queryDbs(self,api,path=None,runselection=None,useParent=None):
187	+
188	+	allowedRetriveValue = [#'retrive_parent',
189	+	'retrive_block',
190	+	#'retrive_lumi',
191	+	'retrive_run'
192	+	]
193	+	try:
194	+	if len(runselection) <=0 :
195	+	if useParent==1 or self.splitByRun==1 :
196	+	if self.ads==1 :
197	+	files = api.listFiles(analysisDataset=path, retriveList=allowedRetriveValue)
198	+	else :
199	+	files = api.listFiles(path=path, retriveList=allowedRetriveValue)
200	+	common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
201	+	common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
202	+	else:
203	+	files = api.listDatasetFiles(self.datasetPath)
204	+	else :
205	+	files=[]
206	+	for arun in runselection:
207	+	try:
208	+	if self.ads==1 : filesinrun = api.listFiles(analysisDataset=path,retriveList=allowedRetriveValue,runNumber=arun)
209	+	else: filesinrun = api.listFiles(path=path,retriveList=allowedRetriveValue,runNumber=arun)
210	+	files.extend(filesinrun)
211	+	except:
212	+	msg="WARNING: problem extracting info from DBS for run %s "%arun
213	+	common.logger.message(msg)
214	+	pass
215	+
216	+	except DbsBadRequest, msg:
217	+	raise DataDiscoveryError(msg)
218	+	except DBSError, msg:
219	+	raise DataDiscoveryError(msg)
220	+
221	+	return files
222	+
223		# #################################################
224		def getMaxEvents(self):
225		"""
#	Line 236 | Line 255 | class DataDiscovery:
255		"""
256		return self.parent
257
258	+	# #################################################
259	+	def getListFiles(self):
260	+	"""
261	+	return parent grouped by file
262	+	"""
263	+	return self.files
264	+
265		########################################################################

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines