ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DataDiscovery.py

Comparing COMP/CRAB/python/DataDiscovery.py (file contents):
Revision 1.24 by spiga, Mon Jun 16 10:44:33 2008 UTC vs.
Revision 1.27 by spiga, Wed Feb 4 15:24:41 2009 UTC

#	Line 89 | Line 89 | class DataDiscovery:
89		"""
90		Contact DBS
91		"""
92	–
92		## get DBS URL
93	<	dbs_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94	<	if (self.cfg_params.has_key('CMSSW.dbs_url')):
95	<	dbs_url=self.cfg_params['CMSSW.dbs_url']
93	>	global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
94	>	caf_url = "http://cmsdbsprod.cern.ch/cms_dbs_caf_analysis_01/servlet/DBSServlet"
95	>	dbs_url_map  =   {'glite':    global_url,
96	>	'glitecoll':global_url,\
97	>	'condor':   global_url,\
98	>	'condor_g': global_url,\
99	>	'glidein':  global_url,\
100	>	'lsf':      global_url,\
101	>	'caf':      caf_url,\
102	>	'sge':      global_url
103	>	}
104
105	+	dbs_url_default = dbs_url_map[(common.scheduler.name()).lower()]
106	+	dbs_url=  self.cfg_params.get('CMSSW.dbs_url', dbs_url_default)
107		common.logger.debug(3,"Accessing DBS at: "+dbs_url)
108
109		## check if runs are selected
#	Line 102 | Line 111 | class DataDiscovery:
111		if (self.cfg_params.has_key('CMSSW.runselection')):
112		runselection = parseRange2(self.cfg_params['CMSSW.runselection'])
113
114	+
115	+	self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
116	+
117	+	self.ads = int(self.cfg_params.get('CMSSW.ads', 0))
118	+
119		common.logger.debug(6,"runselection is: %s"%runselection)
120		## service API
121		args = {}
#	Line 109 | Line 123 | class DataDiscovery:
123		args['level']   = 'CRITICAL'
124
125		## check if has been requested to use the parent info
126	<	useParent = self.cfg_params.get('CMSSW.use_parent',False)
126	>	useparent = int(self.cfg_params.get('CMSSW.use_parent',0))
127
128		## check if has been asked for a non default file to store/read analyzed fileBlocks
129		defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'
130		fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))
131
132		api = DBSAPI.dbsApi.DbsApi(args)
119	–	allowedRetriveValue = ['retrive_parent',
120	–	'retrive_block',
121	–	'retrive_lumi',
122	–	'retrive_run'
123	–	]
124	–	try:
125	–	if len(runselection) <= 0 :
126	–	if useParent:
127	–	files = api.listFiles(path=self.datasetPath, retriveList=allowedRetriveValue)
128	–	common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
129	–	common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
130	–	else:
131	–	files = api.listDatasetFiles(self.datasetPath)
132	–	else :
133	–	files=[]
134	–	for arun in runselection:
135	–	try:
136	–	filesinrun = api.listFiles(path=self.datasetPath,retriveList=allowedRetriveValue,runNumber=arun)
137	–	files.extend(filesinrun)
138	–	except:
139	–	msg="WARNING: problem extracting info from DBS for run %s "%arun
140	–	common.logger.message(msg)
141	–	pass
133
134	<	except DbsBadRequest, msg:
144	<	raise DataDiscoveryError(msg)
145	<	except DBSError, msg:
146	<	raise DataDiscoveryError(msg)
134	>	self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)
135
136		anFileBlocks = []
137		if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)
138
139		# parse files and fill arrays
140	<	for file in files :
140	>	for file in self.files :
141		parList = []
142		# skip already analyzed blocks
143		fileblock = file['Block']['Name']
144		if fileblock not in anFileBlocks :
145		filename = file['LogicalFileName']
146		# asked retry the list of parent for the given child
147	<	if useParent: parList = [x['LogicalFileName'] for x in file['ParentList']]
147	>	if useparent==1: parList = [x['LogicalFileName'] for x in file['ParentList']]
148		self.parent[filename] = parList
149		if filename.find('.dat') < 0 :
150		events    = file['NumberOfEvents']
#	Line 192 | Line 180 | class DataDiscovery:
180		% self.datasetPath)
181
182
183	+	###########################
184	+
185	+	def queryDbs(self,api,path=None,runselection=None,useParent=None):
186	+
187	+	allowedRetriveValue = [#'retrive_parent',
188	+	'retrive_block',
189	+	#'retrive_lumi',
190	+	'retrive_run'
191	+	]
192	+	try:
193	+	if len(runselection) <=0 :
194	+	if useParent==1 or self.splitByRun==1 :
195	+	if self.ads==1 :
196	+	files = api.listFiles(analysisDataset=path, retriveList=allowedRetriveValue)
197	+	else :
198	+	files = api.listFiles(path=path, retriveList=allowedRetriveValue)
199	+	common.logger.debug(5,"Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
200	+	common.logger.write("Set of input parameters used for DBS query : \n"+str(allowedRetriveValue))
201	+	else:
202	+	print 'MALE2'
203	+	files = api.listDatasetFiles(self.datasetPath)
204	+	else :
205	+	files=[]
206	+	for arun in runselection:
207	+	try:
208	+	if self.ads==1 : filesinrun = api.listFiles(analysisDataset=path,retriveList=allowedRetriveValue,runNumber=arun)
209	+	else: filesinrun = api.listFiles(path=path,retriveList=allowedRetriveValue,runNumber=arun)
210	+	files.extend(filesinrun)
211	+	except:
212	+	msg="WARNING: problem extracting info from DBS for run %s "%arun
213	+	common.logger.message(msg)
214	+	pass
215	+
216	+	except DbsBadRequest, msg:
217	+	raise DataDiscoveryError(msg)
218	+	except DBSError, msg:
219	+	raise DataDiscoveryError(msg)
220	+
221	+	return files
222	+
223		# #################################################
224		def getMaxEvents(self):
225		"""
#	Line 227 | Line 255 | class DataDiscovery:
255		"""
256		return self.parent
257
258	+	# #################################################
259	+	def getListFiles(self):
260	+	"""
261	+	return parent grouped by file
262	+	"""
263	+	return self.files
264	+
265		########################################################################

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines