ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DBSInvalidateDataset.py

Comparing COMP/CRAB/python/DBSInvalidateDataset.py (file contents):
Revision 1.1.2.1 by spiga, Mon Oct 12 20:11:58 2009 UTC vs.
Revision 1.1.2.2 by spiga, Tue Oct 13 13:58:38 2009 UTC

#	Line 0 | Line 1
1	+	#!/usr/bin/env python
2	+	"""
3	+	_DBSInvalidateDataset_
4	+
5	+	Command line tool to invalidate a dataset.
6	+
7	+	"""
8	+	from DBSAPI.dbsApi import DbsApi
9	+	from DBSAPI.dbsOptions import DbsOptionParser
10	+
11	+	import string,sys,os
12	+
13	+	#
14	+	# Check if the parents of a dataset are all valid (or really not invalid)
15	+	#
16	+	def parentsValid(dbsapi,ads):
17	+	parents=dbsapi.listDatasetParents(ads)
18	+	parentsOK=True
19	+
20	+	for p in parents:
21	+	path=p['PathList'][0]
22	+	dsStatus=datasetStatus(dbsapi,path)
23	+	# do we want to check other statuses?
24	+	if dsStatus == "INVALID" or dsStatus == "INVALID-RO":
25	+	parentsOK=False
26	+	print "Parent dataset %s is %s, must be a valid status to validate child" % (path, dsStatus)
27	+
28	+	return parentsOK
29	+
30	+	#
31	+	# Check that the children of a dataset are all invalid
32	+	#
33	+	def childrenInvalid(dbsapi,ads):
34	+	childrenOK=True
35	+	dsBlocks=dbsapi.listBlocks(dataset=ads)
36	+	childPaths = {}
37	+
38	+	#
39	+	# To find the children we have to go through the block parentage, so accumulate the list of parent
40	+	# datasets and check each one only once
41	+	#
42	+	for b in dsBlocks:
43	+	blockChildren=dbsapi.listBlockChildren(b['Name'])
44	+	for c in blockChildren:
45	+	childPaths[c['Path']] = 1
46	+
47	+	for c in childPaths.keys():
48	+	dsStatus=datasetStatus(dbsapi,c)
49	+	if dsStatus != "INVALID" and dsStatus != "INVALID-RO":
50	+	childrenOK=False
51	+	print "Child dataset %s is %s, must be invalid to invalidate parent" % (c, dsStatus)
52	+
53	+	return childrenOK
54	+
55	+	#
56	+	# Get the current status of a dataset
57	+	#
58	+	def datasetStatus(dbsapi,ads):
59	+
60	+	params = ads.split('/')
61	+	proc = dbsapi.listProcessedDatasets(patternPrim=params[1],  patternProc=params[2], patternDT=params[3])
62	+
63	+	if proc and proc[0]['Status']:
64	+	dsStatus = proc[0]['Status']
65	+	else:
66	+	# listProcessedDatasets will not list INVALID datasets, so check the list of all datasets to see
67	+	# if it is there
68	+	paths = dbsapi.listDatasetPaths()
69	+	pathset = set(paths)
70	+	if ads in pathset:
71	+	dsStatus = 'INVALID'
72	+	else:
73	+	dsStatus = 'NOT FOUND'
74	+
75	+	return dsStatus
76	+
77	+	#
78	+	# Invalidate/validate dataset
79	+	#
80	+	def setDatasetStatus(dbsapi,ads,valid):
81	+	oldStatus = datasetStatus(dbsapi, ads)
82	+	didit=False
83	+
84	+	if valid:
85	+	if oldStatus == 'INVALID':
86	+	if parentsValid(dbsapi,ads):
87	+	print "Validating Dataset %s"%ads
88	+	dbsapi.updateProcDSStatus(ads,"VALID")
89	+	didit=True
90	+	else:
91	+	print "Dataset %s status is %s, must be INVALID to validate" % (ads, oldStatus)
92	+
93	+	else:
94	+	if oldStatus == 'VALID':
95	+	if childrenInvalid(dbsapi,ads):
96	+	print "Invalidating Dataset %s"%ads
97	+	dbsapi.updateProcDSStatus(ads,"INVALID")
98	+	didit=True
99	+	else:
100	+	print "Dataset %s status is %s, must be VALID to invalidate" % (ads, oldStatus)
101	+
102	+	return didit
103	+
104	+	#
105	+	# Invalidate/validate all the files in a dataset
106	+	#
107	+	def setDatasetFilesStatus(dbsapi,ads,valid):
108	+
109	+	retrieveList=['retrive_status']
110	+
111	+	if valid:
112	+	retrieveList.append('retrive_invalid_files')
113	+	newStatus='VALID'
114	+	oldStatus='INVALID'
115	+	else:
116	+	newStatus='INVALID'
117	+	oldStatus='VALID'
118	+
119	+	files=dbsapi.listFiles(path=ads,retriveList=retrieveList)
120	+
121	+	if files:
122	+	print "Setting files to status " + newStatus
123	+
124	+	for f in files:
125	+	if f['Status'] == oldStatus:
126	+	dbsapi.updateFileStatus(f['LogicalFileName'], newStatus)
127	+
128	+	return
129	+
130	+	#
131	+	# Process a list of datasets
132	+	#
133	+	def updateDatasetStatus(dbsapi,lds,valid,files):
134	+	if (lds != None):
135	+	datasetList=lds.split(',')
136	+	for ads in datasetList:
137	+	if setDatasetStatus(dbsapi,ads,valid) and files:
138	+	setDatasetFilesStatus(dbsapi,ads,valid)
139	+
140	+	return
141	+
142	+	def main ():
143	+	from optparse import OptionParser
144	+
145	+	usage="""\npython DBSInvalidateDataset <options> \nOptions: \n --DBSURL=<URL> \t\t DBS URL \n --datasetPath=<dataset> \t dataset \n --valid \t\t\t re-validate an invalid dataset\n --files \t\t\t change status of all the files in the dataset"""
146	+	parser = OptionParser(usage=usage)
147	+
148	+	parser.add_option('-D', '--DBSURL', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL')
149	+	parser.add_option('-d', '--datasetPath', dest='dataset', default=None, help='Dataset')
150	+	parser.add_option('-v', '--valid', action="store_true", default=False,dest='valid', help='Validate status instead of invalidate')
151	+	parser.add_option('-f', '--files', action="store_true", default=False,dest='files', help='Validate or invalidate all files in dataset')
152	+
153	+	(opts, args) = parser.parse_args()
154	+
155	+	if opts.url == None:
156	+	print "--url option not provided."
157	+	print "Using %s"%opts.url
158	+
159	+	if opts.dataset == None:
160	+	print "--dataset option must be provided"
161	+	print usage;
162	+	sys.exit(1)
163	+
164	+	dbsargs = {'url' : opts.url}
165	+	dbsapi = DbsApi(dbsargs)
166	+
167	+	try:
168	+	if opts.dataset != None:
169	+	updateDatasetStatus(dbsapi, opts.dataset, opts.valid, opts.files)
170	+
171	+	except Exception, ex:
172	+	print "Caught exception %s:"%str(ex)
173	+	sys.exit(1)
174	+
175	+	sys.exit(0)
176	+
177	+	if __name__ == "__main__":
178	+	main()
179	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines