| Revision: | 1.1 |
| Committed: | Mon Oct 12 20:11:58 2009 UTC (15 years, 6 months ago) by dsr |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_7_1_pre2, CRAB_2_7_1_pre1, CRAB_2_7_0, CRAB_2_7_0_pre8, CRAB_2_7_0_pre7, CRAB_2_7_0_pre6, CRAB_2_7_0_pre5, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Log Message: | Initial version of script for CRAB users to invalidate a dataset in DBS |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | dsr | 1.1 | #!/usr/bin/env python |
| 2 | """ | ||
| 3 | _DBSInvalidateDataset_ | ||
| 4 | |||
| 5 | Command line tool to invalidate a dataset. | ||
| 6 | |||
| 7 | """ | ||
| 8 | from DBSAPI.dbsApi import DbsApi | ||
| 9 | from DBSAPI.dbsOptions import DbsOptionParser | ||
| 10 | |||
| 11 | import string,sys,os | ||
| 12 | |||
| 13 | # | ||
| 14 | # Check if the parents of a dataset are all valid (or really not invalid) | ||
| 15 | # | ||
| 16 | def parentsValid(dbsapi,ads): | ||
| 17 | parents=dbsapi.listDatasetParents(ads) | ||
| 18 | parentsOK=True | ||
| 19 | |||
| 20 | for p in parents: | ||
| 21 | path=p['PathList'][0] | ||
| 22 | dsStatus=datasetStatus(dbsapi,path) | ||
| 23 | # do we want to check other statuses? | ||
| 24 | if dsStatus == "INVALID" or dsStatus == "INVALID-RO": | ||
| 25 | parentsOK=False | ||
| 26 | print "Parent dataset %s is %s, must be a valid status to validate child" % (path, dsStatus) | ||
| 27 | |||
| 28 | return parentsOK | ||
| 29 | |||
| 30 | # | ||
| 31 | # Check that the children of a dataset are all invalid | ||
| 32 | # | ||
| 33 | def childrenInvalid(dbsapi,ads): | ||
| 34 | childrenOK=True | ||
| 35 | dsBlocks=dbsapi.listBlocks(dataset=ads) | ||
| 36 | childPaths = {} | ||
| 37 | |||
| 38 | # | ||
| 39 | # To find the children we have to go through the block parentage, so accumulate the list of parent | ||
| 40 | # datasets and check each one only once | ||
| 41 | # | ||
| 42 | for b in dsBlocks: | ||
| 43 | blockChildren=dbsapi.listBlockChildren(b['Name']) | ||
| 44 | for c in blockChildren: | ||
| 45 | childPaths[c['Path']] = 1 | ||
| 46 | |||
| 47 | for c in childPaths.keys(): | ||
| 48 | dsStatus=datasetStatus(dbsapi,c) | ||
| 49 | if dsStatus != "INVALID" and dsStatus != "INVALID-RO": | ||
| 50 | childrenOK=False | ||
| 51 | print "Child dataset %s is %s, must be invalid to invalidate parent" % (c, dsStatus) | ||
| 52 | |||
| 53 | return childrenOK | ||
| 54 | |||
| 55 | # | ||
| 56 | # Get the current status of a dataset | ||
| 57 | # | ||
| 58 | def datasetStatus(dbsapi,ads): | ||
| 59 | |||
| 60 | params = ads.split('/') | ||
| 61 | proc = dbsapi.listProcessedDatasets(patternPrim=params[1], patternProc=params[2], patternDT=params[3]) | ||
| 62 | |||
| 63 | if proc and proc[0]['Status']: | ||
| 64 | dsStatus = proc[0]['Status'] | ||
| 65 | else: | ||
| 66 | # listProcessedDatasets will not list INVALID datasets, so check the list of all datasets to see | ||
| 67 | # if it is there | ||
| 68 | paths = dbsapi.listDatasetPaths() | ||
| 69 | pathset = set(paths) | ||
| 70 | if ads in pathset: | ||
| 71 | dsStatus = 'INVALID' | ||
| 72 | else: | ||
| 73 | dsStatus = 'NOT FOUND' | ||
| 74 | |||
| 75 | return dsStatus | ||
| 76 | |||
| 77 | # | ||
| 78 | # Invalidate/validate dataset | ||
| 79 | # | ||
| 80 | def setDatasetStatus(dbsapi,ads,valid): | ||
| 81 | oldStatus = datasetStatus(dbsapi, ads) | ||
| 82 | didit=False | ||
| 83 | |||
| 84 | if valid: | ||
| 85 | if oldStatus == 'INVALID': | ||
| 86 | if parentsValid(dbsapi,ads): | ||
| 87 | print "Validating Dataset %s"%ads | ||
| 88 | dbsapi.updateProcDSStatus(ads,"VALID") | ||
| 89 | didit=True | ||
| 90 | else: | ||
| 91 | print "Dataset %s status is %s, must be INVALID to validate" % (ads, oldStatus) | ||
| 92 | |||
| 93 | else: | ||
| 94 | if oldStatus == 'VALID': | ||
| 95 | if childrenInvalid(dbsapi,ads): | ||
| 96 | print "Invalidating Dataset %s"%ads | ||
| 97 | dbsapi.updateProcDSStatus(ads,"INVALID") | ||
| 98 | didit=True | ||
| 99 | else: | ||
| 100 | print "Dataset %s status is %s, must be VALID to invalidate" % (ads, oldStatus) | ||
| 101 | |||
| 102 | return didit | ||
| 103 | |||
| 104 | # | ||
| 105 | # Invalidate/validate all the files in a dataset | ||
| 106 | # | ||
| 107 | def setDatasetFilesStatus(dbsapi,ads,valid): | ||
| 108 | |||
| 109 | retrieveList=['retrive_status'] | ||
| 110 | |||
| 111 | if valid: | ||
| 112 | retrieveList.append('retrive_invalid_files') | ||
| 113 | newStatus='VALID' | ||
| 114 | oldStatus='INVALID' | ||
| 115 | else: | ||
| 116 | newStatus='INVALID' | ||
| 117 | oldStatus='VALID' | ||
| 118 | |||
| 119 | files=dbsapi.listFiles(path=ads,retriveList=retrieveList) | ||
| 120 | |||
| 121 | if files: | ||
| 122 | print "Setting files to status " + newStatus | ||
| 123 | |||
| 124 | for f in files: | ||
| 125 | if f['Status'] == oldStatus: | ||
| 126 | dbsapi.updateFileStatus(f['LogicalFileName'], newStatus) | ||
| 127 | |||
| 128 | return | ||
| 129 | |||
| 130 | # | ||
| 131 | # Process a list of datasets | ||
| 132 | # | ||
| 133 | def updateDatasetStatus(dbsapi,lds,valid,files): | ||
| 134 | if (lds != None): | ||
| 135 | datasetList=lds.split(',') | ||
| 136 | for ads in datasetList: | ||
| 137 | if setDatasetStatus(dbsapi,ads,valid) and files: | ||
| 138 | setDatasetFilesStatus(dbsapi,ads,valid) | ||
| 139 | |||
| 140 | return | ||
| 141 | |||
| 142 | def main (): | ||
| 143 | from optparse import OptionParser | ||
| 144 | |||
| 145 | usage="""\npython DBSInvalidateDataset <options> \nOptions: \n --DBSURL=<URL> \t\t DBS URL \n --datasetPath=<dataset> \t dataset \n --valid \t\t\t re-validate an invalid dataset\n --files \t\t\t change status of all the files in the dataset""" | ||
| 146 | parser = OptionParser(usage=usage) | ||
| 147 | |||
| 148 | parser.add_option('-D', '--DBSURL', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL') | ||
| 149 | parser.add_option('-d', '--datasetPath', dest='dataset', default=None, help='Dataset') | ||
| 150 | parser.add_option('-v', '--valid', action="store_true", default=False,dest='valid', help='Validate status instead of invalidate') | ||
| 151 | parser.add_option('-f', '--files', action="store_true", default=False,dest='files', help='Validate or invalidate all files in dataset') | ||
| 152 | |||
| 153 | (opts, args) = parser.parse_args() | ||
| 154 | |||
| 155 | if opts.url == None: | ||
| 156 | print "--url option not provided." | ||
| 157 | print "Using %s"%opts.url | ||
| 158 | |||
| 159 | if opts.dataset == None: | ||
| 160 | print "--dataset option must be provided" | ||
| 161 | print usage; | ||
| 162 | sys.exit(1) | ||
| 163 | |||
| 164 | dbsargs = {'url' : opts.url} | ||
| 165 | dbsapi = DbsApi(dbsargs) | ||
| 166 | |||
| 167 | try: | ||
| 168 | if opts.dataset != None: | ||
| 169 | updateDatasetStatus(dbsapi, opts.dataset, opts.valid, opts.files) | ||
| 170 | |||
| 171 | except Exception, ex: | ||
| 172 | print "Caught exception %s:"%str(ex) | ||
| 173 | sys.exit(1) | ||
| 174 | |||
| 175 | sys.exit(0) | ||
| 176 | |||
| 177 | if __name__ == "__main__": | ||
| 178 | main() | ||
| 179 |