| Revision: | 1.1 |
| Committed: | Mon Oct 12 20:11:58 2009 UTC (15 years, 6 months ago) by dsr |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_7_1_pre2, CRAB_2_7_1_pre1, CRAB_2_7_0, CRAB_2_7_0_pre8, CRAB_2_7_0_pre7, CRAB_2_7_0_pre6, CRAB_2_7_0_pre5, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Log Message: | Initial version of script for CRAB users to invalidate a dataset in DBS |
| # | Content |
|---|---|
| 1 | #!/usr/bin/env python |
| 2 | """ |
| 3 | _DBSInvalidateDataset_ |
| 4 | |
| 5 | Command line tool to invalidate a dataset. |
| 6 | |
| 7 | """ |
| 8 | from DBSAPI.dbsApi import DbsApi |
| 9 | from DBSAPI.dbsOptions import DbsOptionParser |
| 10 | |
| 11 | import string,sys,os |
| 12 | |
| 13 | # |
| 14 | # Check if the parents of a dataset are all valid (or really not invalid) |
| 15 | # |
| 16 | def parentsValid(dbsapi,ads): |
| 17 | parents=dbsapi.listDatasetParents(ads) |
| 18 | parentsOK=True |
| 19 | |
| 20 | for p in parents: |
| 21 | path=p['PathList'][0] |
| 22 | dsStatus=datasetStatus(dbsapi,path) |
| 23 | # do we want to check other statuses? |
| 24 | if dsStatus == "INVALID" or dsStatus == "INVALID-RO": |
| 25 | parentsOK=False |
| 26 | print "Parent dataset %s is %s, must be a valid status to validate child" % (path, dsStatus) |
| 27 | |
| 28 | return parentsOK |
| 29 | |
| 30 | # |
| 31 | # Check that the children of a dataset are all invalid |
| 32 | # |
| 33 | def childrenInvalid(dbsapi,ads): |
| 34 | childrenOK=True |
| 35 | dsBlocks=dbsapi.listBlocks(dataset=ads) |
| 36 | childPaths = {} |
| 37 | |
| 38 | # |
| 39 | # To find the children we have to go through the block parentage, so accumulate the list of parent |
| 40 | # datasets and check each one only once |
| 41 | # |
| 42 | for b in dsBlocks: |
| 43 | blockChildren=dbsapi.listBlockChildren(b['Name']) |
| 44 | for c in blockChildren: |
| 45 | childPaths[c['Path']] = 1 |
| 46 | |
| 47 | for c in childPaths.keys(): |
| 48 | dsStatus=datasetStatus(dbsapi,c) |
| 49 | if dsStatus != "INVALID" and dsStatus != "INVALID-RO": |
| 50 | childrenOK=False |
| 51 | print "Child dataset %s is %s, must be invalid to invalidate parent" % (c, dsStatus) |
| 52 | |
| 53 | return childrenOK |
| 54 | |
| 55 | # |
| 56 | # Get the current status of a dataset |
| 57 | # |
| 58 | def datasetStatus(dbsapi,ads): |
| 59 | |
| 60 | params = ads.split('/') |
| 61 | proc = dbsapi.listProcessedDatasets(patternPrim=params[1], patternProc=params[2], patternDT=params[3]) |
| 62 | |
| 63 | if proc and proc[0]['Status']: |
| 64 | dsStatus = proc[0]['Status'] |
| 65 | else: |
| 66 | # listProcessedDatasets will not list INVALID datasets, so check the list of all datasets to see |
| 67 | # if it is there |
| 68 | paths = dbsapi.listDatasetPaths() |
| 69 | pathset = set(paths) |
| 70 | if ads in pathset: |
| 71 | dsStatus = 'INVALID' |
| 72 | else: |
| 73 | dsStatus = 'NOT FOUND' |
| 74 | |
| 75 | return dsStatus |
| 76 | |
| 77 | # |
| 78 | # Invalidate/validate dataset |
| 79 | # |
| 80 | def setDatasetStatus(dbsapi,ads,valid): |
| 81 | oldStatus = datasetStatus(dbsapi, ads) |
| 82 | didit=False |
| 83 | |
| 84 | if valid: |
| 85 | if oldStatus == 'INVALID': |
| 86 | if parentsValid(dbsapi,ads): |
| 87 | print "Validating Dataset %s"%ads |
| 88 | dbsapi.updateProcDSStatus(ads,"VALID") |
| 89 | didit=True |
| 90 | else: |
| 91 | print "Dataset %s status is %s, must be INVALID to validate" % (ads, oldStatus) |
| 92 | |
| 93 | else: |
| 94 | if oldStatus == 'VALID': |
| 95 | if childrenInvalid(dbsapi,ads): |
| 96 | print "Invalidating Dataset %s"%ads |
| 97 | dbsapi.updateProcDSStatus(ads,"INVALID") |
| 98 | didit=True |
| 99 | else: |
| 100 | print "Dataset %s status is %s, must be VALID to invalidate" % (ads, oldStatus) |
| 101 | |
| 102 | return didit |
| 103 | |
| 104 | # |
| 105 | # Invalidate/validate all the files in a dataset |
| 106 | # |
| 107 | def setDatasetFilesStatus(dbsapi,ads,valid): |
| 108 | |
| 109 | retrieveList=['retrive_status'] |
| 110 | |
| 111 | if valid: |
| 112 | retrieveList.append('retrive_invalid_files') |
| 113 | newStatus='VALID' |
| 114 | oldStatus='INVALID' |
| 115 | else: |
| 116 | newStatus='INVALID' |
| 117 | oldStatus='VALID' |
| 118 | |
| 119 | files=dbsapi.listFiles(path=ads,retriveList=retrieveList) |
| 120 | |
| 121 | if files: |
| 122 | print "Setting files to status " + newStatus |
| 123 | |
| 124 | for f in files: |
| 125 | if f['Status'] == oldStatus: |
| 126 | dbsapi.updateFileStatus(f['LogicalFileName'], newStatus) |
| 127 | |
| 128 | return |
| 129 | |
| 130 | # |
| 131 | # Process a list of datasets |
| 132 | # |
| 133 | def updateDatasetStatus(dbsapi,lds,valid,files): |
| 134 | if (lds != None): |
| 135 | datasetList=lds.split(',') |
| 136 | for ads in datasetList: |
| 137 | if setDatasetStatus(dbsapi,ads,valid) and files: |
| 138 | setDatasetFilesStatus(dbsapi,ads,valid) |
| 139 | |
| 140 | return |
| 141 | |
| 142 | def main (): |
| 143 | from optparse import OptionParser |
| 144 | |
| 145 | usage="""\npython DBSInvalidateDataset <options> \nOptions: \n --DBSURL=<URL> \t\t DBS URL \n --datasetPath=<dataset> \t dataset \n --valid \t\t\t re-validate an invalid dataset\n --files \t\t\t change status of all the files in the dataset""" |
| 146 | parser = OptionParser(usage=usage) |
| 147 | |
| 148 | parser.add_option('-D', '--DBSURL', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL') |
| 149 | parser.add_option('-d', '--datasetPath', dest='dataset', default=None, help='Dataset') |
| 150 | parser.add_option('-v', '--valid', action="store_true", default=False,dest='valid', help='Validate status instead of invalidate') |
| 151 | parser.add_option('-f', '--files', action="store_true", default=False,dest='files', help='Validate or invalidate all files in dataset') |
| 152 | |
| 153 | (opts, args) = parser.parse_args() |
| 154 | |
| 155 | if opts.url == None: |
| 156 | print "--url option not provided." |
| 157 | print "Using %s"%opts.url |
| 158 | |
| 159 | if opts.dataset == None: |
| 160 | print "--dataset option must be provided" |
| 161 | print usage; |
| 162 | sys.exit(1) |
| 163 | |
| 164 | dbsargs = {'url' : opts.url} |
| 165 | dbsapi = DbsApi(dbsargs) |
| 166 | |
| 167 | try: |
| 168 | if opts.dataset != None: |
| 169 | updateDatasetStatus(dbsapi, opts.dataset, opts.valid, opts.files) |
| 170 | |
| 171 | except Exception, ex: |
| 172 | print "Caught exception %s:"%str(ex) |
| 173 | sys.exit(1) |
| 174 | |
| 175 | sys.exit(0) |
| 176 | |
| 177 | if __name__ == "__main__": |
| 178 | main() |
| 179 |