| Revision: | 1.1 |
| Committed: | Fri Dec 7 16:55:18 2007 UTC (17 years, 4 months ago) by fanzago |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_4_0_test, CRAB_2_3_1, CRAB_2_3_1_pre6, CRAB_2_3_1_pre5, CRAB_2_3_1_pre4, CRAB_2_3_1_pre3, CRAB_2_3_1_pre2, CRAB_2_3_1_pre1, CRAB_2_3_0, CRAB_2_3_0_pre6, CRAB_2_3_0_pre1, CRAB_2_2_2_pre5, CRAB_2_2_2_pre4, CRAB_2_2_2_pre3, CRAB_2_2_2_pre2, CRAB_2_2_2_pre1, CRAB_2_2_1, CRAB_2_2_1_pre6, CRAB_2_2_1_pre5, CRAB_2_2_1_pre4, PRODCOMMON_0_10_7_testCS2, CRAB_2_2_1_pre3, CRAB_2_2_1_pre2, CRAB_2_2_1_pre1, CRAB_2_2_0, CRAB_2_2_0_pre21, CRAB_2_2_0_pre19, CRAB_2_2_0_pre18, CRAB_2_2_0_pre17, CRAB_2_2_0_pre16, CRAB_2_2_0_pre15, CRAB_2_2_0_pre13, CRAB_2_2_0_pre12, CRAB_2_2_0_pre11, CRAB_2_2_0_pre10, bp_osg_bdii, CRAB_2_2_0_pre9, CRAB_2_2_0_pre8, CRAB_2_2_0_pre7, CRAB_2_1_2, CRAB_2_2_0_pre5, CRAB_2_1_2_pre2, CRAB_2_1_2_pre1, CRAB_2_2_0_pre4, CRAB_2_2_0_pre2, CRAB_2_1_1, CRAB_2_1_1_pre3, CRAB_2_2_0_pre1, CRAB_2_1_1_pre1, CRAB_2_1_0, CRAB_2_1_0_pre6, CRAB_2_1_0_pre5, CRAB_2_1_0_pre4, CRAB_2_1_0_pre3, CRAB_2_1_0_pre2, CRAB_2_1_0_pre1, CRAB_2_0_4, CRAB_2_0_4_pre2, CRAB_2_0_4_pre1 |
| Branch point for: | CRAB_2_3_0_br, osg_bdii, CRAB_2_1_2_br, CRAB_2_1_1_pre2 |
| Log Message: | script to check data publication in dbs |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | fanzago | 1.1 | #!/usr/bin/env python |
| 2 | import sys | ||
| 3 | for p in sys.path: | ||
| 4 | if p.find( "python2.3/lib-dynload" ) != -1 : | ||
| 5 | sys.path.pop( sys.path.index(p) ) | ||
| 6 | ## | ||
| 7 | |||
| 8 | from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter | ||
| 9 | from ProdCommon.DataMgmt.DBS.DBSReader import DBSReader | ||
| 10 | from ProdCommon.DataMgmt.DBS.DBSErrors import DBSWriterError, formatEx | ||
| 11 | from DBSAPI.dbsApiException import DbsException | ||
| 12 | # | ||
| 13 | import os,getopt | ||
| 14 | |||
| 15 | # // | ||
| 16 | # // Get DBS instance to use | ||
| 17 | # // | ||
| 18 | usage="\n Usage: python InspectDBS2.py <options> \n Options: \n --datasetPath=/primarydataset/procdataset/tier \t\t dataset path \n --DBSURL=<URL> \t\t DBS URL \n --help \t\t\t\t print this help \n" | ||
| 19 | valid = ['DBSURL=','datasetPath=','full','help'] | ||
| 20 | try: | ||
| 21 | opts, args = getopt.getopt(sys.argv[1:], "", valid) | ||
| 22 | except getopt.GetoptError, ex: | ||
| 23 | print usage | ||
| 24 | print str(ex) | ||
| 25 | sys.exit(1) | ||
| 26 | |||
| 27 | #url = "http://cmssrv17.fnal.gov:8989/DBS/servlet/DBSServlet" | ||
| 28 | url = "http://cmssrv17.fnal.gov:8989/DBS108LOC1/servlet/DBSServlet" | ||
| 29 | |||
| 30 | datasetPath = None | ||
| 31 | full = False | ||
| 32 | |||
| 33 | for opt, arg in opts: | ||
| 34 | if opt == "--DBSURL": | ||
| 35 | url = arg | ||
| 36 | if opt == "--datasetPath": | ||
| 37 | datasetPath = arg | ||
| 38 | if opt == "--full": | ||
| 39 | full = True | ||
| 40 | if opt == "--help": | ||
| 41 | print usage | ||
| 42 | sys.exit(1) | ||
| 43 | |||
| 44 | if datasetPath == None: | ||
| 45 | print "--datasetPath option not provided. For example : --datasetPath /primarydataset/processeddataset/datatier" | ||
| 46 | print usage | ||
| 47 | sys.exit() | ||
| 48 | print ">>>>> DBS URL : %s"%(url,) | ||
| 49 | |||
| 50 | |||
| 51 | import logging | ||
| 52 | logging.disable(logging.INFO) | ||
| 53 | |||
| 54 | # // | ||
| 55 | # // Get API to DBS | ||
| 56 | #// | ||
| 57 | dbsreader = DBSReader(url) | ||
| 58 | # // | ||
| 59 | # // Get list of datasets | ||
| 60 | #// | ||
| 61 | if datasetPath: | ||
| 62 | primds=datasetPath.split('/')[1] | ||
| 63 | procds=datasetPath.split('/')[2] | ||
| 64 | tier=datasetPath.split('/')[3] | ||
| 65 | # print " matchProcessedDatasets(%s,%s,%s)"%(primds,tier,procds) | ||
| 66 | datasets=dbsreader.matchProcessedDatasets(primds,tier,procds) | ||
| 67 | else: | ||
| 68 | datasets=dbsreader.matchProcessedDatasets("*","*","*") | ||
| 69 | |||
| 70 | |||
| 71 | for dataset in datasets: | ||
| 72 | # // | ||
| 73 | # // Get list of blocks for the dataset and their location | ||
| 74 | #// | ||
| 75 | for datasetpath in dataset.get('PathList'): | ||
| 76 | nevttot=0 | ||
| 77 | print "===== dataset %s"%datasetpath | ||
| 78 | blocks=dbsreader.getFileBlocksInfo(datasetpath) | ||
| 79 | for block in blocks: | ||
| 80 | SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query | ||
| 81 | print "== File block %s is located at: %s"%(block['Name'],SEList) | ||
| 82 | print "File block name: %s" %block['Name'] | ||
| 83 | print "File block status: %s" %block['OpenForWriting'] | ||
| 84 | print "Number of files: %s"%block['NumberOfFiles'] | ||
| 85 | print "Number of Bytes: %s"%block['BlockSize'] | ||
| 86 | print "Number of Events: %s"%block['NumberOfEvents'] | ||
| 87 | if full: | ||
| 88 | print "--------- info about files --------" | ||
| 89 | print " Size \t Events \t LFN \t FileStatus " | ||
| 90 | files=dbsreader.listFilesInBlock(block['Name']) | ||
| 91 | for file in files: | ||
| 92 | print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status']) | ||
| 93 | nevttot = nevttot + block['NumberOfEvents'] | ||
| 94 | |||
| 95 | print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath) | ||
| 96 | |||
| 97 | if len(dataset.get('PathList'))==0: | ||
| 98 | print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList')) | ||
| 99 | |||
| 100 | |||
| 101 | |||
| 102 |