| Revision: | 1.2 |
| Committed: | Thu May 28 15:36:38 2009 UTC (15 years, 11 months ago) by fanzago |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_6_6_pre6, CRAB_2_7_1_pre2, CRAB_2_6_6_pre5, CRAB_2_7_1_pre1, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_7_0, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, CRAB_2_6_0_pre7, CRAB_2_6_0_pre6, CRAB_2_6_0_pre5 |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Changes since 1.1: | +3 -3 lines |
| Log Message: | common.logger.debugLevel --> common.debugLevel |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | fanzago | 1.1 | #!/usr/bin/env python |
| 2 | import sys | ||
| 3 | for p in sys.path: | ||
| 4 | if p.find( "python2.3/lib-dynload" ) != -1 : | ||
| 5 | sys.path.pop( sys.path.index(p) ) | ||
| 6 | |||
| 7 | from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter | ||
| 8 | from ProdCommon.DataMgmt.DBS.DBSReader import DBSReader | ||
| 9 | from ProdCommon.DataMgmt.DBS.DBSErrors import DBSWriterError, formatEx | ||
| 10 | from DBSAPI.dbsApiException import DbsException | ||
| 11 | import os,getopt | ||
| 12 | from Actor import * | ||
| 13 | import common | ||
| 14 | |||
| 15 | |||
| 16 | class InspectDBS(Actor): | ||
| 17 | def __init__(self, cfg_params): | ||
| 18 | """ | ||
| 19 | InspectDBS class: | ||
| 20 | |||
| 21 | - check data publication in a DBS | ||
| 22 | """ | ||
| 23 | |||
| 24 | try: | ||
| 25 | self.DBSURL=cfg_params['USER.dbs_url_for_publication'] | ||
| 26 | except KeyError: | ||
| 27 | msg = "Warning. You have to specify the url of DBS in the USER.dbs_url_for_publication parameter of crab.cfg or as command line option: \n" | ||
| 28 | msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n" | ||
| 29 | raise CrabException(msg) | ||
| 30 | |||
| 31 | try: | ||
| 32 | self.dataset_to_check=cfg_params['USER.dataset_to_check'] | ||
| 33 | except KeyError: | ||
| 34 | msg = "Warning. You have to speficy the dataset you want to check in the USER.dataset_to_check parameter of crab.cfg or as command line option: \n" | ||
| 35 | msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n" | ||
| 36 | raise CrabException(msg) | ||
| 37 | |||
| 38 | |||
| 39 | def checkPublication(self): | ||
| 40 | """ | ||
| 41 | check dataset publication in a dbs | ||
| 42 | """ | ||
| 43 | |||
| 44 | common.logger.info('--->>> Check data publication: dataset '+self.dataset_to_check+' in DBS url '+ self.DBSURL+'\n') | ||
| 45 | # // | ||
| 46 | # // Get API to DBS | ||
| 47 | #// | ||
| 48 | dbsreader = DBSReader(self.DBSURL) | ||
| 49 | # // | ||
| 50 | # // Get list of datasets | ||
| 51 | #// | ||
| 52 | if len(self.dataset_to_check.split('/')) < 4: | ||
| 53 | msg = "the provided dataset name is not correct" | ||
| 54 | raise CrabException(msg) | ||
| 55 | else: | ||
| 56 | primds=self.dataset_to_check.split('/')[1] | ||
| 57 | procds=self.dataset_to_check.split('/')[2] | ||
| 58 | tier=self.dataset_to_check.split('/')[3] | ||
| 59 | datasets=dbsreader.matchProcessedDatasets(primds,tier,procds) | ||
| 60 | fanzago | 1.2 | if common.debugLevel: |
| 61 | fanzago | 1.1 | print "PrimaryDataset = ", primds |
| 62 | print "ProcessedDataset = ", procds | ||
| 63 | print "DataTier = ", tier | ||
| 64 | print "datasets matching your requirements= ", datasets | ||
| 65 | |||
| 66 | for dataset in datasets: | ||
| 67 | # // | ||
| 68 | # // Get list of blocks for the dataset and their location | ||
| 69 | #// | ||
| 70 | if len(dataset.get('PathList'))==0: | ||
| 71 | print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList')) | ||
| 72 | else: | ||
| 73 | for datasetpath in dataset.get('PathList'): | ||
| 74 | nevttot=0 | ||
| 75 | print "=== dataset %s"%datasetpath | ||
| 76 | blocks=dbsreader.getFileBlocksInfo(datasetpath) | ||
| 77 | for block in blocks: | ||
| 78 | SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query | ||
| 79 | print "===== File block name: %s" %block['Name'] | ||
| 80 | print " File block located at: ", SEList | ||
| 81 | print " File block status: %s" %block['OpenForWriting'] | ||
| 82 | print " Number of files: %s"%block['NumberOfFiles'] | ||
| 83 | print " Number of Bytes: %s"%block['BlockSize'] | ||
| 84 | print " Number of Events: %s"%block['NumberOfEvents'] | ||
| 85 | fanzago | 1.2 | if common.debugLevel: |
| 86 | fanzago | 1.1 | print "--------- info about files --------" |
| 87 | print " Size \t Events \t LFN \t FileStatus " | ||
| 88 | files=dbsreader.listFilesInBlock(block['Name']) | ||
| 89 | for file in files: | ||
| 90 | print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status']) | ||
| 91 | nevttot = nevttot + block['NumberOfEvents'] | ||
| 92 | print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath) | ||
| 93 | fanzago | 1.2 | if not common.debugLevel: |
| 94 | fanzago | 1.1 | common.logger.info('You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='+self.dataset_to_check+' -USER.dbs_url_for_publication='+self.DBSURL+' -debug') |
| 95 | |||
| 96 | def run(self): | ||
| 97 | """ | ||
| 98 | parse of all xml file on res dir and creation of distionary | ||
| 99 | """ | ||
| 100 | self.checkPublication() |