| 1 |
#!/usr/bin/env python
|
| 2 |
import sys
|
| 3 |
for p in sys.path:
|
| 4 |
if p.find( "python2.3/lib-dynload" ) != -1 :
|
| 5 |
sys.path.pop( sys.path.index(p) )
|
| 6 |
|
| 7 |
from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter
|
| 8 |
from ProdCommon.DataMgmt.DBS.DBSReader import DBSReader
|
| 9 |
from ProdCommon.DataMgmt.DBS.DBSErrors import DBSWriterError, formatEx
|
| 10 |
from DBSAPI.dbsApiException import DbsException
|
| 11 |
import os,getopt
|
| 12 |
from Actor import *
|
| 13 |
import common
|
| 14 |
|
| 15 |
|
| 16 |
class InspectDBS(Actor):
|
| 17 |
def __init__(self, cfg_params):
|
| 18 |
"""
|
| 19 |
InspectDBS class:
|
| 20 |
|
| 21 |
- check data publication in a DBS
|
| 22 |
"""
|
| 23 |
|
| 24 |
try:
|
| 25 |
self.DBSURL=cfg_params['USER.dbs_url_for_publication']
|
| 26 |
except KeyError:
|
| 27 |
msg = "Warning. You have to specify the url of DBS in the USER.dbs_url_for_publication parameter of crab.cfg or as command line option: \n"
|
| 28 |
msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n"
|
| 29 |
raise CrabException(msg)
|
| 30 |
|
| 31 |
try:
|
| 32 |
self.dataset_to_check=cfg_params['USER.dataset_to_check']
|
| 33 |
except KeyError:
|
| 34 |
msg = "Warning. You have to speficy the dataset you want to check in the USER.dataset_to_check parameter of crab.cfg or as command line option: \n"
|
| 35 |
msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n"
|
| 36 |
raise CrabException(msg)
|
| 37 |
|
| 38 |
|
| 39 |
def checkPublication(self):
|
| 40 |
"""
|
| 41 |
check dataset publication in a dbs
|
| 42 |
"""
|
| 43 |
|
| 44 |
common.logger.info('--->>> Check data publication: dataset '+self.dataset_to_check+' in DBS url '+ self.DBSURL+'\n')
|
| 45 |
# //
|
| 46 |
# // Get API to DBS
|
| 47 |
#//
|
| 48 |
dbsreader = DBSReader(self.DBSURL)
|
| 49 |
# //
|
| 50 |
# // Get list of datasets
|
| 51 |
#//
|
| 52 |
if len(self.dataset_to_check.split('/')) < 4:
|
| 53 |
msg = "the provided dataset name is not correct"
|
| 54 |
raise CrabException(msg)
|
| 55 |
else:
|
| 56 |
primds=self.dataset_to_check.split('/')[1]
|
| 57 |
procds=self.dataset_to_check.split('/')[2]
|
| 58 |
tier=self.dataset_to_check.split('/')[3]
|
| 59 |
datasets=dbsreader.matchProcessedDatasets(primds,tier,procds)
|
| 60 |
if common.debugLevel:
|
| 61 |
print "PrimaryDataset = ", primds
|
| 62 |
print "ProcessedDataset = ", procds
|
| 63 |
print "DataTier = ", tier
|
| 64 |
print "datasets matching your requirements= ", datasets
|
| 65 |
|
| 66 |
for dataset in datasets:
|
| 67 |
# //
|
| 68 |
# // Get list of blocks for the dataset and their location
|
| 69 |
#//
|
| 70 |
if len(dataset.get('PathList'))==0:
|
| 71 |
print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList'))
|
| 72 |
else:
|
| 73 |
for datasetpath in dataset.get('PathList'):
|
| 74 |
nevttot=0
|
| 75 |
print "=== dataset %s"%datasetpath
|
| 76 |
### FEDE #######
|
| 77 |
if dataset['Description'] != None:
|
| 78 |
print "=== dataset description = ", dataset['Description']
|
| 79 |
################
|
| 80 |
blocks=dbsreader.getFileBlocksInfo(datasetpath)
|
| 81 |
for block in blocks:
|
| 82 |
SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query
|
| 83 |
print "===== File block name: %s" %block['Name']
|
| 84 |
print " File block located at: ", SEList
|
| 85 |
print " File block status: %s" %block['OpenForWriting']
|
| 86 |
print " Number of files: %s"%block['NumberOfFiles']
|
| 87 |
print " Number of Bytes: %s"%block['BlockSize']
|
| 88 |
print " Number of Events: %s"%block['NumberOfEvents']
|
| 89 |
if common.debugLevel:
|
| 90 |
print "--------- info about files --------"
|
| 91 |
print " Size \t Events \t LFN \t FileStatus "
|
| 92 |
files=dbsreader.listFilesInBlock(block['Name'])
|
| 93 |
for file in files:
|
| 94 |
print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status'])
|
| 95 |
nevttot = nevttot + block['NumberOfEvents']
|
| 96 |
print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath)
|
| 97 |
if not common.debugLevel:
|
| 98 |
common.logger.info('You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='+self.dataset_to_check+' -USER.dbs_url_for_publication='+self.DBSURL+' -debug')
|
| 99 |
|
| 100 |
def run(self):
|
| 101 |
"""
|
| 102 |
parse of all xml file on res dir and creation of distionary
|
| 103 |
"""
|
| 104 |
self.checkPublication()
|
