| 1 |
fanzago |
1.1 |
#!/usr/bin/env python
|
| 2 |
|
|
import sys
|
| 3 |
|
|
for p in sys.path:
|
| 4 |
|
|
if p.find( "python2.3/lib-dynload" ) != -1 :
|
| 5 |
|
|
sys.path.pop( sys.path.index(p) )
|
| 6 |
|
|
|
| 7 |
|
|
from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter
|
| 8 |
|
|
from ProdCommon.DataMgmt.DBS.DBSReader import DBSReader
|
| 9 |
|
|
from ProdCommon.DataMgmt.DBS.DBSErrors import DBSWriterError, formatEx
|
| 10 |
|
|
from DBSAPI.dbsApiException import DbsException
|
| 11 |
|
|
import os,getopt
|
| 12 |
|
|
from Actor import *
|
| 13 |
|
|
import common
|
| 14 |
|
|
|
| 15 |
|
|
|
| 16 |
|
|
class InspectDBS(Actor):
|
| 17 |
|
|
def __init__(self, cfg_params):
|
| 18 |
|
|
"""
|
| 19 |
|
|
InspectDBS class:
|
| 20 |
|
|
|
| 21 |
|
|
- check data publication in a DBS
|
| 22 |
|
|
"""
|
| 23 |
|
|
|
| 24 |
|
|
try:
|
| 25 |
|
|
self.DBSURL=cfg_params['USER.dbs_url_for_publication']
|
| 26 |
|
|
except KeyError:
|
| 27 |
|
|
msg = "Warning. You have to specify the url of DBS in the USER.dbs_url_for_publication parameter of crab.cfg or as command line option: \n"
|
| 28 |
|
|
msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n"
|
| 29 |
|
|
raise CrabException(msg)
|
| 30 |
|
|
|
| 31 |
|
|
try:
|
| 32 |
|
|
self.dataset_to_check=cfg_params['USER.dataset_to_check']
|
| 33 |
|
|
except KeyError:
|
| 34 |
|
|
msg = "Warning. You have to speficy the dataset you want to check in the USER.dataset_to_check parameter of crab.cfg or as command line option: \n"
|
| 35 |
|
|
msg += "crab -checkPublication -USER.dbs_url_for_publication=<DBS url where data are published> -USER.dataset_to_check=<datasetpath to check>\n"
|
| 36 |
|
|
raise CrabException(msg)
|
| 37 |
|
|
|
| 38 |
|
|
|
| 39 |
|
|
def checkPublication(self):
|
| 40 |
|
|
"""
|
| 41 |
|
|
check dataset publication in a dbs
|
| 42 |
|
|
"""
|
| 43 |
|
|
|
| 44 |
|
|
common.logger.info('--->>> Check data publication: dataset '+self.dataset_to_check+' in DBS url '+ self.DBSURL+'\n')
|
| 45 |
|
|
# //
|
| 46 |
|
|
# // Get API to DBS
|
| 47 |
|
|
#//
|
| 48 |
|
|
dbsreader = DBSReader(self.DBSURL)
|
| 49 |
|
|
# //
|
| 50 |
|
|
# // Get list of datasets
|
| 51 |
|
|
#//
|
| 52 |
|
|
if len(self.dataset_to_check.split('/')) < 4:
|
| 53 |
|
|
msg = "the provided dataset name is not correct"
|
| 54 |
|
|
raise CrabException(msg)
|
| 55 |
|
|
else:
|
| 56 |
|
|
primds=self.dataset_to_check.split('/')[1]
|
| 57 |
|
|
procds=self.dataset_to_check.split('/')[2]
|
| 58 |
|
|
tier=self.dataset_to_check.split('/')[3]
|
| 59 |
|
|
datasets=dbsreader.matchProcessedDatasets(primds,tier,procds)
|
| 60 |
|
|
if common.logger.debugLevel():
|
| 61 |
|
|
print "PrimaryDataset = ", primds
|
| 62 |
|
|
print "ProcessedDataset = ", procds
|
| 63 |
|
|
print "DataTier = ", tier
|
| 64 |
|
|
print "datasets matching your requirements= ", datasets
|
| 65 |
|
|
|
| 66 |
|
|
for dataset in datasets:
|
| 67 |
|
|
# //
|
| 68 |
|
|
# // Get list of blocks for the dataset and their location
|
| 69 |
|
|
#//
|
| 70 |
|
|
if len(dataset.get('PathList'))==0:
|
| 71 |
|
|
print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList'))
|
| 72 |
|
|
else:
|
| 73 |
|
|
for datasetpath in dataset.get('PathList'):
|
| 74 |
|
|
nevttot=0
|
| 75 |
|
|
print "=== dataset %s"%datasetpath
|
| 76 |
|
|
blocks=dbsreader.getFileBlocksInfo(datasetpath)
|
| 77 |
|
|
for block in blocks:
|
| 78 |
|
|
SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query
|
| 79 |
|
|
print "===== File block name: %s" %block['Name']
|
| 80 |
|
|
print " File block located at: ", SEList
|
| 81 |
|
|
print " File block status: %s" %block['OpenForWriting']
|
| 82 |
|
|
print " Number of files: %s"%block['NumberOfFiles']
|
| 83 |
|
|
print " Number of Bytes: %s"%block['BlockSize']
|
| 84 |
|
|
print " Number of Events: %s"%block['NumberOfEvents']
|
| 85 |
|
|
if common.logger.debugLevel():
|
| 86 |
|
|
print "--------- info about files --------"
|
| 87 |
|
|
print " Size \t Events \t LFN \t FileStatus "
|
| 88 |
|
|
files=dbsreader.listFilesInBlock(block['Name'])
|
| 89 |
|
|
for file in files:
|
| 90 |
|
|
print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status'])
|
| 91 |
|
|
nevttot = nevttot + block['NumberOfEvents']
|
| 92 |
|
|
print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath)
|
| 93 |
|
|
if not common.logger.debugLevel():
|
| 94 |
|
|
common.logger.info('You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='+self.dataset_to_check+' -USER.dbs_url_for_publication='+self.DBSURL+' -debug')
|
| 95 |
|
|
|
| 96 |
|
|
def run(self):
|
| 97 |
|
|
"""
|
| 98 |
|
|
parse of all xml file on res dir and creation of distionary
|
| 99 |
|
|
"""
|
| 100 |
|
|
self.checkPublication()
|
