| 12 |
|
from ProdCommon.DataMgmt.DBS.DBSReader import DBSReader |
| 13 |
|
from ProdCommon.DataMgmt.DBS.DBSWriter import DBSWriter,DBSWriterObjects |
| 14 |
|
import sys |
| 15 |
+ |
from DBSAPI.dbsApiException import DbsException |
| 16 |
+ |
from DBSAPI.dbsApi import DbsApi |
| 17 |
|
|
| 18 |
|
class Publisher(Actor): |
| 19 |
|
def __init__(self, cfg_params): |
| 27 |
|
|
| 28 |
|
self.cfg_params=cfg_params |
| 29 |
|
|
| 30 |
< |
try: |
| 29 |
< |
self.userprocessedData = cfg_params['USER.publish_data_name'] |
| 30 |
< |
self.processedData = None |
| 31 |
< |
except KeyError: |
| 30 |
> |
if not cfg_params.has_key('USER.publish_data_name'): |
| 31 |
|
raise CrabException('Cannot publish output data, because you did not specify USER.publish_data_name parameter in the crab.cfg file') |
| 32 |
+ |
self.userprocessedData = cfg_params['USER.publish_data_name'] |
| 33 |
+ |
self.processedData = None |
| 34 |
|
|
| 35 |
< |
try: |
| 36 |
< |
if (int(cfg_params['USER.copy_data']) != 1): |
| 36 |
< |
raise KeyError |
| 37 |
< |
except KeyError: |
| 35 |
> |
if (not cfg_params.has_key('USER.copy_data') or int(cfg_params['USER.copy_data']) != 1 ) or \ |
| 36 |
> |
(not cfg_params.has_key('USER.publish_data') or int(cfg_params['USER.publish_data']) != 1 ): |
| 37 |
|
msg = 'You can not publish data because you did not selected \n' |
| 38 |
< |
msg += '\t*** copy_data = 1 or publish_data = 1 *** in the crab.cfg file' |
| 38 |
> |
msg += '\t*** copy_data = 1 and publish_data = 1 *** in the crab.cfg file' |
| 39 |
|
raise CrabException(msg) |
| 40 |
< |
try: |
| 41 |
< |
self.pset = cfg_params['CMSSW.pset'] |
| 43 |
< |
except KeyError: |
| 40 |
> |
|
| 41 |
> |
if not cfg_params.has_key('CMSSW.pset'): |
| 42 |
|
raise CrabException('Cannot publish output data, because you did not specify the psetname in [CMSSW] of your crab.cfg file') |
| 43 |
< |
try: |
| 44 |
< |
self.globalDBS=cfg_params['CMSSW.dbs_url'] |
| 45 |
< |
except KeyError: |
| 46 |
< |
self.globalDBS="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" |
| 47 |
< |
try: |
| 50 |
< |
self.DBSURL=cfg_params['USER.dbs_url_for_publication'] |
| 51 |
< |
common.logger.info('<dbs_url_for_publication> = '+self.DBSURL) |
| 52 |
< |
if (self.DBSURL == "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet") or (self.DBSURL == "https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet"): |
| 53 |
< |
msg = "You can not publish your data in the globalDBS = " + self.DBSURL + "\n" |
| 54 |
< |
msg = msg + "Please write your local one in the [USER] section 'dbs_url_for_publication'" |
| 55 |
< |
raise CrabException(msg) |
| 56 |
< |
except KeyError: |
| 43 |
> |
self.pset = cfg_params['CMSSW.pset'] |
| 44 |
> |
|
| 45 |
> |
self.globalDBS=cfg_params.get('CMSSW.dbs_url',"http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet") |
| 46 |
> |
|
| 47 |
> |
if not cfg_params.has_key('USER.dbs_url_for_publication'): |
| 48 |
|
msg = "Warning. The [USER] section does not have 'dbs_url_for_publication'" |
| 49 |
|
msg = msg + " entry, necessary to publish the data.\n" |
| 50 |
|
msg = msg + "Use the command **crab -publish -USER.dbs_url_for_publication=dbs_url_for_publication*** \nwhere dbs_url_for_publication is your local dbs instance." |
| 51 |
|
raise CrabException(msg) |
| 52 |
+ |
|
| 53 |
+ |
self.DBSURL=cfg_params['USER.dbs_url_for_publication'] |
| 54 |
+ |
common.logger.info('<dbs_url_for_publication> = '+self.DBSURL) |
| 55 |
+ |
if (self.DBSURL == "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet") or (self.DBSURL == "https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet"): |
| 56 |
+ |
msg = "You can not publish your data in the globalDBS = " + self.DBSURL + "\n" |
| 57 |
+ |
msg = msg + "Please write your local one in the [USER] section 'dbs_url_for_publication'" |
| 58 |
+ |
raise CrabException(msg) |
| 59 |
|
|
| 60 |
|
self.content=file(self.pset).read() |
| 61 |
|
self.resDir = common.work_space.resDir() |
| 75 |
|
self.dataset_to_import.append(dataset) |
| 76 |
|
### |
| 77 |
|
|
| 78 |
+ |
self.import_all_parents = cfg_params.get('USER.publish_with_import_all_parents',0) |
| 79 |
|
self.skipOcheck=cfg_params.get('CMSSW.publish_zero_event',0) |
| 80 |
|
|
| 81 |
|
self.SEName='' |
| 88 |
|
|
| 89 |
|
def importParentDataset(self,globalDBS, datasetpath): |
| 90 |
|
""" |
| 91 |
< |
""" |
| 91 |
> |
""" |
| 92 |
|
dbsWriter = DBSWriter(self.DBSURL,level='ERROR') |
| 93 |
|
|
| 94 |
|
try: |
| 95 |
< |
#dbsWriter.importDatasetWithoutParentage(globalDBS, self.datasetpath, self.DBSURL) |
| 96 |
< |
dbsWriter.importDataset(globalDBS, self.datasetpath, self.DBSURL) |
| 95 |
> |
if (self.import_all_parents=='1'): |
| 96 |
> |
common.logger.info("--->>> Importing all parents level") |
| 97 |
> |
dbsWriter.importDataset(globalDBS, datasetpath, self.DBSURL) |
| 98 |
> |
else: |
| 99 |
> |
common.logger.info("--->>> Importing only the datasetpath " + datasetpath) |
| 100 |
> |
dbsWriter.importDatasetWithoutParentage(globalDBS, datasetpath, self.DBSURL) |
| 101 |
|
except DBSWriterError, ex: |
| 102 |
|
msg = "Error importing dataset to be processed into local DBS\n" |
| 103 |
|
msg += "Source Dataset: %s\n" % datasetpath |
| 107 |
|
common.logger.debug(str(ex)) |
| 108 |
|
return 1 |
| 109 |
|
return 0 |
| 110 |
< |
|
| 110 |
> |
""" |
| 111 |
> |
print " patch for importParentDataset: datasetpath = ", datasetpath |
| 112 |
> |
try: |
| 113 |
> |
args={} |
| 114 |
> |
args['url']=self.DBSURL |
| 115 |
> |
args['mode']='POST' |
| 116 |
> |
block = "" |
| 117 |
> |
api = DbsApi(args) |
| 118 |
> |
#api.migrateDatasetContents(srcURL, dstURL, path, block , False) |
| 119 |
> |
api.migrateDatasetContents(globalDBS, self.DBSURL, datasetpath, block , False) |
| 120 |
> |
|
| 121 |
> |
except DbsException, ex: |
| 122 |
> |
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ) |
| 123 |
> |
if ex.getErrorCode() not in (None, ""): |
| 124 |
> |
print "DBS Exception Error Code: ", ex.getErrorCode() |
| 125 |
> |
return 1 |
| 126 |
> |
print "Done" |
| 127 |
> |
return 0 |
| 128 |
> |
""" |
| 129 |
|
def publishDataset(self,file): |
| 130 |
|
""" |
| 131 |
|
""" |
