| 75 |
|
self.dataset_to_import.append(dataset) |
| 76 |
|
### |
| 77 |
|
|
| 78 |
< |
self.import_all_parents = cfg_params.get('USER.publish_with_import_all_parents',0) |
| 78 |
> |
self.import_all_parents = cfg_params.get('USER.publish_with_import_all_parents',1) |
| 79 |
|
self.skipOcheck=cfg_params.get('CMSSW.publish_zero_event',0) |
| 80 |
|
|
| 81 |
|
self.SEName='' |
| 92 |
|
dbsWriter = DBSWriter(self.DBSURL,level='ERROR') |
| 93 |
|
|
| 94 |
|
try: |
| 95 |
< |
if (self.import_all_parents=='1'): |
| 95 |
> |
if (self.import_all_parents==1): |
| 96 |
|
common.logger.info("--->>> Importing all parents level") |
| 97 |
< |
dbsWriter.importDataset(globalDBS, datasetpath, self.DBSURL) |
| 97 |
> |
start = time.time() |
| 98 |
> |
common.logger.debug("start import time: " + str(start)) |
| 99 |
> |
### to skip the ProdCommon api exception in the case of block without location |
| 100 |
> |
### skipNoSiteError=True |
| 101 |
> |
#dbsWriter.importDataset(globalDBS, datasetpath, self.DBSURL, skipNoSiteError=True) |
| 102 |
> |
### calling dbs api directly |
| 103 |
> |
dbsWriter.dbs.migrateDatasetContents(globalDBS, self.DBSURL, datasetpath) |
| 104 |
> |
stop = time.time() |
| 105 |
> |
common.logger.debug("stop import time: " + str(stop)) |
| 106 |
> |
common.logger.info("--->>> duration of all parents import (sec): "+str(stop - start)) |
| 107 |
> |
|
| 108 |
|
else: |
| 109 |
|
common.logger.info("--->>> Importing only the datasetpath " + datasetpath) |
| 110 |
< |
dbsWriter.importDatasetWithoutParentage(globalDBS, datasetpath, self.DBSURL) |
| 110 |
> |
start = time.time() |
| 111 |
> |
#dbsWriter.importDatasetWithoutParentage(globalDBS, datasetpath, self.DBSURL, skipNoSiteError=True) |
| 112 |
> |
### calling dbs api directly |
| 113 |
> |
common.logger.debug("start import time: " + str(start)) |
| 114 |
> |
dbsWriter.dbs.migrateDatasetContents(globalDBS, self.DBSURL, datasetpath, noParentsReadOnly = True ) |
| 115 |
> |
stop = time.time() |
| 116 |
> |
common.logger.debug("stop import time: " + str(stop)) |
| 117 |
> |
common.logger.info("--->>> duration of first level parent import (sec): "+str(stop - start)) |
| 118 |
|
except DBSWriterError, ex: |
| 119 |
|
msg = "Error importing dataset to be processed into local DBS\n" |
| 120 |
|
msg += "Source Dataset: %s\n" % datasetpath |
| 121 |
|
msg += "Source DBS: %s\n" % globalDBS |
| 122 |
|
msg += "Destination DBS: %s\n" % self.DBSURL |
| 123 |
|
common.logger.info(msg) |
| 124 |
< |
common.logger.debug(str(ex)) |
| 124 |
> |
common.logger.info(str(ex)) |
| 125 |
|
return 1 |
| 126 |
|
return 0 |
| 127 |
< |
""" |
| 111 |
< |
print " patch for importParentDataset: datasetpath = ", datasetpath |
| 112 |
< |
try: |
| 113 |
< |
args={} |
| 114 |
< |
args['url']=self.DBSURL |
| 115 |
< |
args['mode']='POST' |
| 116 |
< |
block = "" |
| 117 |
< |
api = DbsApi(args) |
| 118 |
< |
#api.migrateDatasetContents(srcURL, dstURL, path, block , False) |
| 119 |
< |
api.migrateDatasetContents(globalDBS, self.DBSURL, datasetpath, block , False) |
| 120 |
< |
|
| 121 |
< |
except DbsException, ex: |
| 122 |
< |
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ) |
| 123 |
< |
if ex.getErrorCode() not in (None, ""): |
| 124 |
< |
print "DBS Exception Error Code: ", ex.getErrorCode() |
| 125 |
< |
return 1 |
| 126 |
< |
print "Done" |
| 127 |
< |
return 0 |
| 128 |
< |
""" |
| 127 |
> |
|
| 128 |
|
def publishDataset(self,file): |
| 129 |
|
""" |
| 130 |
|
""" |
| 155 |
|
self.exit_status = '0' |
| 156 |
|
except IndexError: |
| 157 |
|
self.exit_status = '1' |
| 158 |
< |
msg = "Error: No file to publish in xml file"+file+" file" |
| 158 |
> |
msg = "Error: No EDM file to publish in xml file"+file+" file" |
| 159 |
|
common.logger.info(msg) |
| 160 |
|
return self.exit_status |
| 161 |
|
|
| 204 |
|
""" |
| 205 |
|
input: xml file, processedDataset |
| 206 |
|
""" |
| 207 |
+ |
common.logger.debug("FJR = %s"%file) |
| 208 |
|
try: |
| 209 |
|
jobReport = readJobReport(file)[0] |
| 210 |
|
self.exit_status = '0' |
| 246 |
|
filestopublish.append(file) |
| 247 |
|
|
| 248 |
|
jobReport.files = filestopublish |
| 249 |
+ |
for file in filestopublish: |
| 250 |
+ |
common.logger.debug("--->>> LFN of file to publish = " + str(file['LFN'])) |
| 251 |
|
### if all files of FJR have number of events = 0 |
| 252 |
|
if (len(filestopublish) == 0): |
| 253 |
< |
return None |
| 253 |
> |
return None |
| 254 |
|
|
| 255 |
|
#// DBS to contact |
| 256 |
|
dbswriter = DBSWriter(self.DBSURL) |
| 258 |
|
Blocks=None |
| 259 |
|
try: |
| 260 |
|
Blocks=dbswriter.insertFiles(jobReport) |
| 261 |
< |
common.logger.info("Inserting file in blocks = %s"%Blocks) |
| 261 |
> |
common.logger.debug("--->>> Inserting file in blocks = %s"%Blocks) |
| 262 |
|
except DBSWriterError, ex: |
| 263 |
< |
common.logger.info("Insert file error: %s"%ex) |
| 263 |
> |
common.logger.debug("--->>> Insert file error: %s"%ex) |
| 264 |
|
return Blocks |
| 265 |
|
|
| 266 |
|
def run(self): |
| 269 |
|
""" |
| 270 |
|
|
| 271 |
|
file_list = glob.glob(self.resDir+"crab_fjr*.xml") |
| 272 |
+ |
|
| 273 |
|
## Select only those fjr that are succesfull |
| 274 |
+ |
if (len(file_list)==0): |
| 275 |
+ |
common.logger.info("--->>> "+self.resDir+" empty: no file to publish on DBS") |
| 276 |
+ |
self.exit_status = '1' |
| 277 |
+ |
return self.exit_status |
| 278 |
+ |
|
| 279 |
|
good_list=[] |
| 280 |
|
for fjr in file_list: |
| 281 |
|
reports = readJobReport(fjr) |
| 298 |
|
|
| 299 |
|
common.logger.info("--->>> Start files publication") |
| 300 |
|
for file in file_list: |
| 293 |
– |
common.logger.debug( "file = "+file) |
| 301 |
|
Blocks=self.publishAJobReport(file,self.processedData) |
| 302 |
|
if Blocks: |
| 303 |
|
for x in Blocks: # do not allow multiple entries of the same block |
| 338 |
|
return self.exit_status |
| 339 |
|
|
| 340 |
|
else: |
| 341 |
< |
common.logger.info("--->>> "+self.resDir+" empty: no file to publish on DBS") |
| 341 |
> |
common.logger.info("--->>> No valid files to publish on DBS. Your jobs do not report exit codes = 0") |
| 342 |
|
self.exit_status = '1' |
| 343 |
|
return self.exit_status |
| 344 |
|
|
