ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/MitAna/TreeMod/src/Analysis.cc
Revision: 1.29
Committed: Thu Mar 12 18:24:10 2009 UTC (16 years, 1 month ago) by loizides
Content type: text/plain
Branch: MAIN
CVS Tags: Mit_008pre2
Changes since 1.28: +2 -13 lines
Log Message:
Cleanup

File Contents

# Content
1 // $Id: Analysis.cc,v 1.28 2009/03/11 10:07:40 loizides Exp $
2
3 #include "MitAna/TreeMod/interface/Analysis.h"
4 #include <Riostream.h>
5 #include <TFile.h>
6 #include <TList.h>
7 #include <TDSet.h>
8 #include <TChain.h>
9 #include <TObjString.h>
10 #include <TSystem.h>
11 #include <TProof.h>
12 #include <TROOT.h>
13 #include <TBrowser.h>
14 #include "MitAna/DataUtil/interface/Debug.h"
15 #include "MitAna/DataTree/interface/Names.h"
16 #include "MitAna/TAM/interface/TAMVirtualLoader.h"
17 #include "MitAna/TAM/interface/TAModule.h"
18 #include "MitAna/TreeMod/interface/Selector.h"
19 #include "MitAna/TreeMod/interface/TreeLoader.h"
20 #include "MitAna/TreeMod/interface/AnaFwkMod.h"
21 #include "MitAna/TreeMod/interface/HLTFwkMod.h"
22 #include "MitAna/Catalog/interface/Catalog.h"
23 #include "MitAna/Catalog/interface/Dataset.h"
24
25 ClassImp(mithep::Analysis)
26
27 using namespace mithep;
28
29 //--------------------------------------------------------------------------------------------------
30 Analysis::Analysis(Bool_t useproof) :
31 fUseProof(useproof),
32 fUseHLT(kTRUE),
33 fHierarchy(kTRUE),
34 fDoProxy(kFALSE),
35 fDoObjTabClean(kTRUE),
36 fState(kPristine),
37 fNFriends(0),
38 fList(new TList),
39 fOutput(0),
40 fPackages(new TList),
41 fLoaders(new TList),
42 fSuperMods(new TList),
43 fSelector(0),
44 fChain(0),
45 fSet(0),
46 fDeleteList(new TList),
47 fTreeName(Names::gkEvtTreeName),
48 fCompLevel(7),
49 fProof(0),
50 fDoNEvents(TChain::kBigNumber),
51 fCacheSize(64*1024*1024)
52 {
53 // Default constructor.
54
55 fList->SetOwner();
56 fDeleteList->SetOwner();
57
58 /* default packages for PROOF upload */
59 fPackages->SetOwner();
60 // nothing to be done since we do not use par files (yet?)
61 }
62
63 //--------------------------------------------------------------------------------------------------
64 Analysis::~Analysis()
65 {
66 // Destructor.
67
68 if (fState == kInit || fState == kRun)
69 Terminate();
70
71 delete fList;
72 delete fPackages;
73 delete fLoaders;
74 delete fDeleteList;
75 delete fSelector;
76 delete fSuperMods;
77 fOutput = 0; // owned by TAM
78
79 delete fProof;
80 }
81
82 //--------------------------------------------------------------------------------------------------
83 Bool_t Analysis::AddDataset(const Dataset *dataset)
84 {
85 // Add a full dataset to the analysis.
86
87 Bool_t status = true;
88
89 for (UInt_t i=0; i<dataset->NFiles(); ++i)
90 status = (status && AddFile(dataset->FileUrl(i)));
91
92 return status;
93 }
94
95 //--------------------------------------------------------------------------------------------------
96 Bool_t Analysis::AddFile(const char *pname)
97 {
98 // Add file with given name to the list of files to be processed. Using the token "|", you can
99 // specify an arbritray number of paths to tree files that will be concatenated as friend trees.
100
101 if (fState != kPristine) {
102 Error("AddFile", "Analysis already initialized");
103 return kFALSE;
104 }
105
106 TString pnamestr(pname);
107 if (pnamestr.IsNull())
108 return kFALSE;
109
110 TString tok("|");
111 TObjArray *arr = pnamestr.Tokenize(tok);
112 if (!arr)
113 return kFALSE;
114
115 TString msg;
116 for (Int_t i=0; i<arr->GetEntries(); ++i) {
117
118 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
119 if (!dummy)
120 continue;
121
122 AddFile(dummy->GetName(),i);
123 if (i==0)
124 msg=dummy->GetName();
125 else
126 Info("AddFile", "Add file %s as friend to %s", dummy->GetName(), msg.Data());
127 }
128 delete arr;
129
130 return kTRUE;
131 }
132
133 //--------------------------------------------------------------------------------------------------
134 void Analysis::AddFile(const char *pname, Int_t eventlist)
135 {
136 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
137 // different types of events. In case you dont want friend trees, just give no eventlist argument
138 // (default 0).
139
140 MitAssert("AddFile", pname != 0);
141
142 TList *l = 0;
143 if (eventlist >= 0 && eventlist < fNFriends) {
144
145 l = dynamic_cast<TList*>(fList->At(eventlist));
146 if (!l) {
147 Fatal("AddFile", "Requested list %d not found!", eventlist);
148 return;
149 }
150
151 } else if (eventlist == fNFriends) {
152
153 l = new TList;
154 l->SetOwner();
155 fList->Add(l);
156 ++fNFriends;
157
158 } else if (eventlist < 0 || eventlist > fNFriends) {
159 Error("AddFile", "Specified list %d not in [0,%d]", eventlist, fNFriends);
160 return;
161 }
162
163 if (!IsValidName(pname)) return;
164
165 l->Add(new TObjString(pname));
166
167 MDB(kAnalysis, 2)
168 Info("AddFile", "Added %s to list of files.", pname);
169 }
170
171 //--------------------------------------------------------------------------------------------------
172 void Analysis::AddFile(const TObject *oname, Int_t eventlist)
173 {
174 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
175 // different types of events. In case you dont want mixing, just give no eventlist argument
176 // (default 0).
177
178 MitAssert("AddFile", oname != 0);
179
180 return AddFile(oname->GetName(), eventlist);
181 }
182
183 //________________________________________________________________________
184 Bool_t Analysis::AddFiles(const char *pname, Int_t nmax)
185 {
186 // Add files from text file with given name. If nmax>0, maximum nmax files will be added.
187
188 MitAssert("AddFiles", pname != 0);
189
190 ifstream in;
191 in.open(pname);
192 if (!in) {
193 Error("AddFiles", "Can not open file with name %s", pname);
194 return kFALSE;
195 }
196
197 Int_t fc = 0;
198 while (in) {
199 TString line;
200 line.ReadLine(in);
201 cout << line << endl;
202 if (!line.EndsWith(".root"))
203 continue;
204 cout << line << endl;
205
206 if (!AddFile(line)) {
207 Error("AddFiles", "Error adding file with name %s", line.Data());
208 return kFALSE;
209 }
210
211 ++fc;
212 if (nmax>0 && fc>=nmax) {
213 Info("AddFiles", "Maximal number (%d) of files added", nmax);
214 break;
215 }
216 }
217
218 return kTRUE;
219 }
220
221 //--------------------------------------------------------------------------------------------------
222 void Analysis::AddList(TList *list, Int_t eventlist)
223 {
224 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
225 // different types of events. In case you dont want mixing, just give no eventlist argument
226 // (default 0).
227
228 MitAssert("AddList", list != 0);
229
230 TIter next(list);
231 while (TObject *obj = next())
232 AddFile(obj->GetName(), eventlist);
233 }
234
235 //--------------------------------------------------------------------------------------------------
236 void Analysis::AddLoader(TAMVirtualLoader *l)
237 {
238 // Add loader to the list of loaders.
239
240 fLoaders->Add(l);
241 }
242
243 //--------------------------------------------------------------------------------------------------
244 void Analysis::AddPackage(const char* name)
245 {
246 // Add package to the list of uploaded packages.
247
248 MitAssert("AddPackage", name != 0);
249 fPackages->Add(new TObjString(name));
250 }
251
252 //--------------------------------------------------------------------------------------------------
253 void Analysis::AddPackages(TList *list)
254 {
255 // Add list of packages to the list of uploaded packages.
256
257 MitAssert("AddPackage", list != 0);
258
259 TIter next(list);
260 while (TObject *obj = next()) {
261 fPackages->Add(new TObjString(obj->GetName()));
262 }
263 }
264
265 //--------------------------------------------------------------------------------------------------
266 void Analysis::AddSuperModule(TAModule *mod)
267 {
268 // Add a top-level module to list of top-level (super) modules.
269
270 fSuperMods->Add(mod);
271 }
272
273 //--------------------------------------------------------------------------------------------------
274 void Analysis::FileInputFromEnv()
275 {
276 // Attempt to get list of filesets/files from environment.
277
278 TString catalog(gSystem->Getenv("MIT_CATALOG"));
279 TString book(gSystem->Getenv("MIT_BOOK"));
280 TString dataset(gSystem->Getenv("MIT_DATASET"));
281 TString filesets(gSystem->Getenv("MIT_FILESETS"));
282 TString files(gSystem->Getenv("MIT_FILES"));
283
284 if ((catalog.IsNull() || book.IsNull() || dataset.IsNull()) && files.IsNull()) {
285 Warning("FileInputFromEnv", "Called to get file info from environment, but did not get"
286 " consistent set of variables:\n\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\t"
287 "MIT_DATASET=%s\n\tMIT_FILESETS=%s\n\tMIT_FILES=%s\n",
288 catalog.Data(), book.Data(), dataset.Data(), filesets.Data(), files.Data());
289 return;
290 }
291
292 if (!files.IsNull()) { // add local files
293 Info("FileInputFromEnv", "Got from environment:\n"
294 "\n\tMIT_FILES=%s\n", files.Data());
295 TString tok(";");
296 TObjArray *arr = files.Tokenize(tok);
297 if (arr) {
298 for (Int_t i=0; i<arr->GetEntries(); ++i) {
299 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
300 if (!dummy) continue;
301 AddFile(dummy->GetName(),0);
302 }
303 delete arr;
304 }
305 return;
306 }
307
308 Info("FileInputFromEnv", "Got from environment:\n"
309 "\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\tMIT_DATASET=%s\n\tMIT_FILESETS=%s\n",
310 catalog.Data(), book.Data(), dataset.Data(), filesets.Data());
311
312 Catalog cat(catalog);
313 if (filesets.IsNull()) {
314 Dataset *d = cat.FindDataset(book, dataset);
315 if (d) {
316 AddDataset(d);
317 delete d;
318 }
319 } else {
320 TString tok(";");
321 TObjArray *arr = filesets.Tokenize(tok);
322 if (arr) {
323 for (Int_t i=0; i<arr->GetEntries(); ++i) {
324 TObjString *fileset = dynamic_cast<TObjString*>(arr->At(i));
325 if (!fileset) continue;
326 Dataset *d = cat.FindDataset(book, dataset, fileset->String());
327 if (!d)
328 continue;
329 AddDataset(d);
330 delete d;
331 }
332 delete arr;
333 }
334 }
335 }
336
337 //--------------------------------------------------------------------------------------------------
338 Bool_t Analysis::Init()
339 {
340 // Setup the TDSet and TChain to be used for the analysis with or without PROOF. If more than one
341 // list of file names was given, friend trees are supported.
342
343 if (fState == kRun || fState == kInit) {
344 Error("Init", "Init in state %d is not possible! Call Terminate() first.",
345 Int_t(fState));
346 return kFALSE;
347 }
348
349 // check if we should attempt to get filesets/filenames from environment
350 if (fNFriends == 0)
351 FileInputFromEnv();
352
353 if (fNFriends <= 0) {
354 Error("Init", "List of friend lists is empty!");
355 return kFALSE;
356 }
357
358 if (!fSuperMods->First()) {
359 Error("Init", "Top-level TAM module is NULL!");
360 return kFALSE;
361 }
362
363 if (fUseProof) { // first init our PROOF session
364 if (!InitProof())
365 return kFALSE;
366 }
367
368 // we do this here instead in Terminate() so that
369 // we can browse the output even after Terminate()
370 delete fSelector;
371 fSelector = 0;
372
373 fChain = new TChain(fTreeName);
374 fSet = new TDSet("TTree",fTreeName);
375
376 fChain->SetCacheSize(fCacheSize);
377
378 for (Int_t i=0; i<fNFriends; ++i) {
379
380 TList *l = dynamic_cast<TList*>(fList->At(i));
381 if (!l) {
382 Fatal("Init", "List %d not found!", i);
383 return kFALSE;
384 }
385
386 if (i == 0) {
387 TIter next(l);
388 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
389 fChain->Add(obj->GetName());
390 fSet->Add(obj->GetName());
391 }
392
393 } else {
394
395 TChain *chain = new TChain(fTreeName);
396 TDSet *set = new TDSet("TTree",fTreeName);
397
398 TIter next(l);
399 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
400 chain->Add(obj->GetName());
401 set->Add(obj->GetName());
402 }
403
404 TString alias("TAMTREE_"); // aliases currently not used
405 alias+=i;
406
407 fChain->AddFriend(chain,alias.Data());
408 fSet->AddFriend(set,alias.Data());
409
410 fDeleteList->Add(chain);
411 fDeleteList->Add(set);
412 }
413
414 }
415
416 // create our customized loader plugin for TAM
417 TreeLoader *bl = new TreeLoader;
418 fLoaders->Add(bl);
419 fDeleteList->Add(bl);
420
421 // create our ana framework module
422 AnaFwkMod *anamod = new AnaFwkMod;
423 fDeleteList->Add(anamod);
424
425 // create our HLT framework module
426 HLTFwkMod *hltmod = 0;
427 if (fUseHLT) {
428 hltmod = new HLTFwkMod;
429 fDeleteList->Add(hltmod);
430 }
431
432 if (fUseProof) {
433
434 fProof->AddInput(anamod);
435 if (hltmod)
436 fProof->AddInput(hltmod);
437
438 TIter iter(fSuperMods->MakeIterator());
439 while (1) {
440 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
441 if (!next) break;
442 fProof->AddInput(next);
443 }
444
445 fLoaders->SetName("TAM_LOADERS");
446 fProof->AddInput(fLoaders);
447
448 } else {
449
450 // when not running Proof, we must make a selector
451 fSelector = new Selector;
452 fSelector->SetDoProxy(fDoProxy);
453 fSelector->SetDoObjTabClean(fDoObjTabClean);
454
455 fSelector->AddInput(anamod);
456
457 if (hltmod)
458 fSelector->AddInput(hltmod);
459
460 TIter iter(fSuperMods->MakeIterator());
461 while (1) {
462 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
463 if (!next) break;
464 fSelector->AddInput(next);
465 }
466
467 MDB(kAnalysis, 2)
468 fSelector->SetVerbosity(1);
469
470 // pass loaders to selector
471 TIter next(fLoaders);
472 while (TAMVirtualLoader *l = dynamic_cast<TAMVirtualLoader*>(next()))
473 fSelector->AddLoader(l);
474 }
475
476 fState = kInit;
477 return kTRUE;
478 }
479
480 //--------------------------------------------------------------------------------------------------
481 Bool_t Analysis::InitProof()
482 {
483 // Initialize PROOF connection.
484
485 if (fProof && fProof->IsValid())
486 return kTRUE;
487
488 delete fProof;
489
490 if (fMaster.Contains("rcf.bnl.gov")) {
491 for (Int_t i=0; i<5; ++i) {
492 Warning("InitProof", "*** DID YOU RUN PROOF_KINIT? %d (5) ***", i);
493 gSystem->Sleep(1000);
494 }
495 }
496
497 MDB(kAnalysis, 1)
498 Info("InitProof", "Starting PROOF on master %s with config %s",
499 fMaster.Data(), fConfig.Data());
500
501 fProof = dynamic_cast<TProof*>(TProof::Open(fMaster, fConfig));
502 if (!fProof) {
503 Error("InitProof", "Could not start PROOF!");
504 return kFALSE;
505 }
506
507 MDB(kAnalysis, 3)
508 gROOT->Print();
509
510 //fProof->AddInput(new TNamed("PROOF_NewPacketizer",""));
511
512 Bool_t ret=kTRUE;
513 if (fPackages) {
514 // tell Proof what additional libraries we will need on each worker computer
515 ret = UploadPackages(fPackages);
516 }
517
518 return ret;
519 }
520
521 //--------------------------------------------------------------------------------------------------
522 void Analysis::Run()
523 {
524 // Run the analysis on the created file set.
525
526 if (fState == kPristine || fState == kRun) {
527 Error("Run", "Run in state %d is not possible! Call Init() first.",
528 Int_t(fState));
529 }
530
531 if (fUseProof) {
532
533 MDB(kAnalysis, 1)
534 Info("Run", "Start processing with PROOF...");
535
536 fSet->Process("Selector","",fDoNEvents);
537
538 } else {
539
540 MDB(kAnalysis, 1)
541 Info("Run", "Start processing (no PROOF)...");
542
543 fChain->Process(fSelector,"",fDoNEvents);
544 }
545
546 MDB(kAnalysis, 1)
547 Info("Run", "Processing complete!");
548
549 fState = kRun;
550 }
551
552 //--------------------------------------------------------------------------------------------------
553 Bool_t Analysis::Run(Bool_t browse)
554 {
555 // Execute analysis and open TBrowser if requested.
556
557 if (Init()) {
558 Run();
559 Terminate();
560 if (browse) {
561 new TBrowser;
562 }
563 return kTRUE;
564 }
565
566 Error("Execute", "Could not initialize analysis.");
567 return kFALSE;
568 }
569
570 //--------------------------------------------------------------------------------------------------
571 void Analysis::SetSuperModule(TAModule *mod)
572 {
573 // Set the first top-level module in the list of top-level (super) modules.
574
575 fSuperMods->AddFirst(mod);
576 }
577
578 //--------------------------------------------------------------------------------------------------
579 void Analysis::Terminate()
580 {
581 // Terminate current analysis run.
582
583 if (fState == kPristine || fState == kTerminate) {
584 Error("Terminate", "Terminate in state %d is not possible! Call Init() first.",
585 Int_t(fState));
586 return;
587 }
588
589 if (fState == kRun) {
590
591 if (fUseProof) {
592 // the output list from Proof can (in principal) contain other objects
593 // besides the module output hierarchy.
594 TList* outputlist = fProof->GetOutputList();
595 TIter nextOut(outputlist);
596 while (TObject *obj = nextOut()) {
597 if (obj->InheritsFrom(TAMOutput::Class())) {
598 fOutput = dynamic_cast<TList*>(obj);
599 break;
600 }
601 }
602
603 } else {
604 fOutput = fSelector->GetModOutput();
605 }
606
607
608
609 if (fOutput && !fAnaOutput.IsNull()) {
610 TDirectory::TContext context(0); // automatically restore gDirectory
611
612 std::auto_ptr<TFile> outf(TFile::Open(fAnaOutput,"recreate","", fCompLevel));
613 if (outf.get() == 0) {
614 Error("Terminate", "Could not open file %s for output!", fAnaOutput.Data());
615 } else {
616 MDB(kAnalysis, 1)
617 Info("Terminate", "Saving output to %s!", fAnaOutput.Data());
618
619 if (fHierarchy)
620 fOutput->Write(0,-99);
621 else
622 fOutput->Write();
623 }
624 }
625 // set state to terminate
626 fState = kTerminate;
627 }
628
629 delete fChain;
630 delete fSet;
631 fDeleteList->Delete();
632 }
633
634 //--------------------------------------------------------------------------------------------------
635 Bool_t Analysis::UploadPackages(TList *packages)
636 {
637 // Upload list of par files to the server.
638
639 MitAssert("UploadPackages", packages != 0);
640
641 for (Int_t i=0; i<packages->GetEntries(); ++i) {
642
643 TObject* objstr = packages->At(i);
644 if (!objstr) {
645 Error("InitProof", "Problem at package number %d!", i);
646 return kFALSE;
647 }
648
649 TString packname = objstr->GetName();
650 Int_t en = 0;
651 if (packname.EndsWith("+")) {
652 en=1;
653 packname.Resize(packname.Length()-1);
654 }
655
656 ifstream ftest(gSystem->ExpandPathName(packname.Data()),ios_base::binary);
657 if (!ftest.good()) {
658 Error("InitProof", "Could not open %s for upload!", packname.Data());
659 return kFALSE;
660 }
661
662 if (fProof->UploadPackage(packname)<0) {
663 Error("UploadPackage", "Upload for %s failed!", packname.Data());
664 return kFALSE;
665 }
666
667 if (en == 1) {
668 Int_t pos=packname.Last('/')+1;
669 if (pos)
670 packname.Remove(0,pos);
671 if (fProof->EnablePackage(packname)<0) {
672 Error("UploadPackage", "Enabling for %s failed!", packname.Data());
673 return kFALSE;
674 }
675 }
676 }
677
678 return kTRUE;
679 }