ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/MitAna/TreeMod/src/Analysis.cc
Revision: 1.32
Committed: Mon Mar 23 22:15:15 2009 UTC (16 years, 1 month ago) by loizides
Content type: text/plain
Branch: MAIN
CVS Tags: Mit_009c, Mit_009b, Mit_009a, Mit_009, Mit_008
Changes since 1.31: +3 -3 lines
Log Message:
Cosmetics

File Contents

# Content
1 // $Id: Analysis.cc,v 1.31 2009/03/23 14:39:52 loizides Exp $
2
3 #include "MitAna/TreeMod/interface/Analysis.h"
4 #include <Riostream.h>
5 #include <TFile.h>
6 #include <TList.h>
7 #include <TDSet.h>
8 #include <TChain.h>
9 #include <TObjString.h>
10 #include <TSystem.h>
11 #include <TProof.h>
12 #include <TROOT.h>
13 #include <TBrowser.h>
14 #include "MitAna/DataUtil/interface/Debug.h"
15 #include "MitAna/DataTree/interface/Names.h"
16 #include "MitAna/TAM/interface/TAMVirtualLoader.h"
17 #include "MitAna/TAM/interface/TAModule.h"
18 #include "MitAna/TreeMod/interface/Selector.h"
19 #include "MitAna/TreeMod/interface/TreeLoader.h"
20 #include "MitAna/TreeMod/interface/AnaFwkMod.h"
21 #include "MitAna/TreeMod/interface/HLTFwkMod.h"
22 #include "MitAna/Catalog/interface/Catalog.h"
23 #include "MitAna/Catalog/interface/Dataset.h"
24
25 ClassImp(mithep::Analysis)
26
27 using namespace mithep;
28
29 //--------------------------------------------------------------------------------------------------
30 Analysis::Analysis(Bool_t useproof) :
31 fUseProof(useproof),
32 fUseHLT(kTRUE),
33 fHierarchy(kTRUE),
34 fDoProxy(kFALSE),
35 fDoObjTabClean(kTRUE),
36 fState(kPristine),
37 fNFriends(0),
38 fList(new TList),
39 fOutput(0),
40 fPackages(new TList),
41 fLoaders(new TList),
42 fSuperMods(new TList),
43 fSelector(0),
44 fChain(0),
45 fSet(0),
46 fDeleteList(new TList),
47 fTreeName(Names::gkEvtTreeName),
48 fCompLevel(7),
49 fProof(0),
50 fDoNEvents(TChain::kBigNumber),
51 fSkipNEvents(0),
52 fCacheSize(-1)
53 {
54 // Default constructor.
55
56 fList->SetOwner();
57 fDeleteList->SetOwner();
58
59 /* default packages for PROOF upload */
60 fPackages->SetOwner();
61 // nothing to be done since we do not use par files (yet?)
62 }
63
64 //--------------------------------------------------------------------------------------------------
65 Analysis::~Analysis()
66 {
67 // Destructor.
68
69 if (fState == kInit || fState == kRun)
70 Terminate();
71
72 delete fList;
73 delete fPackages;
74 delete fLoaders;
75 delete fDeleteList;
76 delete fSelector;
77 delete fSuperMods;
78 fOutput = 0; // owned by TAM
79
80 delete fProof;
81 }
82
83 //--------------------------------------------------------------------------------------------------
84 Bool_t Analysis::AddDataset(const Dataset *dataset)
85 {
86 // Add a full dataset to the analysis.
87
88 Bool_t status = true;
89
90 for (UInt_t i=0; i<dataset->NFiles(); ++i)
91 status = (status && AddFile(dataset->FileUrl(i)));
92
93 return status;
94 }
95
96 //--------------------------------------------------------------------------------------------------
97 Bool_t Analysis::AddFile(const char *pname)
98 {
99 // Add file with given name to the list of files to be processed. Using the token "|", you can
100 // specify an arbritray number of paths to tree files that will be concatenated as friend trees.
101
102 if (fState != kPristine) {
103 Error("AddFile", "Analysis already initialized");
104 return kFALSE;
105 }
106
107 TString pnamestr(pname);
108 if (pnamestr.IsNull())
109 return kFALSE;
110
111 TString tok("|");
112 TObjArray *arr = pnamestr.Tokenize(tok);
113 if (!arr)
114 return kFALSE;
115
116 TString msg;
117 for (Int_t i=0; i<arr->GetEntries(); ++i) {
118
119 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
120 if (!dummy)
121 continue;
122
123 AddFile(dummy->GetName(),i);
124 if (i==0)
125 msg=dummy->GetName();
126 else
127 Info("AddFile", "Add file %s as friend to %s", dummy->GetName(), msg.Data());
128 }
129 delete arr;
130
131 return kTRUE;
132 }
133
134 //--------------------------------------------------------------------------------------------------
135 void Analysis::AddFile(const char *pname, Int_t eventlist)
136 {
137 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
138 // different types of events. In case you dont want friend trees, just give no eventlist argument
139 // (default 0).
140
141 MitAssert("AddFile", pname != 0);
142
143 TList *l = 0;
144 if (eventlist >= 0 && eventlist < fNFriends) {
145
146 l = dynamic_cast<TList*>(fList->At(eventlist));
147 if (!l) {
148 Fatal("AddFile", "Requested list %d not found!", eventlist);
149 return;
150 }
151
152 } else if (eventlist == fNFriends) {
153
154 l = new TList;
155 l->SetOwner();
156 fList->Add(l);
157 ++fNFriends;
158
159 } else if (eventlist < 0 || eventlist > fNFriends) {
160 Error("AddFile", "Specified list %d not in [0,%d]", eventlist, fNFriends);
161 return;
162 }
163
164 if (!IsValidName(pname)) return;
165
166 l->Add(new TObjString(pname));
167
168 MDB(kAnalysis, 2)
169 Info("AddFile", "Added %s to list of files.", pname);
170 }
171
172 //--------------------------------------------------------------------------------------------------
173 void Analysis::AddFile(const TObject *oname, Int_t eventlist)
174 {
175 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
176 // different types of events. In case you dont want mixing, just give no eventlist argument
177 // (default 0).
178
179 MitAssert("AddFile", oname != 0);
180
181 return AddFile(oname->GetName(), eventlist);
182 }
183
184 //________________________________________________________________________
185 Bool_t Analysis::AddFiles(const char *pname, Int_t nmax)
186 {
187 // Add files from text file with given name. If nmax>0, maximum nmax files will be added.
188
189 MitAssert("AddFiles", pname != 0);
190
191 ifstream in;
192 in.open(pname);
193 if (!in) {
194 Error("AddFiles", "Cannot open file with name %s", pname);
195 return kFALSE;
196 }
197
198 Int_t fc = 0;
199 while (in) {
200 TString line;
201 line.ReadLine(in);
202 cout << line << endl;
203 if (!line.EndsWith(".root"))
204 continue;
205 cout << line << endl;
206
207 if (!AddFile(line)) {
208 Error("AddFiles", "Error adding file with name %s", line.Data());
209 return kFALSE;
210 }
211
212 ++fc;
213 if (nmax>0 && fc>=nmax) {
214 Info("AddFiles", "Maximal number (%d) of files added", nmax);
215 break;
216 }
217 }
218
219 return kTRUE;
220 }
221
222 //--------------------------------------------------------------------------------------------------
223 void Analysis::AddList(TList *list, Int_t eventlist)
224 {
225 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
226 // different types of events. In case you dont want mixing, just give no eventlist argument
227 // (default 0).
228
229 MitAssert("AddList", list != 0);
230
231 TIter next(list);
232 while (TObject *obj = next())
233 AddFile(obj->GetName(), eventlist);
234 }
235
236 //--------------------------------------------------------------------------------------------------
237 void Analysis::AddLoader(TAMVirtualLoader *l)
238 {
239 // Add loader to the list of loaders.
240
241 fLoaders->Add(l);
242 }
243
244 //--------------------------------------------------------------------------------------------------
245 void Analysis::AddPackage(const char* name)
246 {
247 // Add package to the list of uploaded packages.
248
249 MitAssert("AddPackage", name != 0);
250 fPackages->Add(new TObjString(name));
251 }
252
253 //--------------------------------------------------------------------------------------------------
254 void Analysis::AddPackages(TList *list)
255 {
256 // Add list of packages to the list of uploaded packages.
257
258 MitAssert("AddPackage", list != 0);
259
260 TIter next(list);
261 while (TObject *obj = next()) {
262 fPackages->Add(new TObjString(obj->GetName()));
263 }
264 }
265
266 //--------------------------------------------------------------------------------------------------
267 void Analysis::AddSuperModule(TAModule *mod)
268 {
269 // Add a top-level module to list of top-level (super) modules.
270
271 fSuperMods->Add(mod);
272 }
273
274 //--------------------------------------------------------------------------------------------------
275 void Analysis::FileInputFromEnv()
276 {
277 // Attempt to get list of filesets/files from environment.
278
279 TString catalog(gSystem->Getenv("MIT_CATALOG"));
280 TString book(gSystem->Getenv("MIT_BOOK"));
281 TString dataset(gSystem->Getenv("MIT_DATASET"));
282 TString filesets(gSystem->Getenv("MIT_FILESETS"));
283 TString files(gSystem->Getenv("MIT_FILES"));
284
285 if ((catalog.IsNull() || book.IsNull() || dataset.IsNull()) && files.IsNull()) {
286 Warning("FileInputFromEnv", "Called to get file info from environment, but did not get"
287 " consistent set of variables:\n\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\t"
288 "MIT_DATASET=%s\n\tMIT_FILESETS=%s\n\tMIT_FILES=%s\n",
289 catalog.Data(), book.Data(), dataset.Data(), filesets.Data(), files.Data());
290 return;
291 }
292
293 if (!files.IsNull()) { // add local files
294 Info("FileInputFromEnv", "Got from environment:\n"
295 "\n\tMIT_FILES=%s\n", files.Data());
296 TString tok(";");
297 TObjArray *arr = files.Tokenize(tok);
298 if (arr) {
299 for (Int_t i=0; i<arr->GetEntries(); ++i) {
300 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
301 if (!dummy) continue;
302 AddFile(dummy->GetName(),0);
303 }
304 delete arr;
305 }
306 return;
307 }
308
309 Info("FileInputFromEnv", "Got from environment:\n"
310 "\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\tMIT_DATASET=%s\n\tMIT_FILESETS=%s\n",
311 catalog.Data(), book.Data(), dataset.Data(), filesets.Data());
312
313 Catalog cat(catalog);
314 if (filesets.IsNull()) {
315 Dataset *d = cat.FindDataset(book, dataset);
316 if (d) {
317 AddDataset(d);
318 delete d;
319 }
320 } else {
321 TString tok(";");
322 TObjArray *arr = filesets.Tokenize(tok);
323 if (arr) {
324 for (Int_t i=0; i<arr->GetEntries(); ++i) {
325 TObjString *fileset = dynamic_cast<TObjString*>(arr->At(i));
326 if (!fileset) continue;
327 Dataset *d = cat.FindDataset(book, dataset, fileset->String());
328 if (!d)
329 continue;
330 AddDataset(d);
331 delete d;
332 }
333 delete arr;
334 }
335 }
336 }
337
338 //--------------------------------------------------------------------------------------------------
339 Bool_t Analysis::Init()
340 {
341 // Setup the TDSet and TChain to be used for the analysis with or without PROOF. If more than one
342 // list of file names was given, friend trees are supported.
343
344 if (fState == kRun || fState == kInit) {
345 Error("Init", "Init in state %d is not possible! Call Terminate() first.",
346 Int_t(fState));
347 return kFALSE;
348 }
349
350 // check if we should attempt to get filesets/filenames from environment
351 if (fNFriends == 0)
352 FileInputFromEnv();
353
354 if (fNFriends <= 0) {
355 Error("Init", "List of friend lists is empty!");
356 return kFALSE;
357 }
358
359 if (!fSuperMods->First()) {
360 Error("Init", "Top-level TAM module is NULL!");
361 return kFALSE;
362 }
363
364 if (fUseProof) { // first init our PROOF session
365 if (!InitProof())
366 return kFALSE;
367 }
368
369 // we do this here instead in Terminate() so that
370 // we can browse the output even after Terminate()
371 delete fSelector;
372 fSelector = 0;
373
374 fChain = new TChain(fTreeName);
375 fSet = new TDSet("TTree",fTreeName);
376
377 for (Int_t i=0; i<fNFriends; ++i) {
378 TList *l = dynamic_cast<TList*>(fList->At(i));
379 if (!l) {
380 Fatal("Init", "List %d not found!", i);
381 return kFALSE;
382 }
383
384 if (i == 0) {
385 TIter next(l);
386 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
387 fChain->Add(obj->GetName());
388 fSet->Add(obj->GetName());
389 if (fCacheSize<0 && obj->GetString().Contains("/castor/"))
390 fCacheSize = 64*1024*1024;
391 }
392 } else {
393 TChain *chain = new TChain(fTreeName);
394 TDSet *set = new TDSet("TTree",fTreeName);
395
396 TIter next(l);
397 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
398 chain->Add(obj->GetName());
399 set->Add(obj->GetName());
400 if (fCacheSize<0 && obj->GetString().Contains("/castor/"))
401 fCacheSize = 64*1024*1024;
402 }
403
404 TString alias("TAMTREE_"); // aliases currently not used
405 alias+=i;
406
407 fChain->AddFriend(chain,alias.Data());
408 fSet->AddFriend(set,alias.Data());
409
410 fDeleteList->Add(chain);
411 fDeleteList->Add(set);
412 }
413 }
414
415 if (fCacheSize>=0)
416 fChain->SetCacheSize(fCacheSize);
417
418 // create our customized loader plugin for TAM
419 TreeLoader *bl = new TreeLoader;
420 fLoaders->Add(bl);
421 fDeleteList->Add(bl);
422
423 // create our ana framework module
424 AnaFwkMod *anamod = new AnaFwkMod;
425 anamod->SetSkipNEvents(fSkipNEvents);
426 fDeleteList->Add(anamod);
427
428 // create our HLT framework module
429 HLTFwkMod *hltmod = 0;
430 if (fUseHLT) {
431 hltmod = new HLTFwkMod;
432 fDeleteList->Add(hltmod);
433 }
434
435 if (fUseProof) {
436
437 fProof->AddInput(anamod);
438 if (hltmod)
439 fProof->AddInput(hltmod);
440
441 TIter iter(fSuperMods->MakeIterator());
442 while (1) {
443 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
444 if (!next) break;
445 fProof->AddInput(next);
446 }
447
448 fLoaders->SetName("TAM_LOADERS");
449 fProof->AddInput(fLoaders);
450
451 } else {
452
453 // when not running Proof, we must make a selector
454 fSelector = new Selector;
455 fSelector->SetDoProxy(fDoProxy);
456 fSelector->SetDoObjTabClean(fDoObjTabClean);
457
458 fSelector->AddInput(anamod);
459
460 if (hltmod)
461 fSelector->AddInput(hltmod);
462
463 TIter iter(fSuperMods->MakeIterator());
464 while (1) {
465 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
466 if (!next) break;
467 fSelector->AddInput(next);
468 }
469
470 MDB(kAnalysis, 2)
471 fSelector->SetVerbosity(1);
472
473 // pass loaders to selector
474 TIter next(fLoaders);
475 while (TAMVirtualLoader *l = dynamic_cast<TAMVirtualLoader*>(next()))
476 fSelector->AddLoader(l);
477 }
478
479 fState = kInit;
480 return kTRUE;
481 }
482
483 //--------------------------------------------------------------------------------------------------
484 Bool_t Analysis::InitProof()
485 {
486 // Initialize PROOF connection.
487
488 if (fProof && fProof->IsValid())
489 return kTRUE;
490
491 delete fProof;
492
493 if (fMaster.Contains("rcf.bnl.gov")) {
494 for (Int_t i=0; i<5; ++i) {
495 Warning("InitProof", "*** DID YOU RUN PROOF_KINIT? %d (5) ***", i);
496 gSystem->Sleep(1000);
497 }
498 }
499
500 MDB(kAnalysis, 1)
501 Info("InitProof", "Starting PROOF on master %s with config %s",
502 fMaster.Data(), fConfig.Data());
503
504 fProof = dynamic_cast<TProof*>(TProof::Open(fMaster, fConfig));
505 if (!fProof) {
506 Error("InitProof", "Could not start PROOF!");
507 return kFALSE;
508 }
509
510 MDB(kAnalysis, 3)
511 gROOT->Print();
512
513 //fProof->AddInput(new TNamed("PROOF_NewPacketizer",""));
514
515 Bool_t ret=kTRUE;
516 if (fPackages) {
517 // tell Proof what additional libraries we will need on each worker computer
518 ret = UploadPackages(fPackages);
519 }
520
521 return ret;
522 }
523
524 //--------------------------------------------------------------------------------------------------
525 void Analysis::Run()
526 {
527 // Run the analysis on the created file set.
528
529 if (fState == kPristine || fState == kRun) {
530 Error("Run", "Run in state %d is not possible! Call Init() first.",
531 Int_t(fState));
532 }
533
534 if (fUseProof) {
535
536 MDB(kAnalysis, 1)
537 Info("Run", "Start processing with PROOF...");
538
539 fSet->Process("Selector","",fDoNEvents);
540
541 } else {
542
543 MDB(kAnalysis, 1)
544 Info("Run", "Start processing (no PROOF)...");
545
546 fChain->Process(fSelector,"",fDoNEvents);
547 }
548
549 MDB(kAnalysis, 1)
550 Info("Run", "Processing complete!");
551
552 fState = kRun;
553 }
554
555 //--------------------------------------------------------------------------------------------------
556 Bool_t Analysis::Run(Bool_t browse)
557 {
558 // Execute analysis and open TBrowser if requested.
559
560 if (Init()) {
561 Run();
562 Terminate();
563 if (browse) {
564 new TBrowser;
565 }
566 return kTRUE;
567 }
568
569 Error("Execute", "Could not initialize analysis.");
570 return kFALSE;
571 }
572
573 //--------------------------------------------------------------------------------------------------
574 void Analysis::SetSuperModule(TAModule *mod)
575 {
576 // Set the first top-level module in the list of top-level (super) modules.
577
578 fSuperMods->AddFirst(mod);
579 }
580
581 //--------------------------------------------------------------------------------------------------
582 void Analysis::Terminate()
583 {
584 // Terminate current analysis run.
585
586 if (fState == kPristine || fState == kTerminate) {
587 Error("Terminate", "Terminate in state %d is not possible! Call Init() first.",
588 Int_t(fState));
589 return;
590 }
591
592 if (fState == kRun) {
593
594 if (fUseProof) {
595 // the output list from Proof can (in principal) contain other objects
596 // besides the module output hierarchy.
597 TList* outputlist = fProof->GetOutputList();
598 TIter nextOut(outputlist);
599 while (TObject *obj = nextOut()) {
600 if (obj->InheritsFrom(TAMOutput::Class())) {
601 fOutput = dynamic_cast<TList*>(obj);
602 break;
603 }
604 }
605
606 } else {
607 fOutput = fSelector->GetModOutput();
608 }
609
610
611
612 if (fOutput && !fAnaOutput.IsNull()) {
613 TDirectory::TContext context(0); // automatically restore gDirectory
614
615 std::auto_ptr<TFile> outf(TFile::Open(fAnaOutput,"recreate","", fCompLevel));
616 if (outf.get() == 0) {
617 Error("Terminate", "Could not open file %s for output!", fAnaOutput.Data());
618 } else {
619 MDB(kAnalysis, 1)
620 Info("Terminate", "Saving output to %s!", fAnaOutput.Data());
621
622 if (fHierarchy)
623 fOutput->Write(0,-99);
624 else
625 fOutput->Write();
626 }
627 }
628 // set state to terminate
629 fState = kTerminate;
630 }
631
632 delete fChain;
633 delete fSet;
634 fDeleteList->Delete();
635 }
636
637 //--------------------------------------------------------------------------------------------------
638 Bool_t Analysis::UploadPackages(TList *packages)
639 {
640 // Upload list of par files to the server.
641
642 MitAssert("UploadPackages", packages != 0);
643
644 for (Int_t i=0; i<packages->GetEntries(); ++i) {
645
646 TObject* objstr = packages->At(i);
647 if (!objstr) {
648 Error("InitProof", "Problem at package number %d!", i);
649 return kFALSE;
650 }
651
652 TString packname = objstr->GetName();
653 Int_t en = 0;
654 if (packname.EndsWith("+")) {
655 en=1;
656 packname.Resize(packname.Length()-1);
657 }
658
659 ifstream ftest(gSystem->ExpandPathName(packname.Data()),ios_base::binary);
660 if (!ftest.good()) {
661 Error("InitProof", "Could not open %s for upload!", packname.Data());
662 return kFALSE;
663 }
664
665 if (fProof->UploadPackage(packname)<0) {
666 Error("UploadPackage", "Upload for %s failed!", packname.Data());
667 return kFALSE;
668 }
669
670 if (en == 1) {
671 Int_t pos=packname.Last('/')+1;
672 if (pos)
673 packname.Remove(0,pos);
674 if (fProof->EnablePackage(packname)<0) {
675 Error("UploadPackage", "Enabling for %s failed!", packname.Data());
676 return kFALSE;
677 }
678 }
679 }
680
681 return kTRUE;
682 }