ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/MitAna/TreeMod/src/Analysis.cc
Revision: 1.24
Committed: Fri Dec 12 16:57:42 2008 UTC (16 years, 4 months ago) by bendavid
Content type: text/plain
Branch: MAIN
Changes since 1.23: +5 -2 lines
Log Message:
Added read caching for the events tree, enabled by default set to 128mb

File Contents

# Content
1 // $Id: Analysis.cc,v 1.23 2008/12/11 15:53:26 loizides Exp $
2
3 #include "MitAna/TreeMod/interface/Analysis.h"
4 #include <Riostream.h>
5 #include <TFile.h>
6 #include <TList.h>
7 #include <TDSet.h>
8 #include <TChain.h>
9 #include <TObjString.h>
10 #include <TSystem.h>
11 #include <TProof.h>
12 #include <TROOT.h>
13 #include <TBrowser.h>
14 #include "MitAna/DataUtil/interface/Debug.h"
15 #include "MitAna/DataTree/interface/Names.h"
16 #include "MitAna/TAM/interface/TAMVirtualLoader.h"
17 #include "MitAna/TAM/interface/TAModule.h"
18 #include "MitAna/TreeMod/interface/Selector.h"
19 #include "MitAna/TreeMod/interface/TreeLoader.h"
20 #include "MitAna/TreeMod/interface/AnaFwkMod.h"
21 #include "MitAna/TreeMod/interface/HLTFwkMod.h"
22 #include "MitAna/Catalog/interface/Catalog.h"
23 #include "MitAna/Catalog/interface/Dataset.h"
24
25 ClassImp(mithep::Analysis)
26
27 using namespace mithep;
28
29 //--------------------------------------------------------------------------------------------------
30 Analysis::Analysis(Bool_t useproof) :
31 fUseProof(useproof),
32 fUseHLT(kTRUE),
33 fHierarchy(kTRUE),
34 fDoProxy(kTRUE),
35 fState(kPristine),
36 fNFriends(0),
37 fList(new TList),
38 fOutput(0),
39 fPackages(new TList),
40 fLoaders(new TList),
41 fSuperMods(new TList),
42 fSelector(0),
43 fChain(0),
44 fSet(0),
45 fDeleteList(new TList),
46 fTreeName(Names::gkEvtTreeName),
47 fCompLevel(2),
48 fProof(0),
49 fDoNEvents(TChain::kBigNumber),
50 fCacheSize(134217728)
51 {
52 // Default constructor.
53
54 fList->SetOwner();
55 fDeleteList->SetOwner();
56
57 /* default packages for PROOF upload */
58 fPackages->SetOwner();
59 // nothing to be done since we do not use par files (yet?)
60 }
61
62 //--------------------------------------------------------------------------------------------------
63 Analysis::~Analysis()
64 {
65 // Destructor.
66
67 if (fState == kInit || fState == kRun)
68 Terminate();
69
70 delete fList;
71 delete fPackages;
72 delete fLoaders;
73 delete fDeleteList;
74 delete fSelector;
75 delete fSuperMods;
76 fOutput = 0; // owned by TAM
77
78 delete fProof;
79 }
80
81 //--------------------------------------------------------------------------------------------------
82 Bool_t Analysis::AddDataset(const Dataset *dataset)
83 {
84 // Add a full dataset to the analysis.
85
86 Bool_t status = true;
87
88 for (UInt_t i=0; i<dataset->NFiles(); ++i)
89 status = (status && AddFile(dataset->FileUrl(i)));
90
91 return status;
92 }
93
94 //--------------------------------------------------------------------------------------------------
95 Bool_t Analysis::AddFile(const char *pname)
96 {
97 // Add file with given name to the list of files to be processed. Using the token "|", you can
98 // specify an arbritray number of paths to tree files that will be concatenated as friend trees.
99
100 if (fState != kPristine) {
101 Error("AddFile", "Analysis already initialized");
102 return kFALSE;
103 }
104
105 TString pnamestr(pname);
106 if (pnamestr.IsNull())
107 return kFALSE;
108
109 TString tok("|");
110 TObjArray *arr = pnamestr.Tokenize(tok);
111 if (!arr)
112 return kFALSE;
113
114 TString msg;
115 for (Int_t i=0; i<arr->GetEntries(); ++i) {
116
117 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
118 if (!dummy)
119 continue;
120
121 AddFile(dummy->GetName(),i);
122 if (i==0)
123 msg=dummy->GetName();
124 else
125 Info("AddFile", "Add file %s as friend to %s", dummy->GetName(), msg.Data());
126 }
127 delete arr;
128
129 return kTRUE;
130 }
131
132 //--------------------------------------------------------------------------------------------------
133 void Analysis::AddFile(const char *pname, Int_t eventlist)
134 {
135 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
136 // different types of events. In case you dont want friend trees, just give no eventlist argument
137 // (default 0).
138
139 MitAssert("AddFile", pname != 0);
140
141 TList *l = 0;
142 if (eventlist >= 0 && eventlist < fNFriends) {
143
144 l = dynamic_cast<TList*>(fList->At(eventlist));
145 if (!l) {
146 Fatal("AddFile", "Requested list %d not found!", eventlist);
147 return;
148 }
149
150 } else if (eventlist == fNFriends) {
151
152 l = new TList;
153 l->SetOwner();
154 fList->Add(l);
155 ++fNFriends;
156
157 } else if (eventlist < 0 || eventlist > fNFriends) {
158 Error("AddFile", "Specified list %d not in [0,%d]", eventlist, fNFriends);
159 return;
160 }
161
162 if (!IsValidName(pname)) return;
163
164 l->Add(new TObjString(pname));
165
166 MDB(kAnalysis, 2)
167 Info("AddFile", "Added %s to list of files.", pname);
168 }
169
170 //--------------------------------------------------------------------------------------------------
171 void Analysis::AddFile(const TObject *oname, Int_t eventlist)
172 {
173 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
174 // different types of events. In case you dont want mixing, just give no eventlist argument
175 // (default 0).
176
177 MitAssert("AddFile", oname != 0);
178
179 return AddFile(oname->GetName(), eventlist);
180 }
181
182 //________________________________________________________________________
183 Bool_t Analysis::AddFiles(const char *pname, Int_t nmax)
184 {
185 // Add files from text file with given name. If nmax>0, maximum nmax files will be added.
186
187 MitAssert("AddFiles", pname != 0);
188
189 ifstream in;
190 in.open(pname);
191 if (!in) {
192 Error("AddFiles", "Can not open file with name %s", pname);
193 return kFALSE;
194 }
195
196 Int_t fc = 0;
197 while (in) {
198 TString line;
199 line.ReadLine(in);
200 cout << line << endl;
201 if (!line.EndsWith(".root"))
202 continue;
203 cout << line << endl;
204
205 if (!AddFile(line)) {
206 Error("AddFiles", "Error adding file with name %s", line.Data());
207 return kFALSE;
208 }
209
210 ++fc;
211 if (nmax>0 && fc>=nmax) {
212 Info("AddFiles", "Maximal number (%d) of files added", nmax);
213 break;
214 }
215 }
216
217 return kTRUE;
218 }
219
220 //--------------------------------------------------------------------------------------------------
221 void Analysis::AddList(TList *list, Int_t eventlist)
222 {
223 // Add file name to the event list specified by eventlist. The lists are used to hold filenames of
224 // different types of events. In case you dont want mixing, just give no eventlist argument
225 // (default 0).
226
227 MitAssert("AddList", list != 0);
228
229 TIter next(list);
230 while (TObject *obj = next())
231 AddFile(obj->GetName(), eventlist);
232 }
233
234 //--------------------------------------------------------------------------------------------------
235 void Analysis::AddLoader(TAMVirtualLoader *l)
236 {
237 // Add loader to the list of loaders.
238
239 fLoaders->Add(l);
240 }
241
242 //--------------------------------------------------------------------------------------------------
243 void Analysis::AddPackage(const char* name)
244 {
245 // Add package to the list of uploaded packages.
246
247 MitAssert("AddPackage", name != 0);
248 fPackages->Add(new TObjString(name));
249 }
250
251 //--------------------------------------------------------------------------------------------------
252 void Analysis::AddPackages(TList *list)
253 {
254 // Add list of packages to the list of uploaded packages.
255
256 MitAssert("AddPackage", list != 0);
257
258 TIter next(list);
259 while (TObject *obj = next()) {
260 fPackages->Add(new TObjString(obj->GetName()));
261 }
262 }
263
264 //--------------------------------------------------------------------------------------------------
265 void Analysis::AddSuperModule(TAModule *mod)
266 {
267 // Add a top-level module to list of top-level (super) modules.
268
269 fSuperMods->Add(mod);
270 }
271
272 //--------------------------------------------------------------------------------------------------
273 void Analysis::FileInputFromEnv()
274 {
275 // Attempt to get list of filesets/files from environment.
276
277 TString catalog(gSystem->Getenv("MIT_CATALOG"));
278 TString book(gSystem->Getenv("MIT_BOOK"));
279 TString dataset(gSystem->Getenv("MIT_DATASET"));
280 TString filesets(gSystem->Getenv("MIT_FILESETS"));
281 TString files(gSystem->Getenv("MIT_FILES"));
282
283 if ((catalog.IsNull() || book.IsNull() || dataset.IsNull()) && files.IsNull()) {
284 Warning("FileInputFromEnv", "Called to get file info from environment, but did not get"
285 " consistent set of variables:\n\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\t"
286 "MIT_DATASET=%s\n\tMIT_FILESETS=%s\n\tMIT_FILES=%s\n",
287 catalog.Data(), book.Data(), dataset.Data(), filesets.Data(), files.Data());
288 return;
289 }
290
291
292 if (!files.IsNull()) { // add local files
293 Info("FileInputFromEnv", "Got from environment:\n"
294 "\n\tMIT_FILES=%s\n", files.Data());
295 TString tok(";");
296 TObjArray *arr = files.Tokenize(tok);
297 if (arr) {
298 for (Int_t i=0; i<arr->GetEntries(); ++i) {
299 TObjString *dummy = dynamic_cast<TObjString*>(arr->At(i));
300 if (!dummy) continue;
301 AddFile(dummy->GetName(),0);
302 }
303 delete arr;
304 }
305 return;
306 }
307
308 Info("FileInputFromEnv", "Got from environment:\n"
309 "\tMIT_CATALOG=%s\n\tMIT_BOOK=%s\n\tMIT_DATASET=%s\n\tMIT_FILESETS=%s\n",
310 catalog.Data(), book.Data(), dataset.Data(), filesets.Data());
311
312 Catalog cat(catalog);
313 TString tok(";");
314 TObjArray *arr = filesets.Tokenize(tok);
315 if (arr) {
316 for (Int_t i=0; i<arr->GetEntries(); ++i) {
317 TObjString *fileset = dynamic_cast<TObjString*>(arr->At(i));
318 if (!fileset) continue;
319 Dataset *d = cat.FindDataset(book, dataset, fileset->String());
320 if (!d)
321 continue;
322 AddDataset(d);
323 delete d;
324 }
325 delete arr;
326 }
327 }
328
329 //--------------------------------------------------------------------------------------------------
330 Bool_t Analysis::Init()
331 {
332 // Setup the TDSet and TChain to be used for the analysis with or without PROOF. If more than one
333 // list of file names was given, friend trees are supported.
334
335 if (fState == kRun || fState == kInit) {
336 Error("Init", "Init in state %d is not possible! Call Terminate() first.",
337 Int_t(fState));
338 return kFALSE;
339 }
340
341 // check if we should attempt to get filesets/filenames from environment
342 if (fNFriends == 0)
343 FileInputFromEnv();
344
345 if (fNFriends <= 0) {
346 Error("Init", "List of friend lists is empty!");
347 return kFALSE;
348 }
349
350 if (!fSuperMods->First()) {
351 Error("Init", "Top-level TAM module is NULL!");
352 return kFALSE;
353 }
354
355 if (fUseProof) { // first init our PROOF session
356 if (!InitProof())
357 return kFALSE;
358 }
359
360 // we do this here instead in Terminate() so that
361 // we can browse the output even after Terminate()
362 delete fSelector;
363 fSelector = 0;
364
365 fChain = new TChain(fTreeName);
366 fSet = new TDSet("TTree",fTreeName);
367
368 fChain->SetCacheSize(fCacheSize);
369
370 for (Int_t i=0; i<fNFriends; ++i) {
371
372 TList *l = dynamic_cast<TList*>(fList->At(i));
373 if (!l) {
374 Fatal("Init", "List %d not found!", i);
375 return kFALSE;
376 }
377
378 if (i == 0) {
379 TIter next(l);
380 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
381 fChain->Add(obj->GetName());
382 fSet->Add(obj->GetName());
383 }
384
385 } else {
386
387 TChain *chain = new TChain(fTreeName);
388 TDSet *set = new TDSet("TTree",fTreeName);
389
390 TIter next(l);
391 while (TObjString *obj = dynamic_cast<TObjString*>(next())) {
392 chain->Add(obj->GetName());
393 set->Add(obj->GetName());
394 }
395
396 TString alias("TAMTREE_"); // aliases currently not used
397 alias+=i;
398
399 fChain->AddFriend(chain,alias.Data());
400 fSet->AddFriend(set,alias.Data());
401
402 fDeleteList->Add(chain);
403 fDeleteList->Add(set);
404 }
405
406 }
407
408 // create our customized loader plugin for TAM
409 TreeLoader *bl = new TreeLoader;
410 fLoaders->Add(bl);
411 fDeleteList->Add(bl);
412
413 // create our ana framework module
414 AnaFwkMod *anamod = new AnaFwkMod;
415 fDeleteList->Add(anamod);
416
417 // create our HLT framework module
418 HLTFwkMod *hltmod = 0;
419 if (fUseHLT) {
420 hltmod = new HLTFwkMod;
421 fDeleteList->Add(hltmod);
422 }
423
424 if (fUseProof) {
425
426 fProof->AddInput(anamod);
427 if (hltmod)
428 fProof->AddInput(hltmod);
429
430 TIter iter(fSuperMods->MakeIterator());
431 while (1) {
432 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
433 if (!next) break;
434 fProof->AddInput(next);
435 }
436
437 fLoaders->SetName("TAM_LOADERS");
438 fProof->AddInput(fLoaders);
439
440 } else {
441
442 // when not running Proof, we must make a selector
443 fSelector = new Selector;
444 fSelector->SetDoProxy(fDoProxy);
445
446 fSelector->AddInput(anamod);
447
448 if (hltmod)
449 fSelector->AddInput(hltmod);
450
451 TIter iter(fSuperMods->MakeIterator());
452 while (1) {
453 TAModule *next = dynamic_cast<TAModule*>(iter.Next());
454 if (!next) break;
455 fSelector->AddInput(next);
456 }
457
458 MDB(kAnalysis, 2)
459 fSelector->SetVerbosity(1);
460
461 // pass loaders to selector
462 TIter next(fLoaders);
463 while (TAMVirtualLoader *l = dynamic_cast<TAMVirtualLoader*>(next()))
464 fSelector->AddLoader(l);
465 }
466
467 fState = kInit;
468 return kTRUE;
469 }
470
471 //--------------------------------------------------------------------------------------------------
472 Bool_t Analysis::InitProof()
473 {
474 // Initialize PROOF connection.
475
476 if (fProof && fProof->IsValid())
477 return kTRUE;
478
479 delete fProof;
480
481 if (fMaster.Contains("rcf.bnl.gov")) {
482 for (Int_t i=0; i<5; ++i) {
483 Warning("InitProof", "*** DID YOU RUN PROOF_KINIT? %d (5) ***", i);
484 gSystem->Sleep(1000);
485 }
486 }
487
488 MDB(kAnalysis, 1)
489 Info("InitProof", "Starting PROOF on master %s with config %s",
490 fMaster.Data(), fConfig.Data());
491
492 fProof = dynamic_cast<TProof*>(TProof::Open(fMaster, fConfig));
493 if (!fProof) {
494 Error("InitProof", "Could not start PROOF!");
495 return kFALSE;
496 }
497
498 MDB(kAnalysis, 3)
499 gROOT->Print();
500
501 //fProof->AddInput(new TNamed("PROOF_NewPacketizer",""));
502
503 Bool_t ret=kTRUE;
504 if (fPackages) {
505 // tell Proof what additional libraries we will need on each worker computer
506 ret = UploadPackages(fPackages);
507 }
508
509 return ret;
510 }
511
512 //--------------------------------------------------------------------------------------------------
513 void Analysis::Run()
514 {
515 // Run the analysis on the created file set.
516
517 if (fState == kPristine || fState == kRun) {
518 Error("Run", "Run in state %d is not possible! Call Init() first.",
519 Int_t(fState));
520 }
521
522 if (fUseProof) {
523
524 MDB(kAnalysis, 1)
525 Info("Run", "Start processing with PROOF...");
526
527 fSet->Process("Selector","",fDoNEvents);
528
529 } else {
530
531 MDB(kAnalysis, 1)
532 Info("Run", "Start processing (no PROOF)...");
533
534 fChain->Process(fSelector,"",fDoNEvents);
535 }
536
537 MDB(kAnalysis, 1)
538 Info("Run", "Processing complete!");
539
540 fState = kRun;
541 }
542
543 //--------------------------------------------------------------------------------------------------
544 Bool_t Analysis::Run(Bool_t browse)
545 {
546 // Execute analysis and open TBrowser if requested.
547
548 if (Init()) {
549 Run();
550 Terminate();
551 if (browse) {
552 new TBrowser;
553 }
554 return kTRUE;
555 }
556
557 Error("Execute", "Could not initialize analysis.");
558 return kFALSE;
559 }
560
561 //--------------------------------------------------------------------------------------------------
562 void Analysis::SetSuperModule(TAModule *mod)
563 {
564 // Set the first top-level module in the list of top-level (super) modules.
565
566 fSuperMods->AddFirst(mod);
567 }
568
569 //--------------------------------------------------------------------------------------------------
570 void Analysis::Terminate()
571 {
572 // Terminate current analysis run.
573
574 if (fState == kPristine || fState == kTerminate) {
575 Error("Terminate", "Terminate in state %d is not possible! Call Init() first.",
576 Int_t(fState));
577 return;
578 }
579
580 if (fState == kRun) {
581
582 if (fUseProof) {
583 // the output list from Proof can (in principal) contain other objects
584 // besides the module output hierarchy.
585 TList* outputlist = fProof->GetOutputList();
586 TIter nextOut(outputlist);
587 while (TObject *obj = nextOut()) {
588 if (obj->InheritsFrom(TAMOutput::Class())) {
589 fOutput = dynamic_cast<TList*>(obj);
590 break;
591 }
592 }
593
594 } else {
595 fOutput = fSelector->GetModOutput();
596 }
597
598
599
600 if (fOutput && !fAnaOutput.IsNull()) {
601 TDirectory::TContext context(0); // automatically restore gDirectory
602
603 std::auto_ptr<TFile> outf(TFile::Open(fAnaOutput,"recreate","", fCompLevel));
604 if (outf.get() == 0) {
605 Error("Terminate", "Could not open file %s for output!", fAnaOutput.Data());
606 } else {
607 MDB(kAnalysis, 1)
608 Info("Terminate", "Saving output to %s!", fAnaOutput.Data());
609
610 if (fHierarchy)
611 fOutput->Write(0,-99);
612 else
613 fOutput->Write();
614 }
615 }
616 // set state to terminate
617 fState = kTerminate;
618 }
619
620 delete fChain;
621 delete fSet;
622 fDeleteList->Delete();
623 }
624
625 //--------------------------------------------------------------------------------------------------
626 Bool_t Analysis::UploadPackages(TList *packages)
627 {
628 // Upload list of par files to the server.
629
630 MitAssert("UploadPackages", packages != 0);
631
632 for (Int_t i=0; i<packages->GetEntries(); ++i) {
633
634 TObject* objstr = packages->At(i);
635 if (!objstr) {
636 Error("InitProof", "Problem at package number %d!", i);
637 return kFALSE;
638 }
639
640 TString packname = objstr->GetName();
641 Int_t en = 0;
642 if (packname.EndsWith("+")) {
643 en=1;
644 packname.Resize(packname.Length()-1);
645 }
646
647 ifstream ftest(gSystem->ExpandPathName(packname.Data()),ios_base::binary);
648 if (!ftest.good()) {
649 Error("InitProof", "Could not open %s for upload!", packname.Data());
650 return kFALSE;
651 }
652
653 if (fProof->UploadPackage(packname)<0) {
654 Error("UploadPackage", "Upload for %s failed!", packname.Data());
655 return kFALSE;
656 }
657
658 if (en == 1) {
659 Int_t pos=packname.Last('/')+1;
660 if (pos)
661 packname.Remove(0,pos);
662 if (fProof->EnablePackage(packname)<0) {
663 Error("UploadPackage", "Enabling for %s failed!", packname.Data());
664 return kFALSE;
665 }
666 }
667 }
668
669 return kTRUE;
670 }