ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/AnaTools/scripts/mergeHists
Revision: 1.27
Committed: Tue Jun 18 02:00:39 2013 UTC (11 years, 10 months ago) by ahart
Branch: MAIN
CVS Tags: V02-03-01, V02-03-00
Changes since 1.26: +15 -2 lines
Log Message:
Exclude EDM format ROOT files when merging with mergeTFileServiceHistograms.

File Contents

# User Rev Content
1 ahart 1.1 #!/usr/bin/env perl
2    
3     use strict;
4     use Getopt::Long;
5 ahart 1.3 use POSIX;
6    
7     sub processArgs;
8     sub printHelp;
9     sub getRunList;
10     sub countEvents;
11 ahart 1.27 sub isEDM;
12 ahart 1.1
13     my %opt;
14     Getopt::Long::Configure ("bundling");
15 ahart 1.6 GetOptions (\%opt, "cutflow|c=s", "luminosity|l=s", "prefix|p=s", "weight|w=s", "xsection|x=s", "help|h");
16 ahart 1.1
17     printHelp () if $opt{"help"} || !$opt{"prefix"};
18     my $files = processArgs (\@ARGV);
19 ahart 1.11 my %rootFiles;
20     my %weights;
21 ahart 1.13 my @weights;
22 ahart 1.5 my $nGoodJobs = 0;
23     my $nBadJobs = 0;
24 ahart 1.17 my $nIncompleteJobs = 0;
25 ahart 1.3 my $counting = 0;
26 ahart 1.4 my %exitCodes;
27 ahart 1.8 my %signals;
28 ahart 1.17 my %partial;
29 ahart 1.5 my %crossSections;
30 ahart 1.14 my %dirs;
31 ahart 1.5 my $integratedLuminosity = 10000;
32     $integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
33 ahart 1.6 my $cutFlow = "cutFlow";
34     $cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
35 ahart 1.3 foreach my $file (@$files)
36     {
37     next if $file eq ".";
38     next if $file eq "..";
39 ahart 1.5 my $dir = $file;
40     $dir =~ s/^(.*)\/[^\/]*$/$1/;
41 ahart 1.4 if ($file =~ m/^.*\/condor_[^_]*\.log$/)
42 ahart 1.3 {
43     my $jobNumber = $file;
44     $jobNumber =~ s/^.*\/condor_([^_]*)\.log$/$1/;
45     open (FILE, "<$file");
46     my @fileContents = <FILE>;
47     close (FILE);
48     my $fileContents = join ("", @fileContents);
49     $fileContents =~ s/\n/ /g;
50 ahart 1.19 $counting = 1;
51 ahart 1.8 if ($fileContents =~ m/return value/)
52 ahart 1.3 {
53 ahart 1.8 $fileContents =~ s/.*\(return value ([^)]*)\).*/$1/g;
54 ahart 1.9 $nGoodJobs++;
55     print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
56 ahart 1.8 $exitCodes{$dir}{$jobNumber} = $fileContents;
57     }
58 ahart 1.17 elsif ($fileContents =~ m/signal/)
59 ahart 1.8 {
60     $fileContents =~ s/.*\(signal ([^)]*)\).*/$1/g;
61 ahart 1.5 $nBadJobs++;
62 ahart 1.8 print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
63     $signals{$dir}{$jobNumber} = $fileContents;
64 ahart 1.3 }
65 ahart 1.17 else
66     {
67     $nIncompleteJobs++;
68     $partial{$dir}{$jobNumber} = 1;
69     }
70 ahart 1.3 }
71 ahart 1.5 if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
72     {
73     open (CROSS_SECTION, "<$file");
74     my $crossSection = <CROSS_SECTION>;
75     close (CROSS_SECTION);
76     $crossSections{$dir} = $crossSection;
77     }
78 ahart 1.3 }
79 ahart 1.1 foreach my $file (@$files)
80     {
81     next if $file eq ".";
82     next if $file eq "..";
83 ahart 1.5 my $dir = $file;
84     $dir =~ s/^(.*)\/[^\/]*$/$1/;
85 ahart 1.3 my $badJob = 0;
86 ahart 1.20 my $jobNumber;
87 ahart 1.27 my $fileIsEDM = 0;
88     $fileIsEDM = isEDM ($file) if $file =~ m/^.*\.root$/;
89     if ($file =~ m/^.*_[^_]*\.root$/ && !$fileIsEDM)
90 ahart 1.1 {
91 ahart 1.20 $jobNumber = $file;
92 ahart 1.3 $jobNumber =~ s/^.*_([^_]*)\.root$/$1/;
93 ahart 1.17 $badJob = (defined $signals{$dir} && defined $signals{$dir}{$jobNumber}) || (defined $partial{$dir} && defined $partial{$dir}{$jobNumber});
94 ahart 1.5 }
95     next if $badJob;
96 ahart 1.27 if ($file =~ m/^.*\.root$/ && !$fileIsEDM)
97 ahart 1.5 {
98 ahart 1.11 foreach my $arg (@ARGV)
99     {
100 ahart 1.19 if (substr ($file, 0, length ($arg)) eq $arg)
101 ahart 1.11 {
102 ahart 1.19 $dirs{$arg} = $dir;
103 ahart 1.20 if (countEvents ($file, $cutFlow) < 0)
104     {
105     $nGoodJobs--;
106     $nBadJobs++;
107     print "WARNING: Skipping job $jobNumber. (bad ROOT file)\n";
108     last;
109     }
110 ahart 1.11 push (@{$rootFiles{$arg}}, $file);
111 ahart 1.24 if ($opt{"weight"})
112     {
113     push (@weights, $opt{"weight"}) if !($opt{"weight"} =~ m/,/);
114     push (@weights, 1.0) if $opt{"weight"} =~ m/,/;
115     }
116 ahart 1.13 push (@weights, $opt{"xsection"} * $integratedLuminosity) if !$opt{"weight"} && $opt{"xsection"};
117     push (@weights, $crossSections{$dir} * $integratedLuminosity) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
118     push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
119     $weights{$arg} = $weights[-1];
120 ahart 1.11 }
121     }
122 ahart 1.5 }
123     }
124 ahart 1.11 if (!%rootFiles)
125 ahart 1.6 {
126     print "Found no ROOT files to merge!\n";
127     exit;
128     }
129 ahart 1.11 my %nTotalEvents;
130     my $nTotalEvents = 0;
131 ahart 1.13 my @mergedFiles;
132     my @mergedWeights;
133 ahart 1.11 foreach my $arg (@ARGV)
134     {
135     my $rootFiles = join (" ", @{$rootFiles{$arg}});
136 ahart 1.13 my $tmpName = $arg . "_" . "$opt{'prefix'}.root";
137     $tmpName =~ s/\//_/g;
138     system ("mergeTFileServiceHistograms -i $rootFiles -o $tmpName");
139     my $count = countEvents ($tmpName, $cutFlow);
140     system ("cutFlowLimits $tmpName");
141 ahart 1.11 $nTotalEvents{$arg} = $count;
142     $nTotalEvents += $count;
143 ahart 1.14 $weights{$arg} /= $count if !$opt{"weight"} && $opt{"xsection"};
144     $weights{$arg} /= $count if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dirs{$arg}};
145 ahart 1.19 push (@mergedFiles, $tmpName);
146 ahart 1.14 push (@mergedWeights, $weights{$arg});
147 ahart 1.25 foreach my $rootFile (@{$rootFiles{$arg}})
148     {
149     my $weight = $weights{$arg};
150     system ("weightTrees $rootFile $weight");
151     }
152 ahart 1.26 if (defined $crossSections{$dirs{$arg}})
153     {
154     open (EVENT_COUNT, ">$dirs{$arg}/numberOfEvents.txt");
155     print EVENT_COUNT sprintf ("%.0f", $count) . "\n";
156     close (EVENT_COUNT);
157     }
158 ahart 1.11 }
159 ahart 1.13 my $mergedFiles = join (" ", @mergedFiles);
160     my $mergedWeights = join (",", @mergedWeights);
161 ahart 1.24 $mergedWeights = $opt{"weight"} if $opt{"weight"} && $opt{"weight"} =~ m/,/;
162 ahart 1.13 system ("mergeTFileServiceHistograms -i $mergedFiles -o $opt{'prefix'}.root -w $mergedWeights");
163     foreach my $mergedFile (@mergedFiles)
164 ahart 1.5 {
165 ahart 1.13 unlink ("$mergedFile");
166 ahart 1.1 }
167 ahart 1.19 print "=============================================\n";
168 lantonel 1.23 my $output = sprintf "Cross-section of samples:\n";
169 ahart 1.21 my $printOutput = 0;
170 ahart 1.10 foreach my $arg (keys %weights)
171 lantonel 1.23 {
172 ahart 1.12 my $shortArg = $arg;
173     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
174 lantonel 1.23 $output = sprintf "%s $shortArg: %.5g pb\n", $output, ($crossSections{$dirs{$arg}});
175 ahart 1.21 $printOutput = 1 if $weights{$arg} != 1;
176 lantonel 1.23 }
177     print $output if $printOutput;
178 jbrinson 1.22
179 lantonel 1.23 my $output = sprintf "Effective luminosities of samples:\n";
180     my $printOutput = 0;
181 jbrinson 1.22 foreach my $arg (keys %weights)
182 lantonel 1.23 {
183 jbrinson 1.22 my $shortArg = $arg;
184     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
185 lantonel 1.23 $output = sprintf "%s $shortArg: %.5g/fb\n", $output, ($integratedLuminosity / (1000.0 * $weights{$arg})) if $weights{$arg} != 1;
186 jbrinson 1.22 $printOutput = 1 if $weights{$arg} != 1;
187 lantonel 1.23 }
188     print $output if $printOutput;
189 jbrinson 1.22
190 ahart 1.19 $output = sprintf "Weights for target luminosity of %g/fb:\n", ($integratedLuminosity / 1000.0);
191     $printOutput = 0;
192     foreach my $arg (keys %weights)
193     {
194     my $shortArg = $arg;
195     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
196 ahart 1.21 $output = sprintf "%s $shortArg: %.5g\n", $output, $weights{$arg} if $weights{$arg} != 1;
197 ahart 1.18 $printOutput = 1 if $weights{$arg} != 1;
198 ahart 1.10 }
199 ahart 1.18 print $output if $printOutput;
200 lantonel 1.23
201 ahart 1.6 my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
202 ahart 1.26 printf "$nGoodJobs jobs ran successfully over $nTotalEvents (%.1f weighted) events.\n", $goodEvents if $counting;
203 ahart 1.4 print "$nBadJobs jobs failed to run.\n" if $counting;
204 ahart 1.17 print "$nIncompleteJobs jobs have not finished.\n" if $counting;
205 ahart 1.19 print "=============================================\n";
206 ahart 1.1
207     sub
208     processArgs
209     {
210     my $argv = shift;
211    
212     my @files;
213     foreach my $arg (@$argv)
214     {
215     $arg =~ s/\/*$//;
216     if (!(-e $arg))
217     {
218     print "$arg does not exist!\n";
219     exit;
220     }
221     next if ($arg =~ m/\/\.$/ || $arg =~ m/\/\.\.$/);
222     if (-d $arg)
223     {
224     opendir (DIR, $arg);
225     my @dirContents = readdir (DIR);
226     closedir (DIR);
227     for (my $i = 0; $i < @dirContents; $i++)
228     {
229     $dirContents[$i] = "$arg/$dirContents[$i]";
230     }
231     my $newFiles = processArgs (\@dirContents);
232     push (@files, @$newFiles);
233     }
234     else
235     {
236     push (@files, $arg);
237     }
238     }
239    
240     return \@files;
241     }
242    
243     sub
244     printHelp
245     {
246     my $exeName = $0;
247     $exeName =~ s/^.*\/([^\/]*)$/$1/;
248    
249 ahart 1.6 print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
250 ahart 1.7 print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
251     print "specified directories, checks for nonzero return values. If the directories\n";
252     print "where created by \"osusub\", uses the cross section from the database to weight\n";
253     print "all histograms.\n";
254 ahart 1.1 print "\n";
255     print "Mandatory arguments to long options are mandatory for short options too.\n";
256 ahart 1.6 printf "%-29s%s\n", " -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
257     printf "%-29s%s\n", " ", "cutFlow)";
258 ahart 1.1 printf "%-29s%s\n", " -h, --help", "print this help message";
259 ahart 1.24 printf "%-29s%s\n", " -l, --luminosity", "integrated luminosity to which the histograms are";
260     printf "%-29s%s\n", " ", "weighted (default: 10000/pb)";
261 ahart 1.7 printf "%-29s%s\n", " -p, --prefix PREFIX", "output is named PREFIX.root";
262 ahart 1.5 printf "%-29s%s\n", " -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
263     printf "%-29s%s\n", " ", "automatic weighting using the cross section from";
264 ahart 1.24 printf "%-29s%s\n", " ", "the database; WEIGHT may be a single number or a";
265     printf "%-29s%s\n", " ", "comma-separated list, one for each input file";
266 ahart 1.6 printf "%-29s%s\n", " -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
267     printf "%-29s%s\n", " ", "the value in the database";
268 ahart 1.1
269     exit;
270     }
271 ahart 1.3
272     sub
273     getRunList
274     {
275     my $runListFile = shift;
276    
277     open (RUN_LIST, "<$runListFile");
278     my @runList0 = <RUN_LIST>;
279     close (RUN_LIST);
280     my @runList;
281     foreach my $file (@runList0)
282     {
283     next if !($file =~ m/^.*file:.*\.root.*/);
284     $file =~ s/.*file:(.*)\.root.*/$1.root/;
285     push (@runList, $file);
286     }
287    
288     return \@runList;
289     }
290    
291     sub
292     countEvents
293     {
294 ahart 1.4 my $file = shift;
295 ahart 1.6 my $cutFlow = shift;
296 ahart 1.4
297 ahart 1.6 my $output = `getEventsFromCutFlow $file $cutFlow`;
298 ahart 1.16 if ($output =~ m/Did not find a histogram named/ || $output =~ m/appears to be empty/ || $output =~ m/Failed to open/)
299 ahart 1.5 {
300     print $output;
301     return -1;
302     }
303 ahart 1.4 $output =~ s/^.*: (.*)$/$1/;
304     $output =~ s/\n//g;
305    
306     return $output;
307 ahart 1.3 }
308 ahart 1.27
309     sub
310     isEDM
311     {
312     my $file = shift;
313    
314     my $output = `edmFileUtil $file 2>&1`;
315     return 0 if $output =~ m/appears to be missing/ || $output =~ m/not a ROOT file/;
316     return 1;
317     }