ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/AnaTools/scripts/mergeHists
Revision: 1.24
Committed: Fri Apr 12 05:18:05 2013 UTC (12 years ago) by ahart
Branch: MAIN
CVS Tags: V02-02-00, V02-01-01, V02-01-00, V01-01-00, V01-00-01, V01-00-00, V00-01-00
Changes since 1.23: +10 -3 lines
Log Message:
The "weight" option can now be passed a comma-separated list of weights, one for each input file.

File Contents

# User Rev Content
1 ahart 1.1 #!/usr/bin/env perl
2    
3     use strict;
4     use Getopt::Long;
5 ahart 1.3 use POSIX;
6    
7     sub processArgs;
8     sub printHelp;
9     sub getRunList;
10     sub countEvents;
11 ahart 1.1
12     my %opt;
13     Getopt::Long::Configure ("bundling");
14 ahart 1.6 GetOptions (\%opt, "cutflow|c=s", "luminosity|l=s", "prefix|p=s", "weight|w=s", "xsection|x=s", "help|h");
15 ahart 1.1
16     printHelp () if $opt{"help"} || !$opt{"prefix"};
17     my $files = processArgs (\@ARGV);
18 ahart 1.11 my %rootFiles;
19     my %weights;
20 ahart 1.13 my @weights;
21 ahart 1.5 my $nGoodJobs = 0;
22     my $nBadJobs = 0;
23 ahart 1.17 my $nIncompleteJobs = 0;
24 ahart 1.3 my $counting = 0;
25 ahart 1.4 my %exitCodes;
26 ahart 1.8 my %signals;
27 ahart 1.17 my %partial;
28 ahart 1.5 my %crossSections;
29 ahart 1.14 my %dirs;
30 ahart 1.5 my $integratedLuminosity = 10000;
31     $integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
32 ahart 1.6 my $cutFlow = "cutFlow";
33     $cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
34 ahart 1.3 foreach my $file (@$files)
35     {
36     next if $file eq ".";
37     next if $file eq "..";
38 ahart 1.5 my $dir = $file;
39     $dir =~ s/^(.*)\/[^\/]*$/$1/;
40 ahart 1.4 if ($file =~ m/^.*\/condor_[^_]*\.log$/)
41 ahart 1.3 {
42     my $jobNumber = $file;
43     $jobNumber =~ s/^.*\/condor_([^_]*)\.log$/$1/;
44     open (FILE, "<$file");
45     my @fileContents = <FILE>;
46     close (FILE);
47     my $fileContents = join ("", @fileContents);
48     $fileContents =~ s/\n/ /g;
49 ahart 1.19 $counting = 1;
50 ahart 1.8 if ($fileContents =~ m/return value/)
51 ahart 1.3 {
52 ahart 1.8 $fileContents =~ s/.*\(return value ([^)]*)\).*/$1/g;
53 ahart 1.9 $nGoodJobs++;
54     print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
55 ahart 1.8 $exitCodes{$dir}{$jobNumber} = $fileContents;
56     }
57 ahart 1.17 elsif ($fileContents =~ m/signal/)
58 ahart 1.8 {
59     $fileContents =~ s/.*\(signal ([^)]*)\).*/$1/g;
60 ahart 1.5 $nBadJobs++;
61 ahart 1.8 print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
62     $signals{$dir}{$jobNumber} = $fileContents;
63 ahart 1.3 }
64 ahart 1.17 else
65     {
66     $nIncompleteJobs++;
67     $partial{$dir}{$jobNumber} = 1;
68     }
69 ahart 1.3 }
70 ahart 1.5 if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
71     {
72     open (CROSS_SECTION, "<$file");
73     my $crossSection = <CROSS_SECTION>;
74     close (CROSS_SECTION);
75     $crossSections{$dir} = $crossSection;
76     }
77 ahart 1.3 }
78 ahart 1.1 foreach my $file (@$files)
79     {
80     next if $file eq ".";
81     next if $file eq "..";
82 ahart 1.5 my $dir = $file;
83     $dir =~ s/^(.*)\/[^\/]*$/$1/;
84 ahart 1.3 my $badJob = 0;
85 ahart 1.20 my $jobNumber;
86 ahart 1.3 if ($file =~ m/^.*_[^_]*\.root$/)
87 ahart 1.1 {
88 ahart 1.20 $jobNumber = $file;
89 ahart 1.3 $jobNumber =~ s/^.*_([^_]*)\.root$/$1/;
90 ahart 1.17 $badJob = (defined $signals{$dir} && defined $signals{$dir}{$jobNumber}) || (defined $partial{$dir} && defined $partial{$dir}{$jobNumber});
91 ahart 1.5 }
92     next if $badJob;
93     if ($file =~ m/^.*\.root$/)
94     {
95 ahart 1.11 foreach my $arg (@ARGV)
96     {
97 ahart 1.19 if (substr ($file, 0, length ($arg)) eq $arg)
98 ahart 1.11 {
99 ahart 1.19 $dirs{$arg} = $dir;
100 ahart 1.20 if (countEvents ($file, $cutFlow) < 0)
101     {
102     $nGoodJobs--;
103     $nBadJobs++;
104     print "WARNING: Skipping job $jobNumber. (bad ROOT file)\n";
105     last;
106     }
107 ahart 1.11 push (@{$rootFiles{$arg}}, $file);
108 ahart 1.24 if ($opt{"weight"})
109     {
110     push (@weights, $opt{"weight"}) if !($opt{"weight"} =~ m/,/);
111     push (@weights, 1.0) if $opt{"weight"} =~ m/,/;
112     }
113 ahart 1.13 push (@weights, $opt{"xsection"} * $integratedLuminosity) if !$opt{"weight"} && $opt{"xsection"};
114     push (@weights, $crossSections{$dir} * $integratedLuminosity) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
115     push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
116     $weights{$arg} = $weights[-1];
117 ahart 1.11 }
118     }
119 ahart 1.5 }
120     }
121 ahart 1.11 if (!%rootFiles)
122 ahart 1.6 {
123     print "Found no ROOT files to merge!\n";
124     exit;
125     }
126 ahart 1.11 my %nTotalEvents;
127     my $nTotalEvents = 0;
128 ahart 1.13 my @mergedFiles;
129     my @mergedWeights;
130 ahart 1.11 foreach my $arg (@ARGV)
131     {
132     my $rootFiles = join (" ", @{$rootFiles{$arg}});
133 ahart 1.13 my $tmpName = $arg . "_" . "$opt{'prefix'}.root";
134     $tmpName =~ s/\//_/g;
135     system ("mergeTFileServiceHistograms -i $rootFiles -o $tmpName");
136     my $count = countEvents ($tmpName, $cutFlow);
137     system ("cutFlowLimits $tmpName");
138 ahart 1.11 $nTotalEvents{$arg} = $count;
139     $nTotalEvents += $count;
140 ahart 1.14 $weights{$arg} /= $count if !$opt{"weight"} && $opt{"xsection"};
141     $weights{$arg} /= $count if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dirs{$arg}};
142 ahart 1.19 push (@mergedFiles, $tmpName);
143 ahart 1.14 push (@mergedWeights, $weights{$arg});
144 ahart 1.11 }
145 ahart 1.13 my $mergedFiles = join (" ", @mergedFiles);
146     my $mergedWeights = join (",", @mergedWeights);
147 ahart 1.24 $mergedWeights = $opt{"weight"} if $opt{"weight"} && $opt{"weight"} =~ m/,/;
148 ahart 1.13 system ("mergeTFileServiceHistograms -i $mergedFiles -o $opt{'prefix'}.root -w $mergedWeights");
149     foreach my $mergedFile (@mergedFiles)
150 ahart 1.5 {
151 ahart 1.13 unlink ("$mergedFile");
152 ahart 1.1 }
153 ahart 1.19 print "=============================================\n";
154 lantonel 1.23 my $output = sprintf "Cross-section of samples:\n";
155 ahart 1.21 my $printOutput = 0;
156 ahart 1.10 foreach my $arg (keys %weights)
157 lantonel 1.23 {
158 ahart 1.12 my $shortArg = $arg;
159     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
160 lantonel 1.23 $output = sprintf "%s $shortArg: %.5g pb\n", $output, ($crossSections{$dirs{$arg}});
161 ahart 1.21 $printOutput = 1 if $weights{$arg} != 1;
162 lantonel 1.23 }
163     print $output if $printOutput;
164 jbrinson 1.22
165 lantonel 1.23 my $output = sprintf "Effective luminosities of samples:\n";
166     my $printOutput = 0;
167 jbrinson 1.22 foreach my $arg (keys %weights)
168 lantonel 1.23 {
169 jbrinson 1.22 my $shortArg = $arg;
170     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
171 lantonel 1.23 $output = sprintf "%s $shortArg: %.5g/fb\n", $output, ($integratedLuminosity / (1000.0 * $weights{$arg})) if $weights{$arg} != 1;
172 jbrinson 1.22 $printOutput = 1 if $weights{$arg} != 1;
173 lantonel 1.23 }
174     print $output if $printOutput;
175 jbrinson 1.22
176 ahart 1.19 $output = sprintf "Weights for target luminosity of %g/fb:\n", ($integratedLuminosity / 1000.0);
177     $printOutput = 0;
178     foreach my $arg (keys %weights)
179     {
180     my $shortArg = $arg;
181     $shortArg =~ s/^.*\/([^\/]*)$/$1/;
182 ahart 1.21 $output = sprintf "%s $shortArg: %.5g\n", $output, $weights{$arg} if $weights{$arg} != 1;
183 ahart 1.18 $printOutput = 1 if $weights{$arg} != 1;
184 ahart 1.10 }
185 ahart 1.18 print $output if $printOutput;
186 lantonel 1.23
187 ahart 1.6 my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
188 ahart 1.5 print "$nGoodJobs jobs ran successfully over $nTotalEvents ($goodEvents weighted) events.\n" if $counting;
189 ahart 1.4 print "$nBadJobs jobs failed to run.\n" if $counting;
190 ahart 1.17 print "$nIncompleteJobs jobs have not finished.\n" if $counting;
191 ahart 1.19 print "=============================================\n";
192 ahart 1.1
193     sub
194     processArgs
195     {
196     my $argv = shift;
197    
198     my @files;
199     foreach my $arg (@$argv)
200     {
201     $arg =~ s/\/*$//;
202     if (!(-e $arg))
203     {
204     print "$arg does not exist!\n";
205     exit;
206     }
207     next if ($arg =~ m/\/\.$/ || $arg =~ m/\/\.\.$/);
208     if (-d $arg)
209     {
210     opendir (DIR, $arg);
211     my @dirContents = readdir (DIR);
212     closedir (DIR);
213     for (my $i = 0; $i < @dirContents; $i++)
214     {
215     $dirContents[$i] = "$arg/$dirContents[$i]";
216     }
217     my $newFiles = processArgs (\@dirContents);
218     push (@files, @$newFiles);
219     }
220     else
221     {
222     push (@files, $arg);
223     }
224     }
225    
226     return \@files;
227     }
228    
229     sub
230     printHelp
231     {
232     my $exeName = $0;
233     $exeName =~ s/^.*\/([^\/]*)$/$1/;
234    
235 ahart 1.6 print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
236 ahart 1.7 print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
237     print "specified directories, checks for nonzero return values. If the directories\n";
238     print "where created by \"osusub\", uses the cross section from the database to weight\n";
239     print "all histograms.\n";
240 ahart 1.1 print "\n";
241     print "Mandatory arguments to long options are mandatory for short options too.\n";
242 ahart 1.6 printf "%-29s%s\n", " -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
243     printf "%-29s%s\n", " ", "cutFlow)";
244 ahart 1.1 printf "%-29s%s\n", " -h, --help", "print this help message";
245 ahart 1.24 printf "%-29s%s\n", " -l, --luminosity", "integrated luminosity to which the histograms are";
246     printf "%-29s%s\n", " ", "weighted (default: 10000/pb)";
247 ahart 1.7 printf "%-29s%s\n", " -p, --prefix PREFIX", "output is named PREFIX.root";
248 ahart 1.5 printf "%-29s%s\n", " -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
249     printf "%-29s%s\n", " ", "automatic weighting using the cross section from";
250 ahart 1.24 printf "%-29s%s\n", " ", "the database; WEIGHT may be a single number or a";
251     printf "%-29s%s\n", " ", "comma-separated list, one for each input file";
252 ahart 1.6 printf "%-29s%s\n", " -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
253     printf "%-29s%s\n", " ", "the value in the database";
254 ahart 1.1
255     exit;
256     }
257 ahart 1.3
258     sub
259     getRunList
260     {
261     my $runListFile = shift;
262    
263     open (RUN_LIST, "<$runListFile");
264     my @runList0 = <RUN_LIST>;
265     close (RUN_LIST);
266     my @runList;
267     foreach my $file (@runList0)
268     {
269     next if !($file =~ m/^.*file:.*\.root.*/);
270     $file =~ s/.*file:(.*)\.root.*/$1.root/;
271     push (@runList, $file);
272     }
273    
274     return \@runList;
275     }
276    
277     sub
278     countEvents
279     {
280 ahart 1.4 my $file = shift;
281 ahart 1.6 my $cutFlow = shift;
282 ahart 1.4
283 ahart 1.6 my $output = `getEventsFromCutFlow $file $cutFlow`;
284 ahart 1.16 if ($output =~ m/Did not find a histogram named/ || $output =~ m/appears to be empty/ || $output =~ m/Failed to open/)
285 ahart 1.5 {
286     print $output;
287     return -1;
288     }
289 ahart 1.4 $output =~ s/^.*: (.*)$/$1/;
290     $output =~ s/\n//g;
291    
292     return $output;
293 ahart 1.3 }