[ViewVC] Annotation of: cvsroot/UserCode/OSUT3Analysis/AnaTools/scripts/mergeHists

#!/usr/bin/env perl

use strict;
use Getopt::Long;
use POSIX;

sub processArgs;
sub printHelp;
sub getRunList;
sub countEvents;

my %opt;
Getopt::Long::Configure ("bundling");
GetOptions (\%opt, "cutflow|c=s", "luminosity|l=s", "prefix|p=s", "weight|w=s", "xsection|x=s", "help|h");

printHelp () if $opt{"help"} || !$opt{"prefix"};
my $files = processArgs (\@ARGV);
my %rootFiles;
my %weights;
my @weights;
my $nGoodJobs = 0;
my $nBadJobs = 0;
my $nIncompleteJobs = 0;
my $counting = 0;
my %exitCodes;
my %signals;
my %partial;
my %crossSections;
my %dirs;
my $integratedLuminosity = 10000;
$integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
my $cutFlow = "cutFlow";
$cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
foreach my $file (@$files)
  {
    next if $file eq ".";
    next if $file eq "..";
    my $dir = $file;
    $dir =~ s/^(.*)\/[^\/]*$/$1/;
    if ($file =~ m/^.*\/condor_[^_]*\.log$/)
      {
        my $jobNumber = $file;
        $jobNumber =~ s/^.*\/condor_([^_]*)\.log$/$1/;
        open (FILE, "<$file");
        my @fileContents = <FILE>;
        close (FILE);
        my $fileContents = join ("", @fileContents);
        $fileContents =~ s/\n/ /g;
        $counting = 1;
        if ($fileContents =~ m/return value/)
          {
            $fileContents =~ s/.*\(return value ([^)]*)\).*/$1/g;
            $nGoodJobs++;
            print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
            $exitCodes{$dir}{$jobNumber} = $fileContents;
          }
        elsif ($fileContents =~ m/signal/)
          {
            $fileContents =~ s/.*\(signal ([^)]*)\).*/$1/g;
            $nBadJobs++;
            print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
            $signals{$dir}{$jobNumber} = $fileContents;
          }
        else
          {
            $nIncompleteJobs++;
            $partial{$dir}{$jobNumber} = 1;
          }
      }
    if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
      {
        open (CROSS_SECTION, "<$file");
        my $crossSection = <CROSS_SECTION>;
        close (CROSS_SECTION);
        $crossSections{$dir} = $crossSection;
      }
  }
foreach my $file (@$files)
  {
    next if $file eq ".";
    next if $file eq "..";
    my $dir = $file;
    $dir =~ s/^(.*)\/[^\/]*$/$1/;
    my $badJob = 0;
    my $jobNumber;
    if ($file =~ m/^.*_[^_]*\.root$/)
      {
        $jobNumber = $file;
        $jobNumber =~ s/^.*_([^_]*)\.root$/$1/;
        $badJob = (defined $signals{$dir} && defined $signals{$dir}{$jobNumber}) || (defined $partial{$dir} && defined $partial{$dir}{$jobNumber});
      }
    next if $badJob;
    if ($file =~ m/^.*\.root$/)
      {
        foreach my $arg (@ARGV)
          {
            if (substr ($file, 0, length ($arg)) eq $arg)
              {
                $dirs{$arg} = $dir;
                if (countEvents ($file, $cutFlow) < 0)
                  {
                    $nGoodJobs--;
                    $nBadJobs++;
                    print "WARNING: Skipping job $jobNumber. (bad ROOT file)\n";
                    last;
                  }
                push (@{$rootFiles{$arg}}, $file);
                push (@weights, $opt{"weight"}) if $opt{"weight"};
                push (@weights, $opt{"xsection"} * $integratedLuminosity) if !$opt{"weight"} && $opt{"xsection"};
                push (@weights, $crossSections{$dir} * $integratedLuminosity) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
                push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
                $weights{$arg} = $weights[-1];
              }
          }
      }
  }
if (!%rootFiles)
  {
    print "Found no ROOT files to merge!\n";
    exit;
  }
my %nTotalEvents;
my $nTotalEvents = 0;
my @mergedFiles;
my @mergedWeights;
foreach my $arg (@ARGV)
  {
    my $rootFiles = join (" ", @{$rootFiles{$arg}});
    my $tmpName = $arg . "_" . "$opt{'prefix'}.root";
    $tmpName =~ s/\//_/g;
    system ("mergeTFileServiceHistograms -i $rootFiles -o $tmpName");
    my $count = countEvents ($tmpName, $cutFlow);
    system ("cutFlowLimits $tmpName");
    $nTotalEvents{$arg} = $count;
    $nTotalEvents += $count;
    $weights{$arg} /= $count if !$opt{"weight"} && $opt{"xsection"};
    $weights{$arg} /= $count if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dirs{$arg}};
    push (@mergedFiles, $tmpName);
    push (@mergedWeights, $weights{$arg});
  }
my $mergedFiles = join (" ", @mergedFiles);
my $mergedWeights = join (",", @mergedWeights);
system ("mergeTFileServiceHistograms -i $mergedFiles -o $opt{'prefix'}.root -w $mergedWeights");
foreach my $mergedFile (@mergedFiles)
  {
    unlink ("$mergedFile");
  }
print "=============================================\n";
my $output = sprintf "Effective luminosities of samples:\n";
my $printOutput = 0;
foreach my $arg (keys %weights)
  {
    my $shortArg = $arg;
    $shortArg =~ s/^.*\/([^\/]*)$/$1/;
    $output = sprintf "%s  $shortArg: %.5g/fb\n", $output, ($integratedLuminosity / (1000.0 * $weights{$arg})) if $weights{$arg} != 1;
    $printOutput = 1 if $weights{$arg} != 1;
  }
print $output if $printOutput;
$output = sprintf "Weights for target luminosity of %g/fb:\n", ($integratedLuminosity / 1000.0);
$printOutput = 0;
foreach my $arg (keys %weights)
  {
    my $shortArg = $arg;
    $shortArg =~ s/^.*\/([^\/]*)$/$1/;
    $output = sprintf "%s  $shortArg: %.5g\n", $output, $weights{$arg} if $weights{$arg} != 1;
    $printOutput = 1 if $weights{$arg} != 1;
  }
print $output if $printOutput;
my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
print "$nGoodJobs jobs ran successfully over $nTotalEvents ($goodEvents weighted) events.\n" if $counting;
print "$nBadJobs jobs failed to run.\n" if $counting;
print "$nIncompleteJobs jobs have not finished.\n" if $counting;
print "=============================================\n";

sub
processArgs
{
  my $argv = shift;

  my @files;
  foreach my $arg (@$argv)
    {
      $arg =~ s/\/*$//;
      if (!(-e $arg))
        {
          print "$arg does not exist!\n";
          exit;
        }
      next if ($arg =~ m/\/\.$/ || $arg =~ m/\/\.\.$/);
      if (-d $arg)
        {
          opendir (DIR, $arg);
          my @dirContents = readdir (DIR);
          closedir (DIR);
          for (my $i = 0; $i < @dirContents; $i++)
            {
              $dirContents[$i] = "$arg/$dirContents[$i]";
            }
          my $newFiles = processArgs (\@dirContents);
          push (@files, @$newFiles);
        }
      else
        {
          push (@files, $arg);
        }
    }

  return \@files;
}

sub
printHelp
{
  my $exeName = $0;
  $exeName =~ s/^.*\/([^\/]*)$/$1/;

  print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
  print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
  print "specified directories, checks for nonzero return values. If the directories\n";
  print "where created by \"osusub\", uses the cross section from the database to weight\n";
  print "all histograms.\n";
  print "\n";
  print "Mandatory arguments to long options are mandatory for short options too.\n";
  printf "%-29s%s\n", "  -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
  printf "%-29s%s\n", "                  ", "cutFlow)";
  printf "%-29s%s\n", "  -h, --help", "print this help message";
  printf "%-29s%s\n", "  -l, --luminosity", "integrated luminosity to which the histograms are";
  printf "%-29s%s\n", "                  ", "weighted (default: 10000/pb)";
  printf "%-29s%s\n", "  -p, --prefix PREFIX", "output is  named PREFIX.root";
  printf "%-29s%s\n", "  -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
  printf "%-29s%s\n", "                     ", "automatic weighting using the cross section from";
  printf "%-29s%s\n", "                     ", "the database";
  printf "%-29s%s\n", "  -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
  printf "%-29s%s\n", "                     ", "the value in the database";

  exit;
}

sub
getRunList
{
  my $runListFile = shift;

  open (RUN_LIST, "<$runListFile");
  my @runList0 = <RUN_LIST>;
  close (RUN_LIST);
  my @runList;
  foreach my $file (@runList0)
    {
      next if !($file =~ m/^.*file:.*\.root.*/);
      $file =~ s/.*file:(.*)\.root.*/$1.root/;
      push (@runList, $file);
    }

  return \@runList;
}

sub
countEvents
{
  my $file = shift;
  my $cutFlow = shift;

  my $output = `getEventsFromCutFlow $file $cutFlow`;
  if ($output =~ m/Did not find a histogram named/ || $output =~ m/appears to be empty/ || $output =~ m/Failed to open/)
    {
      print $output;
      return -1;
    }
  $output =~ s/^.*: (.*)$/$1/;
  $output =~ s/\n//g;

  return $output;
}
Revision:	1.21
Committed:	Wed Mar 6 17:01:01 2013 UTC (12 years, 1 month ago) by ahart
Branch:	MAIN
Changes since 1.20:	+4 -3 lines
Log Message:	Now outputs effective luminosities only if the sample is being weighted.
#	User	Rev	Content
1	ahart	1.1	#!/usr/bin/env perl
2
3			use strict;
4			use Getopt::Long;
5	ahart	1.3	use POSIX;
6
7			sub processArgs;
8			sub printHelp;
9			sub getRunList;
10			sub countEvents;
11	ahart	1.1
12			my %opt;
13			Getopt::Long::Configure ("bundling");
14	ahart	1.6	GetOptions (\%opt, "cutflow\|c=s", "luminosity\|l=s", "prefix\|p=s", "weight\|w=s", "xsection\|x=s", "help\|h");
15	ahart	1.1
16			printHelp () if $opt{"help"} \|\| !$opt{"prefix"};
17			my $files = processArgs (\@ARGV);
18	ahart	1.11	my %rootFiles;
19			my %weights;
20	ahart	1.13	my @weights;
21	ahart	1.5	my $nGoodJobs = 0;
22			my $nBadJobs = 0;
23	ahart	1.17	my $nIncompleteJobs = 0;
24	ahart	1.3	my $counting = 0;
25	ahart	1.4	my %exitCodes;
26	ahart	1.8	my %signals;
27	ahart	1.17	my %partial;
28	ahart	1.5	my %crossSections;
29	ahart	1.14	my %dirs;
30	ahart	1.5	my $integratedLuminosity = 10000;
31			$integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
32	ahart	1.6	my $cutFlow = "cutFlow";
33			$cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
34	ahart	1.3	foreach my $file (@$files)
35			{
36			next if $file eq ".";
37			next if $file eq "..";
38	ahart	1.5	my $dir = $file;
39			$dir =~ s/^(.)\/[^\/]$/$1/;
40	ahart	1.4	if ($file =~ m/^.\/condor_[^_]\.log$/)
41	ahart	1.3	{
42			my $jobNumber = $file;
43			$jobNumber =~ s/^.\/condor_([^_])\.log$/$1/;
44			open (FILE, "<$file");
45			my @fileContents = <FILE>;
46			close (FILE);
47			my $fileContents = join ("", @fileContents);
48			$fileContents =~ s/\n/ /g;
49	ahart	1.19	$counting = 1;
50	ahart	1.8	if ($fileContents =~ m/return value/)
51	ahart	1.3	{
52	ahart	1.8	$fileContents =~ s/.\(return value ([^)])\).*/$1/g;
53	ahart	1.9	$nGoodJobs++;
54			print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
55	ahart	1.8	$exitCodes{$dir}{$jobNumber} = $fileContents;
56			}
57	ahart	1.17	elsif ($fileContents =~ m/signal/)
58	ahart	1.8	{
59			$fileContents =~ s/.\(signal ([^)])\).*/$1/g;
60	ahart	1.5	$nBadJobs++;
61	ahart	1.8	print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
62			$signals{$dir}{$jobNumber} = $fileContents;
63	ahart	1.3	}
64	ahart	1.17	else
65			{
66			$nIncompleteJobs++;
67			$partial{$dir}{$jobNumber} = 1;
68			}
69	ahart	1.3	}
70	ahart	1.5	if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
71			{
72			open (CROSS_SECTION, "<$file");
73			my $crossSection = <CROSS_SECTION>;
74			close (CROSS_SECTION);
75			$crossSections{$dir} = $crossSection;
76			}
77	ahart	1.3	}
78	ahart	1.1	foreach my $file (@$files)
79			{
80			next if $file eq ".";
81			next if $file eq "..";
82	ahart	1.5	my $dir = $file;
83			$dir =~ s/^(.)\/[^\/]$/$1/;
84	ahart	1.3	my $badJob = 0;
85	ahart	1.20	my $jobNumber;
86	ahart	1.3	if ($file =~ m/^._[^_]\.root$/)
87	ahart	1.1	{
88	ahart	1.20	$jobNumber = $file;
89	ahart	1.3	$jobNumber =~ s/^._([^_])\.root$/$1/;
90	ahart	1.17	$badJob = (defined $signals{$dir} && defined $signals{$dir}{$jobNumber}) \|\| (defined $partial{$dir} && defined $partial{$dir}{$jobNumber});
91	ahart	1.5	}
92			next if $badJob;
93			if ($file =~ m/^.*\.root$/)
94			{
95	ahart	1.11	foreach my $arg (@ARGV)
96			{
97	ahart	1.19	if (substr ($file, 0, length ($arg)) eq $arg)
98	ahart	1.11	{
99	ahart	1.19	$dirs{$arg} = $dir;
100	ahart	1.20	if (countEvents ($file, $cutFlow) < 0)
101			{
102			$nGoodJobs--;
103			$nBadJobs++;
104			print "WARNING: Skipping job $jobNumber. (bad ROOT file)\n";
105			last;
106			}
107	ahart	1.11	push (@{$rootFiles{$arg}}, $file);
108	ahart	1.13	push (@weights, $opt{"weight"}) if $opt{"weight"};
109			push (@weights, $opt{"xsection"} * $integratedLuminosity) if !$opt{"weight"} && $opt{"xsection"};
110			push (@weights, $crossSections{$dir} * $integratedLuminosity) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
111			push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
112			$weights{$arg} = $weights[-1];
113	ahart	1.11	}
114			}
115	ahart	1.5	}
116			}
117	ahart	1.11	if (!%rootFiles)
118	ahart	1.6	{
119			print "Found no ROOT files to merge!\n";
120			exit;
121			}
122	ahart	1.11	my %nTotalEvents;
123			my $nTotalEvents = 0;
124	ahart	1.13	my @mergedFiles;
125			my @mergedWeights;
126	ahart	1.11	foreach my $arg (@ARGV)
127			{
128			my $rootFiles = join (" ", @{$rootFiles{$arg}});
129	ahart	1.13	my $tmpName = $arg . "_" . "$opt{'prefix'}.root";
130			$tmpName =~ s/\//_/g;
131			system ("mergeTFileServiceHistograms -i $rootFiles -o $tmpName");
132			my $count = countEvents ($tmpName, $cutFlow);
133			system ("cutFlowLimits $tmpName");
134	ahart	1.11	$nTotalEvents{$arg} = $count;
135			$nTotalEvents += $count;
136	ahart	1.14	$weights{$arg} /= $count if !$opt{"weight"} && $opt{"xsection"};
137			$weights{$arg} /= $count if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dirs{$arg}};
138	ahart	1.19	push (@mergedFiles, $tmpName);
139	ahart	1.14	push (@mergedWeights, $weights{$arg});
140	ahart	1.11	}
141	ahart	1.13	my $mergedFiles = join (" ", @mergedFiles);
142			my $mergedWeights = join (",", @mergedWeights);
143			system ("mergeTFileServiceHistograms -i $mergedFiles -o $opt{'prefix'}.root -w $mergedWeights");
144			foreach my $mergedFile (@mergedFiles)
145	ahart	1.5	{
146	ahart	1.13	unlink ("$mergedFile");
147	ahart	1.1	}
148	ahart	1.19	print "=============================================\n";
149			my $output = sprintf "Effective luminosities of samples:\n";
150	ahart	1.21	my $printOutput = 0;
151	ahart	1.10	foreach my $arg (keys %weights)
152			{
153	ahart	1.12	my $shortArg = $arg;
154			$shortArg =~ s/^.\/([^\/])$/$1/;
155	ahart	1.21	$output = sprintf "%s $shortArg: %.5g/fb\n", $output, ($integratedLuminosity / (1000.0 * $weights{$arg})) if $weights{$arg} != 1;
156			$printOutput = 1 if $weights{$arg} != 1;
157	ahart	1.19	}
158			print $output if $printOutput;
159			$output = sprintf "Weights for target luminosity of %g/fb:\n", ($integratedLuminosity / 1000.0);
160			$printOutput = 0;
161			foreach my $arg (keys %weights)
162			{
163			my $shortArg = $arg;
164			$shortArg =~ s/^.\/([^\/])$/$1/;
165	ahart	1.21	$output = sprintf "%s $shortArg: %.5g\n", $output, $weights{$arg} if $weights{$arg} != 1;
166	ahart	1.18	$printOutput = 1 if $weights{$arg} != 1;
167	ahart	1.10	}
168	ahart	1.18	print $output if $printOutput;
169	ahart	1.6	my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
170	ahart	1.5	print "$nGoodJobs jobs ran successfully over $nTotalEvents ($goodEvents weighted) events.\n" if $counting;
171	ahart	1.4	print "$nBadJobs jobs failed to run.\n" if $counting;
172	ahart	1.17	print "$nIncompleteJobs jobs have not finished.\n" if $counting;
173	ahart	1.19	print "=============================================\n";
174	ahart	1.1
175			sub
176			processArgs
177			{
178			my $argv = shift;
179
180			my @files;
181			foreach my $arg (@$argv)
182			{
183			$arg =~ s/\/*$//;
184			if (!(-e $arg))
185			{
186			print "$arg does not exist!\n";
187			exit;
188			}
189			next if ($arg =~ m/\/\.$/ \|\| $arg =~ m/\/\.\.$/);
190			if (-d $arg)
191			{
192			opendir (DIR, $arg);
193			my @dirContents = readdir (DIR);
194			closedir (DIR);
195			for (my $i = 0; $i < @dirContents; $i++)
196			{
197			$dirContents[$i] = "$arg/$dirContents[$i]";
198			}
199			my $newFiles = processArgs (\@dirContents);
200			push (@files, @$newFiles);
201			}
202			else
203			{
204			push (@files, $arg);
205			}
206			}
207
208			return \@files;
209			}
210
211			sub
212			printHelp
213			{
214			my $exeName = $0;
215			$exeName =~ s/^.\/([^\/])$/$1/;
216
217	ahart	1.6	print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
218	ahart	1.7	print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
219			print "specified directories, checks for nonzero return values. If the directories\n";
220			print "where created by \"osusub\", uses the cross section from the database to weight\n";
221			print "all histograms.\n";
222	ahart	1.1	print "\n";
223			print "Mandatory arguments to long options are mandatory for short options too.\n";
224	ahart	1.6	printf "%-29s%s\n", " -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
225			printf "%-29s%s\n", " ", "cutFlow)";
226	ahart	1.1	printf "%-29s%s\n", " -h, --help", "print this help message";
227	ahart	1.5	printf "%-29s%s\n", " -l, --luminosity", "integrated luminosity to which the histograms are";
228			printf "%-29s%s\n", " ", "weighted (default: 10000/pb)";
229	ahart	1.7	printf "%-29s%s\n", " -p, --prefix PREFIX", "output is named PREFIX.root";
230	ahart	1.5	printf "%-29s%s\n", " -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
231			printf "%-29s%s\n", " ", "automatic weighting using the cross section from";
232			printf "%-29s%s\n", " ", "the database";
233	ahart	1.6	printf "%-29s%s\n", " -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
234			printf "%-29s%s\n", " ", "the value in the database";
235	ahart	1.1
236			exit;
237			}
238	ahart	1.3
239			sub
240			getRunList
241			{
242			my $runListFile = shift;
243
244			open (RUN_LIST, "<$runListFile");
245			my @runList0 = <RUN_LIST>;
246			close (RUN_LIST);
247			my @runList;
248			foreach my $file (@runList0)
249			{
250			next if !($file =~ m/^.file:.\.root.*/);
251			$file =~ s/.file:(.)\.root.*/$1.root/;
252			push (@runList, $file);
253			}
254
255			return \@runList;
256			}
257
258			sub
259			countEvents
260			{
261	ahart	1.4	my $file = shift;
262	ahart	1.6	my $cutFlow = shift;
263	ahart	1.4
264	ahart	1.6	my $output = `getEventsFromCutFlow $file $cutFlow`;
265	ahart	1.16	if ($output =~ m/Did not find a histogram named/ \|\| $output =~ m/appears to be empty/ \|\| $output =~ m/Failed to open/)
266	ahart	1.5	{
267			print $output;
268			return -1;
269			}
270	ahart	1.4	$output =~ s/^.: (.)$/$1/;
271			$output =~ s/\n//g;
272
273			return $output;
274	ahart	1.3	}