[ViewVC] Annotation of: cvsroot/UserCode/OSUT3Analysis/AnaTools/scripts/mergeHists

#!/usr/bin/env perl

use strict;
use Getopt::Long;
use POSIX;

sub processArgs;
sub printHelp;
sub getRunList;
sub countEvents;

my %opt;
Getopt::Long::Configure ("bundling");
GetOptions (\%opt, "cutflow|c=s", "luminosity|l=s", "prefix|p=s", "weight|w=s", "xsection|x=s", "help|h");

printHelp () if $opt{"help"} || !$opt{"prefix"};
my $files = processArgs (\@ARGV);
my @rootFiles;
my @weights;
my $nGoodJobs = 0;
my $nBadJobs = 0;
my $counting = 0;
my %exitCodes;
my %signals;
my %crossSections;
my $integratedLuminosity = 10000;
$integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
my $cutFlow = "cutFlow";
$cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
foreach my $file (@$files)
  {
    next if $file eq ".";
    next if $file eq "..";
    my $dir = $file;
    $dir =~ s/^(.*)\/[^\/]*$/$1/;
    if ($file =~ m/^.*\/condor_[^_]*\.log$/)
      {
        my $jobNumber = $file;
        $jobNumber =~ s/^.*\/condor_([^_]*)\.log$/$1/;
        open (FILE, "<$file");
        my @fileContents = <FILE>;
        close (FILE);
        my $fileContents = join ("", @fileContents);
        $fileContents =~ s/\n/ /g;
        if ($fileContents =~ m/return value/)
          {
            $fileContents =~ s/.*\(return value ([^)]*)\).*/$1/g;
            $nGoodJobs++;
            print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
            $exitCodes{$dir}{$jobNumber} = $fileContents;
            $counting = 1;
          }
        if ($fileContents =~ m/signal/)
          {
            $fileContents =~ s/.*\(signal ([^)]*)\).*/$1/g;
            $nBadJobs++;
            print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
            $signals{$dir}{$jobNumber} = $fileContents;
            $counting = 1;
          }
      }
    if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
      {
        open (CROSS_SECTION, "<$file");
        my $crossSection = <CROSS_SECTION>;
        close (CROSS_SECTION);
        $crossSections{$dir} = $crossSection;
      }
  }
foreach my $file (@$files)
  {
    next if $file eq ".";
    next if $file eq "..";
    my $dir = $file;
    $dir =~ s/^(.*)\/[^\/]*$/$1/;
    my $badJob = 0;
    if ($file =~ m/^.*_[^_]*\.root$/)
      {
        my $jobNumber = $file;
        $jobNumber =~ s/^.*_([^_]*)\.root$/$1/;
        $badJob = defined $signals{$dir} && defined $signals{$dir}{$jobNumber};
      }
    next if $badJob;
    if ($file =~ m/^.*\.root$/)
      {
        push (@rootFiles, $file);
        push (@weights, 1.0);
      }
  }
if (!@rootFiles)
  {
    print "Found no ROOT files to merge!\n";
    exit;
  }
my $rootFiles = join (" ", @rootFiles);
my $weights = join (",", @weights);
system ("mergeTFileServiceHistograms -i $rootFiles -o $opt{'prefix'}.root -w $weights");
my $nTotalEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
unlink ("$opt{'prefix'}.root");
@weights = ();
my %weights = ();
foreach my $file (@$files)
  {
    next if $file eq ".";
    next if $file eq "..";
    my $dir = $file;
    $dir =~ s/^(.*)\/[^\/]*$/$1/;
    my $badJob = 0;
    if ($file =~ m/^.*_[^_]*\.root$/)
      {
        my $jobNumber = $file;
        $jobNumber =~ s/^.*_([^_]*)\.root$/$1/;
        $badJob = defined $signals{$dir} && defined $signals{$dir}{$jobNumber};
      }
    next if $badJob;
    if ($file =~ m/^.*\.root$/)
      {
        push (@weights, $opt{"weight"}) if $opt{"weight"};
        push (@weights, ($opt{"xsection"} * $integratedLuminosity) / $nTotalEvents) if !$opt{"weight"} && $opt{"xsection"};
        push (@weights, ($crossSections{$dir} * $integratedLuminosity) / $nTotalEvents) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
        push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
        foreach my $arg (@ARGV)
          {
            if (substr ($file, 0, length ($arg)) eq $arg)
              {
                $arg =~ s/^.*\/([^\/]*)$/$1/;
                $weights{$arg} = $weights[-1]
              }
          }
      }
  }
my $weights = join (",", @weights);
system ("mergeTFileServiceHistograms -i $rootFiles -o $opt{'prefix'}.root -w $weights");
print "Weights:\n";
foreach my $arg (keys %weights)
  {
    printf "  $arg: %.1f (%.1f pb)\n", $weights{$arg}, $weights{$arg} / $integratedLuminosity;
  }
my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
print "$nGoodJobs jobs ran successfully over $nTotalEvents ($goodEvents weighted) events.\n" if $counting;
print "$nBadJobs jobs failed to run.\n" if $counting;

sub
processArgs
{
  my $argv = shift;

  my @files;
  foreach my $arg (@$argv)
    {
      $arg =~ s/\/*$//;
      if (!(-e $arg))
        {
          print "$arg does not exist!\n";
          exit;
        }
      next if ($arg =~ m/\/\.$/ || $arg =~ m/\/\.\.$/);
      if (-d $arg)
        {
          opendir (DIR, $arg);
          my @dirContents = readdir (DIR);
          closedir (DIR);
          for (my $i = 0; $i < @dirContents; $i++)
            {
              $dirContents[$i] = "$arg/$dirContents[$i]";
            }
          my $newFiles = processArgs (\@dirContents);
          push (@files, @$newFiles);
        }
      else
        {
          push (@files, $arg);
        }
    }

  return \@files;
}

sub
printHelp
{
  my $exeName = $0;
  $exeName =~ s/^.*\/([^\/]*)$/$1/;

  print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
  print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
  print "specified directories, checks for nonzero return values. If the directories\n";
  print "where created by \"osusub\", uses the cross section from the database to weight\n";
  print "all histograms.\n";
  print "\n";
  print "Mandatory arguments to long options are mandatory for short options too.\n";
  printf "%-29s%s\n", "  -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
  printf "%-29s%s\n", "                  ", "cutFlow)";
  printf "%-29s%s\n", "  -h, --help", "print this help message";
  printf "%-29s%s\n", "  -l, --luminosity", "integrated luminosity to which the histograms are";
  printf "%-29s%s\n", "                  ", "weighted (default: 10000/pb)";
  printf "%-29s%s\n", "  -p, --prefix PREFIX", "output is  named PREFIX.root";
  printf "%-29s%s\n", "  -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
  printf "%-29s%s\n", "                     ", "automatic weighting using the cross section from";
  printf "%-29s%s\n", "                     ", "the database";
  printf "%-29s%s\n", "  -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
  printf "%-29s%s\n", "                     ", "the value in the database";

  exit;
}

sub
getRunList
{
  my $runListFile = shift;

  open (RUN_LIST, "<$runListFile");
  my @runList0 = <RUN_LIST>;
  close (RUN_LIST);
  my @runList;
  foreach my $file (@runList0)
    {
      next if !($file =~ m/^.*file:.*\.root.*/);
      $file =~ s/.*file:(.*)\.root.*/$1.root/;
      push (@runList, $file);
    }

  return \@runList;
}

sub
countEvents
{
  my $file = shift;
  my $cutFlow = shift;

  my $output = `getEventsFromCutFlow $file $cutFlow`;
  if ($output =~ m/Did not find a histogram named/)
    {
      print $output;
      return -1;
    }
  $output =~ s/^.*: (.*)$/$1/;
  $output =~ s/\n//g;

  return $output;
}
Revision:	1.10
Committed:	Mon Dec 17 21:17:35 2012 UTC (12 years, 4 months ago) by ahart
Branch:	MAIN
Changes since 1.9:	+14 -0 lines
Log Message:	Output the weights.
#	User	Rev	Content
1	ahart	1.1	#!/usr/bin/env perl
2
3			use strict;
4			use Getopt::Long;
5	ahart	1.3	use POSIX;
6
7			sub processArgs;
8			sub printHelp;
9			sub getRunList;
10			sub countEvents;
11	ahart	1.1
12			my %opt;
13			Getopt::Long::Configure ("bundling");
14	ahart	1.6	GetOptions (\%opt, "cutflow\|c=s", "luminosity\|l=s", "prefix\|p=s", "weight\|w=s", "xsection\|x=s", "help\|h");
15	ahart	1.1
16			printHelp () if $opt{"help"} \|\| !$opt{"prefix"};
17			my $files = processArgs (\@ARGV);
18			my @rootFiles;
19	ahart	1.3	my @weights;
20	ahart	1.5	my $nGoodJobs = 0;
21			my $nBadJobs = 0;
22	ahart	1.3	my $counting = 0;
23	ahart	1.4	my %exitCodes;
24	ahart	1.8	my %signals;
25	ahart	1.5	my %crossSections;
26			my $integratedLuminosity = 10000;
27			$integratedLuminosity = $opt{"luminosity"} if $opt{"luminosity"};
28	ahart	1.6	my $cutFlow = "cutFlow";
29			$cutFlow = $opt{"cutflow"} if $opt{"cutflow"};
30	ahart	1.3	foreach my $file (@$files)
31			{
32			next if $file eq ".";
33			next if $file eq "..";
34	ahart	1.5	my $dir = $file;
35			$dir =~ s/^(.)\/[^\/]$/$1/;
36	ahart	1.4	if ($file =~ m/^.\/condor_[^_]\.log$/)
37	ahart	1.3	{
38			my $jobNumber = $file;
39			$jobNumber =~ s/^.\/condor_([^_])\.log$/$1/;
40			open (FILE, "<$file");
41			my @fileContents = <FILE>;
42			close (FILE);
43			my $fileContents = join ("", @fileContents);
44			$fileContents =~ s/\n/ /g;
45	ahart	1.8	if ($fileContents =~ m/return value/)
46	ahart	1.3	{
47	ahart	1.8	$fileContents =~ s/.\(return value ([^)])\).*/$1/g;
48	ahart	1.9	$nGoodJobs++;
49			print "WARNING: Nonzero exit code for job $jobNumber. (return value $fileContents)\n" if $fileContents != 0;
50	ahart	1.8	$exitCodes{$dir}{$jobNumber} = $fileContents;
51			$counting = 1;
52			}
53			if ($fileContents =~ m/signal/)
54			{
55			$fileContents =~ s/.\(signal ([^)])\).*/$1/g;
56	ahart	1.5	$nBadJobs++;
57	ahart	1.8	print "WARNING: Skipping job $jobNumber. (signal $fileContents)\n";
58			$signals{$dir}{$jobNumber} = $fileContents;
59			$counting = 1;
60	ahart	1.3	}
61			}
62	ahart	1.5	if ($file =~ m/^.*\/crossSectionInPicobarn\.txt$/)
63			{
64			open (CROSS_SECTION, "<$file");
65			my $crossSection = <CROSS_SECTION>;
66			close (CROSS_SECTION);
67			$crossSections{$dir} = $crossSection;
68			}
69	ahart	1.3	}
70	ahart	1.1	foreach my $file (@$files)
71			{
72			next if $file eq ".";
73			next if $file eq "..";
74	ahart	1.5	my $dir = $file;
75			$dir =~ s/^(.)\/[^\/]$/$1/;
76	ahart	1.3	my $badJob = 0;
77			if ($file =~ m/^._[^_]\.root$/)
78	ahart	1.1	{
79	ahart	1.3	my $jobNumber = $file;
80			$jobNumber =~ s/^._([^_])\.root$/$1/;
81	ahart	1.8	$badJob = defined $signals{$dir} && defined $signals{$dir}{$jobNumber};
82	ahart	1.5	}
83			next if $badJob;
84			if ($file =~ m/^.*\.root$/)
85			{
86	ahart	1.6	push (@rootFiles, $file);
87			push (@weights, 1.0);
88	ahart	1.5	}
89			}
90	ahart	1.6	if (!@rootFiles)
91			{
92			print "Found no ROOT files to merge!\n";
93			exit;
94			}
95			my $rootFiles = join (" ", @rootFiles);
96			my $weights = join (",", @weights);
97			system ("mergeTFileServiceHistograms -i $rootFiles -o $opt{'prefix'}.root -w $weights");
98			my $nTotalEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
99			unlink ("$opt{'prefix'}.root");
100			@weights = ();
101	ahart	1.10	my %weights = ();
102	ahart	1.5	foreach my $file (@$files)
103			{
104			next if $file eq ".";
105			next if $file eq "..";
106			my $dir = $file;
107			$dir =~ s/^(.)\/[^\/]$/$1/;
108			my $badJob = 0;
109			if ($file =~ m/^._[^_]\.root$/)
110			{
111			my $jobNumber = $file;
112			$jobNumber =~ s/^._([^_])\.root$/$1/;
113	ahart	1.8	$badJob = defined $signals{$dir} && defined $signals{$dir}{$jobNumber};
114	ahart	1.1	}
115	ahart	1.3	next if $badJob;
116			if ($file =~ m/^.*\.root$/)
117	ahart	1.1	{
118	ahart	1.3	push (@weights, $opt{"weight"}) if $opt{"weight"};
119	ahart	1.6	push (@weights, ($opt{"xsection"} * $integratedLuminosity) / $nTotalEvents) if !$opt{"weight"} && $opt{"xsection"};
120			push (@weights, ($crossSections{$dir} * $integratedLuminosity) / $nTotalEvents) if !$opt{"weight"} && !$opt{"xsection"} && defined $crossSections{$dir};
121			push (@weights, 1.0) if !$opt{"weight"} && !$opt{"xsection"} && !(defined $crossSections{$dir});
122	ahart	1.10	foreach my $arg (@ARGV)
123			{
124			if (substr ($file, 0, length ($arg)) eq $arg)
125			{
126			$arg =~ s/^.\/([^\/])$/$1/;
127			$weights{$arg} = $weights[-1]
128			}
129			}
130	ahart	1.1	}
131			}
132	ahart	1.3	my $weights = join (",", @weights);
133			system ("mergeTFileServiceHistograms -i $rootFiles -o $opt{'prefix'}.root -w $weights");
134	ahart	1.10	print "Weights:\n";
135			foreach my $arg (keys %weights)
136			{
137			printf " $arg: %.1f (%.1f pb)\n", $weights{$arg}, $weights{$arg} / $integratedLuminosity;
138			}
139	ahart	1.6	my $goodEvents = countEvents ("$opt{'prefix'}.root", $cutFlow);
140	ahart	1.5	print "$nGoodJobs jobs ran successfully over $nTotalEvents ($goodEvents weighted) events.\n" if $counting;
141	ahart	1.4	print "$nBadJobs jobs failed to run.\n" if $counting;
142	ahart	1.1
143			sub
144			processArgs
145			{
146			my $argv = shift;
147
148			my @files;
149			foreach my $arg (@$argv)
150			{
151			$arg =~ s/\/*$//;
152			if (!(-e $arg))
153			{
154			print "$arg does not exist!\n";
155			exit;
156			}
157			next if ($arg =~ m/\/\.$/ \|\| $arg =~ m/\/\.\.$/);
158			if (-d $arg)
159			{
160			opendir (DIR, $arg);
161			my @dirContents = readdir (DIR);
162			closedir (DIR);
163			for (my $i = 0; $i < @dirContents; $i++)
164			{
165			$dirContents[$i] = "$arg/$dirContents[$i]";
166			}
167			my $newFiles = processArgs (\@dirContents);
168			push (@files, @$newFiles);
169			}
170			else
171			{
172			push (@files, $arg);
173			}
174			}
175
176			return \@files;
177			}
178
179			sub
180			printHelp
181			{
182			my $exeName = $0;
183			$exeName =~ s/^.\/([^\/])$/$1/;
184
185	ahart	1.6	print "Usage: $exeName [OPTION]... -p PREFIX DIRECTORIES_AND_FILES\n";
186	ahart	1.7	print "Merges ROOT files containing histograms. If there are Condor logs in the\n";
187			print "specified directories, checks for nonzero return values. If the directories\n";
188			print "where created by \"osusub\", uses the cross section from the database to weight\n";
189			print "all histograms.\n";
190	ahart	1.1	print "\n";
191			print "Mandatory arguments to long options are mandatory for short options too.\n";
192	ahart	1.6	printf "%-29s%s\n", " -c, --cutflow HISTOGRAM", "name of histogram to use for the cutflow (default:";
193			printf "%-29s%s\n", " ", "cutFlow)";
194	ahart	1.1	printf "%-29s%s\n", " -h, --help", "print this help message";
195	ahart	1.5	printf "%-29s%s\n", " -l, --luminosity", "integrated luminosity to which the histograms are";
196			printf "%-29s%s\n", " ", "weighted (default: 10000/pb)";
197	ahart	1.7	printf "%-29s%s\n", " -p, --prefix PREFIX", "output is named PREFIX.root";
198	ahart	1.5	printf "%-29s%s\n", " -w, --weight WEIGHT", "scale the output by WEIGHT, overriding the";
199			printf "%-29s%s\n", " ", "automatic weighting using the cross section from";
200			printf "%-29s%s\n", " ", "the database";
201	ahart	1.6	printf "%-29s%s\n", " -x, --xsection XSECTION", "use XSECTION to weight the histograms instead of";
202			printf "%-29s%s\n", " ", "the value in the database";
203	ahart	1.1
204			exit;
205			}
206	ahart	1.3
207			sub
208			getRunList
209			{
210			my $runListFile = shift;
211
212			open (RUN_LIST, "<$runListFile");
213			my @runList0 = <RUN_LIST>;
214			close (RUN_LIST);
215			my @runList;
216			foreach my $file (@runList0)
217			{
218			next if !($file =~ m/^.file:.\.root.*/);
219			$file =~ s/.file:(.)\.root.*/$1.root/;
220			push (@runList, $file);
221			}
222
223			return \@runList;
224			}
225
226			sub
227			countEvents
228			{
229	ahart	1.4	my $file = shift;
230	ahart	1.6	my $cutFlow = shift;
231	ahart	1.4
232	ahart	1.6	my $output = `getEventsFromCutFlow $file $cutFlow`;
233	ahart	1.5	if ($output =~ m/Did not find a histogram named/)
234			{
235			print $output;
236			return -1;
237			}
238	ahart	1.4	$output =~ s/^.: (.)$/$1/;
239			$output =~ s/\n//g;
240
241			return $output;
242	ahart	1.3	}