[ViewVC] Annotation of: cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/crabclean

#!/usr/bin/env perl

use strict;
use Getopt::Long;
use Mysql;
use Term::ANSIColor;

sub printHelp;
sub parseCrabDir;

our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";

my %opt;
Getopt::Long::Configure ("bundling");
GetOptions (\%opt, "dumb|d", "force|f", "destination|o=s", "dataset|s", "help|h");

printHelp () if $opt{"help"} || !$ARGV[0] || (!$opt{"dumb"} && !$opt{"dataset"} && !$ARGV[1]);
my $crabDir = $ARGV[0];
my $outputDir = $ARGV[0];
my $dataset = $ARGV[0];
$outputDir = $ARGV[1] if !$opt{"dumb"} && !$opt{"dataset"};

if ($opt{"dataset"})
  {
    my $queryDataset = $dataset;
    $queryDataset =~ s/\*/%/g;
    $queryDataset =~ s/(.*)/%$1%/g;
    my $query = "select workingDirectory,location,dataset,user,creationTime,status from ntuple where dataset like '$queryDataset' order by creationTime";
    $db->selectdb ("ntuple");
    my $results = $db->query ($query);
    if ($results->numrows () == 1)
      {
        my @row = $results->fetchrow ();
        $crabDir = "$row[0]/ntuple";
        $outputDir = $row[1];
      }
    die "Database entry does not exist, stopped" if $results->numrows () == 0;
    if ($results->numrows () > 1)
      {
        my %workingDir;
        my %location;
        print "Found multiple database entries matching\n";
        print "\"$dataset\":\n";
        for (my $i = 1; $i <= $results->numrows (); $i++)
          {
            my @row = $results->fetchrow ();
            $row[4] =~ s/([^ ]*) [^ ]*/$1/g;
            $workingDir{"$i"} = $row[0];
            $location{"$i"} = $row[1];
            printf "(%2d) $row[2]\n", $i;
            print "     (";
            print color "green" if $row[5] eq "present";
            print color "bold yellow" if $row[5] eq "submitted";
            print color "bold red" if $row[5] eq "created" or $row[5] eq "cancelled" or $row[5] eq "deprecated";
            print $row[5];
            print color "reset";
            print ") created by $row[3] on $row[4]\n";
          }
        print "\nWhich entry would you like to use?: ";
        my $response = <STDIN>;
        $response =~ s/[ \t\n]//g;
        die "Your selection was not a valid option, stopped" if !(exists $workingDir{$response});
        $crabDir = "$workingDir{$response}/ntuple";
        $outputDir = $location{$response};
      }
  }

die "Output directory does not exist, stopped" if !(-e $outputDir) || !(-d $outputDir);
opendir (CRAB_OUTPUT, "$outputDir");
my @crabOutput = readdir (CRAB_OUTPUT);
closedir (CRAB_OUTPUT);
my %filesToKeep;
if (!$opt{"dumb"})
  {
    die "CRAB directory does not exist, stopped" if !(-e $crabDir) || !(-d $crabDir);
    parseCrabDir ($crabDir, \%filesToKeep);
  }
else
  {
    foreach my $file (@crabOutput)
      {
        next if ($file eq "." || $file eq "..");
        next if !($file =~ m/^.*_[^_]*_[^_]*_[^_]*\.root$/);
        my $jobNumber = $file;
        my $submissionNumber = $file;
        $jobNumber =~ s/^.*_([^_]*)_[^_]*_[^_]*\.root$/$1/;
        $submissionNumber =~ s/^.*_[^_]*_([^_]*)_[^_]*\.root$/$1/;
        if (!(defined $filesToKeep{$jobNumber})
         || $filesToKeep{$jobNumber} < $submissionNumber)
          {
            $filesToKeep{$jobNumber} = $submissionNumber;
          }
      }
  }
if ($opt{"destination"})
  {
    mkdir $opt{"destination"} or die "Failed to create destination directory, stopped" if !(-e $opt{"destination"});
    die "Destination is not a directory, stopped" if !(-d $opt{"destination"});
    open (TEST_FILE, ">$opt{'destination'}/.crabCleanTest") or die "Unable to write to destination directory, stopped";
    close (TEST_FILE);
    unlink "$opt{'destination'}/.crabCleanTest";
  }
opendir (CRAB_OUTPUT, "$outputDir");
my @crabOutput = readdir (CRAB_OUTPUT);
closedir (CRAB_OUTPUT);
my $nFiles = @crabOutput - 2;
print "Before cleaning: $nFiles files\n";
foreach my $file (@crabOutput)
  {
    next if ($file eq "." || $file eq "..");
    next if !($file =~ m/^.*_[^_]*_[^_]*_[^_]*\.root$/);
    my $jobNumber = $file;
    my $submissionNumber = $file;
    $jobNumber =~ s/^.*_([^_]*)_[^_]*_[^_]*\.root$/$1/;
    $submissionNumber =~ s/^.*_[^_]*_([^_]*)_[^_]*\.root$/$1/;
    if (!(defined $filesToKeep{$jobNumber}))
      {
        print "No information found for job $jobNumber!\n";
      }
    elsif ((($opt{"dumb"} && $filesToKeep{$jobNumber} != $submissionNumber)
        || (!$opt{"dumb"} && $filesToKeep{$jobNumber} ne $file))
        && !$opt{"destination"})
      {
        system ("rm -f $outputDir/$file") if $opt{"force"};
        system ("rm -i $outputDir/$file") if !$opt{"force"};
      }
    elsif ((($opt{"dumb"} && $filesToKeep{$jobNumber} == $submissionNumber)
        || (!$opt{"dumb"} && $filesToKeep{$jobNumber} eq $file))
        && $opt{"destination"})
      {
        system ("mv -f $outputDir/$file $opt{'destination'}/$file") if $opt{"force"};
        system ("mv -i $outputDir/$file $opt{'destination'}/$file") if !$opt{"force"};
      }
  }
opendir (CRAB_OUTPUT, "$outputDir");
my @crabOutput = readdir (CRAB_OUTPUT);
closedir (CRAB_OUTPUT);
opendir (CRAB_OUTPUT, "$opt{'destination'}");
my @crabOutputInDestination = readdir (CRAB_OUTPUT);
closedir (CRAB_OUTPUT);
my $nFiles = @crabOutput - 2;
$nFiles = @crabOutputInDestination - 2 if $opt{"destination"};
print "After cleaning:  $nFiles files\n";

sub
printHelp
{
  my $exeName = $0;
  $exeName =~ s/^.*\/([^\/]*)$/$1/;

  print "Usage: $exeName [OPTION]... CRAB_DIR OUTPUT_DIR\n";
  print "  or: $exeName -d [OPTION]... OUTPUT_DIR\n";
  print "  or: $exeName -s [OPTION]... DATASET\n";
  print "Deletes excess EDM output caused by multiple CRAB submissions using the results\n";
  print "contained in the CRAB working directory provided.\n";
  print "\n";
  print "Mandatory arguments to long options are mandatory for short options too.\n";
  printf "%-29s%s\n", "  -d, --dumb", "tries to use the filenames of the output instead of";
  printf "%-29s%s\n", "            ", "the results from the CRAB working directory; useful";
  printf "%-29s%s\n", "            ", "if the working directory has been lost";
  printf "%-29s%s\n", "  -f, --force", "never prompt (default is to prompt before any";
  printf "%-29s%s\n", "             ", "removal)";
  printf "%-29s%s\n", "  -h, --help", "print this help message";
  printf "%-29s%s\n", "  -o, --destination DIR", "do not remove excess output and move the good files";
  printf "%-29s%s\n", " ", "to DIR";
  printf "%-29s%s\n", "  -s, --dataset", "tells the script to expect a dataset name instead";
  printf "%-29s%s\n", "               ", "of a directory";

  exit;
}

sub
parseCrabDir
{
  my $crabDir = shift;
  my $filesToKeep = shift;

  opendir (CRAB_RESULTS, "$crabDir/res");
  my @crabResults = readdir (CRAB_RESULTS);
  closedir (CRAB_RESULTS);
  for my $file (@crabResults)
    {
      next if !($file =~ m/^crab_fjr_[^\.]*\.xml$/);
      my $jobNumber = $file;
      $jobNumber =~ s/^crab_fjr_([^\.]*)\.xml$/$1/;
      my @status = `grep 'ExitStatus' $crabDir/res/$file`;
      my $wrapperStatus = $status[0];
      $wrapperStatus =~ s/^.*ExitStatus="([^"]*)".*$/$1/;
      $wrapperStatus =~ s/\n//g;
      if ($wrapperStatus != 0)
        {
          print "Job $jobNumber failed (return value $wrapperStatus). Skipping.\n";
          next;
        }
      my @surlForGrid = `grep -A 1 '<SurlForGrid' $crabDir/res/$file`;
      my $fileName = $surlForGrid[1];
      $fileName = $surlForGrid[0] if $surlForGrid[0] =~ m/Value=/;
      $fileName =~ s/^.*\/([^\/]*)\n/$1/ if !($fileName =~ m/Value=/);
      $fileName =~ s/^.*\/([^\/]*)"\/>\n/$1/ if $fileName =~ m/Value=/;
      $filesToKeep->{$jobNumber} = $fileName;
    }
}
Revision:	1.9
Committed:	Wed Aug 7 18:46:11 2013 UTC (11 years, 8 months ago) by ahart
Branch:	MAIN
Changes since 1.8:	+30 -24 lines
Log Message:	Added an option for moving the good files to a separate directory instead of removing the bad files from the original output directory.
#	User	Rev	Content
1	ahart	1.1	#!/usr/bin/env perl
2
3			use strict;
4			use Getopt::Long;
5	ahart	1.4	use Mysql;
6	ahart	1.7	use Term::ANSIColor;
7	ahart	1.1
8	ahart	1.2	sub printHelp;
9			sub parseCrabDir;
10
11	ahart	1.4	our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
12
13	ahart	1.1	my %opt;
14			Getopt::Long::Configure ("bundling");
15	ahart	1.9	GetOptions (\%opt, "dumb\|d", "force\|f", "destination\|o=s", "dataset\|s", "help\|h");
16	ahart	1.1
17	ahart	1.4	printHelp () if $opt{"help"} \|\| !$ARGV[0] \|\| (!$opt{"dumb"} && !$opt{"dataset"} && !$ARGV[1]);
18	ahart	1.2	my $crabDir = $ARGV[0];
19			my $outputDir = $ARGV[0];
20	ahart	1.4	my $dataset = $ARGV[0];
21			$outputDir = $ARGV[1] if !$opt{"dumb"} && !$opt{"dataset"};
22
23			if ($opt{"dataset"})
24			{
25	ahart	1.6	my $queryDataset = $dataset;
26			$queryDataset =~ s/\*/%/g;
27			$queryDataset =~ s/(.*)/%$1%/g;
28	ahart	1.7	my $query = "select workingDirectory,location,dataset,user,creationTime,status from ntuple where dataset like '$queryDataset' order by creationTime";
29	ahart	1.4	$db->selectdb ("ntuple");
30			my $results = $db->query ($query);
31			if ($results->numrows () == 1)
32			{
33			my @row = $results->fetchrow ();
34			$crabDir = "$row[0]/ntuple";
35			$outputDir = $row[1];
36			}
37	ahart	1.9	die "Database entry does not exist, stopped" if $results->numrows () == 0;
38	ahart	1.5	if ($results->numrows () > 1)
39	ahart	1.4	{
40	ahart	1.5	my %workingDir;
41			my %location;
42			print "Found multiple database entries matching\n";
43			print "\"$dataset\":\n";
44			for (my $i = 1; $i <= $results->numrows (); $i++)
45			{
46			my @row = $results->fetchrow ();
47	ahart	1.7	$row[4] =~ s/([^ ]) [^ ]/$1/g;
48	ahart	1.5	$workingDir{"$i"} = $row[0];
49			$location{"$i"} = $row[1];
50			printf "(%2d) $row[2]\n", $i;
51	ahart	1.7	print " (";
52			print color "green" if $row[5] eq "present";
53			print color "bold yellow" if $row[5] eq "submitted";
54			print color "bold red" if $row[5] eq "created" or $row[5] eq "cancelled" or $row[5] eq "deprecated";
55			print $row[5];
56			print color "reset";
57			print ") created by $row[3] on $row[4]\n";
58	ahart	1.5	}
59			print "\nWhich entry would you like to use?: ";
60			my $response = <STDIN>;
61			$response =~ s/[ \t\n]//g;
62	ahart	1.9	die "Your selection was not a valid option, stopped" if !(exists $workingDir{$response});
63	ahart	1.5	$crabDir = "$workingDir{$response}/ntuple";
64			$outputDir = $location{$response};
65	ahart	1.4	}
66			}
67
68	ahart	1.9	die "Output directory does not exist, stopped" if !(-e $outputDir) \|\| !(-d $outputDir);
69	ahart	1.2	opendir (CRAB_OUTPUT, "$outputDir");
70	ahart	1.1	my @crabOutput = readdir (CRAB_OUTPUT);
71			closedir (CRAB_OUTPUT);
72			my %filesToKeep;
73	ahart	1.2	if (!$opt{"dumb"})
74			{
75	ahart	1.9	die "CRAB directory does not exist, stopped" if !(-e $crabDir) \|\| !(-d $crabDir);
76	ahart	1.2	parseCrabDir ($crabDir, \%filesToKeep);
77			}
78			else
79	ahart	1.1	{
80	ahart	1.2	foreach my $file (@crabOutput)
81	ahart	1.1	{
82	ahart	1.2	next if ($file eq "." \|\| $file eq "..");
83			next if !($file =~ m/^._[^_]_[^_]_[^_]\.root$/);
84			my $jobNumber = $file;
85			my $submissionNumber = $file;
86			$jobNumber =~ s/^._([^_])_[^_]_[^_]\.root$/$1/;
87			$submissionNumber =~ s/^._[^_]_([^_])_[^_]\.root$/$1/;
88			if (!(defined $filesToKeep{$jobNumber})
89			\|\| $filesToKeep{$jobNumber} < $submissionNumber)
90			{
91			$filesToKeep{$jobNumber} = $submissionNumber;
92			}
93	ahart	1.1	}
94			}
95	ahart	1.9	if ($opt{"destination"})
96			{
97			mkdir $opt{"destination"} or die "Failed to create destination directory, stopped" if !(-e $opt{"destination"});
98			die "Destination is not a directory, stopped" if !(-d $opt{"destination"});
99			open (TEST_FILE, ">$opt{'destination'}/.crabCleanTest") or die "Unable to write to destination directory, stopped";
100			close (TEST_FILE);
101			unlink "$opt{'destination'}/.crabCleanTest";
102			}
103	ahart	1.7	opendir (CRAB_OUTPUT, "$outputDir");
104			my @crabOutput = readdir (CRAB_OUTPUT);
105			closedir (CRAB_OUTPUT);
106			my $nFiles = @crabOutput - 2;
107			print "Before cleaning: $nFiles files\n";
108	ahart	1.1	foreach my $file (@crabOutput)
109			{
110			next if ($file eq "." \|\| $file eq "..");
111			next if !($file =~ m/^._[^_]_[^_]_[^_]\.root$/);
112			my $jobNumber = $file;
113			my $submissionNumber = $file;
114			$jobNumber =~ s/^._([^_])_[^_]_[^_]\.root$/$1/;
115			$submissionNumber =~ s/^._[^_]_([^_])_[^_]\.root$/$1/;
116	ahart	1.2	if (!(defined $filesToKeep{$jobNumber}))
117			{
118			print "No information found for job $jobNumber!\n";
119			}
120	ahart	1.9	elsif ((($opt{"dumb"} && $filesToKeep{$jobNumber} != $submissionNumber)
121	ahart	1.2	\|\| (!$opt{"dumb"} && $filesToKeep{$jobNumber} ne $file))
122	ahart	1.9	&& !$opt{"destination"})
123	ahart	1.1	{
124	ahart	1.2	system ("rm -f $outputDir/$file") if $opt{"force"};
125			system ("rm -i $outputDir/$file") if !$opt{"force"};
126	ahart	1.1	}
127	ahart	1.9	elsif ((($opt{"dumb"} && $filesToKeep{$jobNumber} == $submissionNumber)
128			\|\| (!$opt{"dumb"} && $filesToKeep{$jobNumber} eq $file))
129			&& $opt{"destination"})
130			{
131			system ("mv -f $outputDir/$file $opt{'destination'}/$file") if $opt{"force"};
132			system ("mv -i $outputDir/$file $opt{'destination'}/$file") if !$opt{"force"};
133			}
134	ahart	1.1	}
135	ahart	1.7	opendir (CRAB_OUTPUT, "$outputDir");
136			my @crabOutput = readdir (CRAB_OUTPUT);
137			closedir (CRAB_OUTPUT);
138	ahart	1.9	opendir (CRAB_OUTPUT, "$opt{'destination'}");
139			my @crabOutputInDestination = readdir (CRAB_OUTPUT);
140			closedir (CRAB_OUTPUT);
141	ahart	1.7	my $nFiles = @crabOutput - 2;
142	ahart	1.9	$nFiles = @crabOutputInDestination - 2 if $opt{"destination"};
143	ahart	1.7	print "After cleaning: $nFiles files\n";
144	ahart	1.1
145			sub
146			printHelp
147			{
148			my $exeName = $0;
149			$exeName =~ s/^.\/([^\/])$/$1/;
150
151	ahart	1.2	print "Usage: $exeName [OPTION]... CRAB_DIR OUTPUT_DIR\n";
152	ahart	1.3	print " or: $exeName -d [OPTION]... OUTPUT_DIR\n";
153	ahart	1.4	print " or: $exeName -s [OPTION]... DATASET\n";
154	ahart	1.2	print "Deletes excess EDM output caused by multiple CRAB submissions using the results\n";
155			print "contained in the CRAB working directory provided.\n";
156	ahart	1.1	print "\n";
157			print "Mandatory arguments to long options are mandatory for short options too.\n";
158	ahart	1.2	printf "%-29s%s\n", " -d, --dumb", "tries to use the filenames of the output instead of";
159			printf "%-29s%s\n", " ", "the results from the CRAB working directory; useful";
160			printf "%-29s%s\n", " ", "if the working directory has been lost";
161	ahart	1.1	printf "%-29s%s\n", " -f, --force", "never prompt (default is to prompt before any";
162			printf "%-29s%s\n", " ", "removal)";
163			printf "%-29s%s\n", " -h, --help", "print this help message";
164	ahart	1.9	printf "%-29s%s\n", " -o, --destination DIR", "do not remove excess output and move the good files";
165			printf "%-29s%s\n", " ", "to DIR";
166	ahart	1.4	printf "%-29s%s\n", " -s, --dataset", "tells the script to expect a dataset name instead";
167			printf "%-29s%s\n", " ", "of a directory";
168	ahart	1.1
169			exit;
170			}
171	ahart	1.2
172			sub
173			parseCrabDir
174			{
175			my $crabDir = shift;
176			my $filesToKeep = shift;
177
178			opendir (CRAB_RESULTS, "$crabDir/res");
179			my @crabResults = readdir (CRAB_RESULTS);
180			closedir (CRAB_RESULTS);
181			for my $file (@crabResults)
182			{
183			next if !($file =~ m/^crab_fjr_[^\.]*\.xml$/);
184			my $jobNumber = $file;
185			$jobNumber =~ s/^crab_fjr_([^\.]*)\.xml$/$1/;
186			my @status = `grep 'ExitStatus' $crabDir/res/$file`;
187			my $wrapperStatus = $status[0];
188			$wrapperStatus =~ s/^.ExitStatus="([^"])".*$/$1/;
189	ahart	1.3	$wrapperStatus =~ s/\n//g;
190	ahart	1.2	if ($wrapperStatus != 0)
191			{
192			print "Job $jobNumber failed (return value $wrapperStatus). Skipping.\n";
193			next;
194			}
195	ahart	1.8	my @surlForGrid = `grep -A 1 '<SurlForGrid' $crabDir/res/$file`;
196	ahart	1.2	my $fileName = $surlForGrid[1];
197	ahart	1.9	$fileName = $surlForGrid[0] if $surlForGrid[0] =~ m/Value=/;
198	ahart	1.8	$fileName =~ s/^.\/([^\/])\n/$1/ if !($fileName =~ m/Value=/);
199	ahart	1.9	$fileName =~ s/^.\/([^\/])"\/>\n/$1/ if $fileName =~ m/Value=/;
200	ahart	1.2	$filesToKeep->{$jobNumber} = $fileName;
201			}
202			}