[ViewVC] Annotation of: cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osucrab

#!/usr/bin/env perl

use strict;
use Mysql;

sub getWorkingDir;
sub dbUpdate;
sub dbReport;
sub dbInsert;
sub addSlashes;
sub getOption;

our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";

my $arg = join (" ", @ARGV);
if ($arg =~ m/-create/)
  {
    my $crabCfgName = $arg;
    $crabCfgName =~ s/.*-cfg  *([^ \t\n]*).*/$1/;
    open (CRAB_CFG, "<$crabCfgName");
    my @crabCfg = <CRAB_CFG>;
    close (CRAB_CFG);
    my $dataset = getOption (\@crabCfg, "datasetpath");
    my $psetName = getOption (\@crabCfg, "pset");
    my $jsonFileName = getOption (\@crabCfg, "lumi_mask");
    my $submissionLogName = getOption (\@crabCfg, "ui_working_dir");
    $submissionLogName .= "/log/crab.log";

    print "Ntuple format (BEAN): ";
    my $format = <STDIN>;
    $format =~ s/\n//g;
    $format = "BEAN" if !$format;
    print "Comment for the ntuple database: ";
    my $comment = <STDIN>;
    $comment =~ s/\n//g;

    system ("crab $arg");

    my $status = "created";
    $status = "submitted" if $arg =~ m/-submit/;
    my $results = dbInsert ($dataset, $ENV{"USER"}, $format, "", 0, 0, $status, $comment, $psetName, $crabCfgName, $jsonFileName, $submissionLogName);
  }
elsif ($arg =~ m/-submit/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }
    my $submissionLogName = $workingDir . "/log/crab.log";

    system ("crab $arg");

    my $results = dbUpdate ($id, "submitted", $submissionLogName) if $id > -1;
  }
elsif ($arg =~ m/-report/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        my $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }
    my $lumiSummaryName = $workingDir . "/res/lumiSummary.json";

    system ("crab $arg");

    my $results = dbReport ($id, "complete", $lumiSummaryName) if $id > -1;
  }
else
  {
    system ("crab $arg");
  }

sub
getWorkingDir
{
  my $argv = shift;

  my $workingDir;
  my $nextToLast = 0;
  foreach my $argument (@$argv)
    {
      if ($nextToLast)
        {
          $workingDir = $argument;
          last;
        }
      $nextToLast = 1 if ($argument eq "-c" || $argument eq "-continue");
    }

  return $workingDir;
}

sub
getOption
{
  my $file = shift;
  my $option = shift;

  my $optionValue;
  foreach my $line (@$file)
    {
      next if (!($line =~ m/$option  *=  *[^ \t\n]*/));
      $optionValue = $line;
      $optionValue =~ s/.*$option  *=  *([^ \t\n]*).*\n/$1/;
    }

  return $optionValue;
}

sub
addSlashes
{
  my $string = shift;

  $string =~ s/\\/\\\\/g;
  $string =~ s/'/\\'/g;
  $string =~ s/"/\\"/g;
  $string =~ s/\\0/\\\\0/g;

  return $string;
}

sub
dbInsert
{
  my $dataset = shift;
  my $user = shift;
  my $format = shift;
  my $location = shift;
  my $nFiles = shift;
  my $sizeInGB = shift;
  my $status = shift;
  my $comment = shift;
  my $psetName = shift;
  my $crabCfgName = shift;
  my $jsonFileName = shift;
  my $submissionLogName = shift;

  open (PY_CONFIG, "<$psetName");
  my @pset = <PY_CONFIG>;
  close (PY_CONFIG);
  my $pset = join ("", @pset);
  open (CRAB_CONFIG, "<$crabCfgName");
  my @crabCfg = <CRAB_CONFIG>;
  close (CRAB_CONFIG);
  my $crabCfg = join ("", @crabCfg);
  open (JSON_FILE, "<$jsonFileName");
  my @jsonFile = <JSON_FILE>;
  close (JSON_FILE);
  my $jsonFile = join ("", @jsonFile);
  open (SUBMISSION_LOG, "<$submissionLogName");
  my @submissionLog = <SUBMISSION_LOG>;
  close (SUBMISSION_LOG);
  my $submissionLog = join ("", @submissionLog);

  $dataset = addSlashes ($dataset);
  $user = addSlashes ($user);
  $format = addSlashes ($format);
  $pset = addSlashes ($pset);
  $crabCfg = addSlashes ($crabCfg);
  $jsonFile = addSlashes ($jsonFile);
  $submissionLog = addSlashes ($submissionLog);
  $location = addSlashes ($location);
  $status = addSlashes ($status);
  $comment = addSlashes ($comment);

  my $results;
  my $query = "select max(id) from ntuple";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);
  my @row = $results->fetchrow ();
  my $id = 1;
  $id = $row[0] + 1 if $results->numrows ();

  my $workingDir = $submissionLogName;
  $workingDir =~ s/(.*)\/log\/crab\.log/$1/;
  open (ID_FILE, ">$workingDir/log/.osuID");
  print ID_FILE "$id\n";
  close (ID_FILE);

  $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, user, format, location, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, submissionLog) values ($id, '$dataset', now(), now(), '$user', '$format', '$location', $nFiles, $sizeInGB, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$submissionLog')";
  $results = $db->query ($query);

  return $results;
}

sub
dbUpdate
{
  my $id = shift;
  my $status = shift;
  my $submissionLogName = shift;

  open (SUBMISSION_LOG, "<$submissionLogName");
  my @submissionLog = <SUBMISSION_LOG>;
  close (SUBMISSION_LOG);
  my $submissionLog = join ("", @submissionLog);

  $submissionLog = addSlashes ($submissionLog);
  $status = addSlashes ($status);

  my $results;
  my $query = "update ntuple set lastUpdateTime=now(), status='$status', submissionLog='$submissionLog' where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}

sub
dbReport
{
  my $id = shift;
  my $status = shift;
  my $lumiSummaryName = shift;

  open (LUMI_SUMMARY, "<$lumiSummaryName");
  my @lumiSummary = <LUMI_SUMMARY>;
  close (LUMI_SUMMARY);
  my $lumiSummary = join ("", @lumiSummary);

  $lumiSummary = addSlashes ($lumiSummary);
  $status = addSlashes ($status);

  my $results;
  my $query = "update ntuple set lastUpdateTime=now(), status='$status', lumiSummary='$lumiSummary' where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}
Revision:	1.3
Committed:	Tue Aug 7 18:10:18 2012 UTC (12 years, 9 months ago) by ahart
Branch:	MAIN
CVS Tags:	V00-00-03
Changes since 1.2:	+18 -10 lines
Log Message:	Check for existence of the ID file before proceeding.
#	User	Rev	Content
1	ahart	1.1	#!/usr/bin/env perl
2
3			use strict;
4			use Mysql;
5
6			sub getWorkingDir;
7			sub dbUpdate;
8			sub dbReport;
9			sub dbInsert;
10			sub addSlashes;
11			sub getOption;
12
13	ahart	1.2	our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
14	ahart	1.1
15			my $arg = join (" ", @ARGV);
16			if ($arg =~ m/-create/)
17			{
18			my $crabCfgName = $arg;
19			$crabCfgName =~ s/.-cfg ([^ \t\n])./$1/;
20			open (CRAB_CFG, "<$crabCfgName");
21			my @crabCfg = <CRAB_CFG>;
22			close (CRAB_CFG);
23			my $dataset = getOption (\@crabCfg, "datasetpath");
24			my $psetName = getOption (\@crabCfg, "pset");
25			my $jsonFileName = getOption (\@crabCfg, "lumi_mask");
26			my $submissionLogName = getOption (\@crabCfg, "ui_working_dir");
27			$submissionLogName .= "/log/crab.log";
28
29			print "Ntuple format (BEAN): ";
30			my $format = <STDIN>;
31			$format =~ s/\n//g;
32			$format = "BEAN" if !$format;
33			print "Comment for the ntuple database: ";
34			my $comment = <STDIN>;
35			$comment =~ s/\n//g;
36
37			system ("crab $arg");
38
39			my $status = "created";
40			$status = "submitted" if $arg =~ m/-submit/;
41			my $results = dbInsert ($dataset, $ENV{"USER"}, $format, "", 0, 0, $status, $comment, $psetName, $crabCfgName, $jsonFileName, $submissionLogName);
42			}
43			elsif ($arg =~ m/-submit/)
44			{
45			my $workingDir = getWorkingDir (\@ARGV);
46	ahart	1.3	my $id = -1;
47			if (-e "$workingDir/log/.osuID")
48			{
49			open (ID_FILE, "<$workingDir/log/.osuID");
50			$id = <ID_FILE>;
51			close (ID_FILE);
52			$id =~ s/(.*)\n/$1/;
53			}
54	ahart	1.1	my $submissionLogName = $workingDir . "/log/crab.log";
55
56			system ("crab $arg");
57
58	ahart	1.3	my $results = dbUpdate ($id, "submitted", $submissionLogName) if $id > -1;
59	ahart	1.1	}
60			elsif ($arg =~ m/-report/)
61			{
62			my $workingDir = getWorkingDir (\@ARGV);
63	ahart	1.3	my $id = -1;
64			if (-e "$workingDir/log/.osuID")
65			{
66			open (ID_FILE, "<$workingDir/log/.osuID");
67			my $id = <ID_FILE>;
68			close (ID_FILE);
69			$id =~ s/(.*)\n/$1/;
70			}
71	ahart	1.1	my $lumiSummaryName = $workingDir . "/res/lumiSummary.json";
72
73			system ("crab $arg");
74
75	ahart	1.3	my $results = dbReport ($id, "complete", $lumiSummaryName) if $id > -1;
76	ahart	1.1	}
77			else
78			{
79			system ("crab $arg");
80			}
81
82			sub
83			getWorkingDir
84			{
85			my $argv = shift;
86
87			my $workingDir;
88			my $nextToLast = 0;
89			foreach my $argument (@$argv)
90			{
91			if ($nextToLast)
92			{
93			$workingDir = $argument;
94			last;
95			}
96			$nextToLast = 1 if ($argument eq "-c" \|\| $argument eq "-continue");
97			}
98
99			return $workingDir;
100			}
101
102			sub
103			getOption
104			{
105			my $file = shift;
106			my $option = shift;
107
108			my $optionValue;
109			foreach my $line (@$file)
110			{
111			next if (!($line =~ m/$option = [^ \t\n]*/));
112			$optionValue = $line;
113			$optionValue =~ s/.$option = ([^ \t\n]).*\n/$1/;
114			}
115
116			return $optionValue;
117			}
118
119			sub
120			addSlashes
121			{
122			my $string = shift;
123
124			$string =~ s/\\/\\\\/g;
125			$string =~ s/'/\\'/g;
126			$string =~ s/"/\\"/g;
127			$string =~ s/\\0/\\\\0/g;
128
129			return $string;
130			}
131
132			sub
133			dbInsert
134			{
135			my $dataset = shift;
136			my $user = shift;
137			my $format = shift;
138			my $location = shift;
139			my $nFiles = shift;
140			my $sizeInGB = shift;
141			my $status = shift;
142			my $comment = shift;
143			my $psetName = shift;
144			my $crabCfgName = shift;
145			my $jsonFileName = shift;
146			my $submissionLogName = shift;
147
148			open (PY_CONFIG, "<$psetName");
149			my @pset = <PY_CONFIG>;
150			close (PY_CONFIG);
151			my $pset = join ("", @pset);
152			open (CRAB_CONFIG, "<$crabCfgName");
153			my @crabCfg = <CRAB_CONFIG>;
154			close (CRAB_CONFIG);
155			my $crabCfg = join ("", @crabCfg);
156			open (JSON_FILE, "<$jsonFileName");
157			my @jsonFile = <JSON_FILE>;
158			close (JSON_FILE);
159			my $jsonFile = join ("", @jsonFile);
160			open (SUBMISSION_LOG, "<$submissionLogName");
161			my @submissionLog = <SUBMISSION_LOG>;
162			close (SUBMISSION_LOG);
163			my $submissionLog = join ("", @submissionLog);
164
165			$dataset = addSlashes ($dataset);
166			$user = addSlashes ($user);
167			$format = addSlashes ($format);
168			$pset = addSlashes ($pset);
169			$crabCfg = addSlashes ($crabCfg);
170			$jsonFile = addSlashes ($jsonFile);
171			$submissionLog = addSlashes ($submissionLog);
172			$location = addSlashes ($location);
173			$status = addSlashes ($status);
174			$comment = addSlashes ($comment);
175
176			my $results;
177			my $query = "select max(id) from ntuple";
178			$db->selectdb ("ntuple");
179			$results = $db->query ($query);
180			my @row = $results->fetchrow ();
181			my $id = 1;
182			$id = $row[0] + 1 if $results->numrows ();
183
184			my $workingDir = $submissionLogName;
185			$workingDir =~ s/(.*)\/log\/crab\.log/$1/;
186			open (ID_FILE, ">$workingDir/log/.osuID");
187			print ID_FILE "$id\n";
188			close (ID_FILE);
189
190			$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, user, format, location, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, submissionLog) values ($id, '$dataset', now(), now(), '$user', '$format', '$location', $nFiles, $sizeInGB, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$submissionLog')";
191			$results = $db->query ($query);
192
193			return $results;
194			}
195
196			sub
197			dbUpdate
198			{
199			my $id = shift;
200			my $status = shift;
201			my $submissionLogName = shift;
202
203			open (SUBMISSION_LOG, "<$submissionLogName");
204			my @submissionLog = <SUBMISSION_LOG>;
205			close (SUBMISSION_LOG);
206			my $submissionLog = join ("", @submissionLog);
207
208			$submissionLog = addSlashes ($submissionLog);
209			$status = addSlashes ($status);
210
211			my $results;
212			my $query = "update ntuple set lastUpdateTime=now(), status='$status', submissionLog='$submissionLog' where id=$id";
213			$db->selectdb ("ntuple");
214			$results = $db->query ($query);
215
216			return $results;
217			}
218
219			sub
220			dbReport
221			{
222			my $id = shift;
223			my $status = shift;
224			my $lumiSummaryName = shift;
225
226			open (LUMI_SUMMARY, "<$lumiSummaryName");
227			my @lumiSummary = <LUMI_SUMMARY>;
228			close (LUMI_SUMMARY);
229			my $lumiSummary = join ("", @lumiSummary);
230
231			$lumiSummary = addSlashes ($lumiSummary);
232			$status = addSlashes ($status);
233
234			my $results;
235			my $query = "update ntuple set lastUpdateTime=now(), status='$status', lumiSummary='$lumiSummary' where id=$id";
236			$db->selectdb ("ntuple");
237			$results = $db->query ($query);
238
239			return $results;
240			}