[ViewVC] Annotation of: cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osucrab

#!/usr/bin/env perl

use strict;
use Mysql;

sub getWorkingDir;
sub dbUpdate;
sub dbStatus;
sub dbReport;
sub dbInsert;
sub addSlashes;
sub getOption;
sub deleteEntry;

our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";

my $arg = join (" ", @ARGV);
if ($arg =~ m/-create/)
  {
    my $crabCfgName = $arg;
    $crabCfgName =~ s/.*-cfg  *([^ \t\n]*).*/$1/;
    open (CRAB_CFG, "<$crabCfgName");
    my @crabCfg = <CRAB_CFG>;
    close (CRAB_CFG);
    my $dataset = getOption (\@crabCfg, "datasetpath");
    my $psetName = getOption (\@crabCfg, "pset");
    my $jsonFileName = getOption (\@crabCfg, "lumi_mask");
    my $workingDir = getOption (\@crabCfg, "ui_working_dir");
    my $publishName = "";
    $publishName = getOption (\@crabCfg, "publish_data_name") if getOption (\@crabCfg, "publish_data") eq "1";
    my $stageOutLocation = getOption (\@crabCfg, "storage_element");

    print "Ntuple format (BEAN): ";
    my $format = <STDIN>;
    $format =~ s/\n//g;
    $format = "BEAN" if !$format;
    print "Comment for the ntuple database: ";
    my $comment = <STDIN>;
    $comment =~ s/\n//g;

    system ("rm -f .crabSubmit.log ; crab $arg 2>&1 | tee .crabSubmit.log");

    my $status = "created";
    $status = "submitted" if $arg =~ m/-submit/;
    my $hostname = `/bin/hostname`;
    $hostname =~ s/[\f\n\r]//g;
    my $results = dbInsert ($dataset, "$ENV{'USER'}\@$hostname", $format, "", 0, 0, $status, $comment, $psetName, $crabCfgName, $jsonFileName, $workingDir, $publishName, $stageOutLocation);
  }
elsif ($arg =~ m/-submit/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }
    my $logFile = $workingDir . "/log/crabSubmit.log";

    system ("crab $arg 2>&1 | tee -a .crabSubmit.log ; if [ -e \"$workingDir/log/crabSubmit.log\" ] ; then rm -f .crabSubmit.log ; else mv -f .crabSubmit.log $workingDir/log/crabSubmit.log ; fi");

    my $hostname = `/bin/hostname`;
    $hostname =~ s/[\f\n\r]//g;
    my $results = dbUpdate ($id, "submitted", "$ENV{'USER'}\@$hostname", $logFile) if $id > -1;
  }
elsif ($arg =~ m/-report/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }
    my $lumiSummaryName = $workingDir . "/res/lumiSummary.json";

    system ("crab $arg");

    my $hostname = `/bin/hostname`;
    $hostname =~ s/[\f\n\r]//g;
    my $results = dbReport ($id, $lumiSummaryName, "$ENV{'USER'}\@$hostname") if $id > -1;
  }
elsif ($arg =~ m/-status/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }
    my $currentStatusName = "$workingDir/log/.currentStatus";
    unlink ($currentStatusName) if -e $currentStatusName;
    system ("crab $arg | tee $currentStatusName");
    my $hostname = `/bin/hostname`;
    $hostname =~ s/[\f\n\r]//g;
    my $results = dbStatus ($id, $currentStatusName, "$ENV{'USER'}\@$hostname") if $id > -1;
    unlink ($currentStatusName);
  }
elsif ($arg =~ m/-kill/)
  {
    my $workingDir = getWorkingDir (\@ARGV);
    my $id = -1;
    if (-e "$workingDir/log/.osuID")
      {
        open (ID_FILE, "<$workingDir/log/.osuID");
        $id = <ID_FILE>;
        close (ID_FILE);
        $id =~ s/(.*)\n/$1/;
      }

    print "Are you cancelling this task? (y/N): ";
    my $response = <STDIN>;
    $response =~ s/\n//g;
    deleteEntry ($id) if lc ($response) eq "y";
    system ("crab $arg");
  }
else
  {
    system ("crab $arg");
  }

sub
getWorkingDir
{
  my $argv = shift;

  my $workingDir;
  my $nextToLast = 0;
  foreach my $argument (@$argv)
    {
      if ($nextToLast)
        {
          $workingDir = $argument;
          last;
        }
      $nextToLast = 1 if ($argument eq "-c" || $argument eq "-continue");
    }

  return $workingDir;
}

sub
getOption
{
  my $file = shift;
  my $option = shift;

  my $optionValue;
  foreach my $line (@$file)
    {
      next if ($line =~ m/^[ \t\n]*#/);
      next if (!($line =~ m/$option  *=  *[^ \t\n]*/));
      $optionValue = $line;
      $optionValue =~ s/.*$option  *=  *([^ \t\n]*).*\n/$1/;
    }

  return $optionValue;
}

sub
addSlashes
{
  my $string = shift;

  $string =~ s/\\/\\\\/g;
  $string =~ s/'/\\'/g;
  $string =~ s/"/\\"/g;
  $string =~ s/\\0/\\\\0/g;

  return $string;
}

sub
dbInsert
{
  my $dataset = shift;
  my $userAndHost = shift;
  my $format = shift;
  my $location = shift;
  my $nFiles = shift;
  my $sizeInGB = shift;
  my $status = shift;
  my $comment = shift;
  my $psetName = shift;
  my $crabCfgName = shift;
  my $jsonFileName = shift;
  my $workingDir = shift;
  my $publishName = shift;
  my $stageOutLocation = shift;

  my $user = $userAndHost;
  $user =~ s/@.*$//;

  open (PY_CONFIG, "<$psetName");
  my @pset = <PY_CONFIG>;
  close (PY_CONFIG);
  my $pset = join ("", @pset);
  open (CRAB_CONFIG, "<$crabCfgName");
  my @crabCfg = <CRAB_CONFIG>;
  close (CRAB_CONFIG);
  my $crabCfg = join ("", @crabCfg);
  open (JSON_FILE, "<$jsonFileName");
  my @jsonFile = <JSON_FILE>;
  close (JSON_FILE);
  my $jsonFile = join ("", @jsonFile);

  $dataset = addSlashes ($dataset);
  $user = addSlashes ($user);
  $format = addSlashes ($format);
  $pset = addSlashes ($pset);
  $crabCfg = addSlashes ($crabCfg);
  $jsonFile = addSlashes ($jsonFile);
  $location = addSlashes ($location);
  $status = addSlashes ($status);
  $comment = addSlashes ($comment);
  $publishName = addSlashes ($publishName);
  $stageOutLocation = addSlashes ($stageOutLocation);

  my $results;
  my $query = "select max(id) from ntuple";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);
  my @row = $results->fetchrow ();
  my $id = 1;
  $id = $row[0] + 1 if $results->numrows ();

  open (ID_FILE, ">$workingDir/log/.osuID");
  print ID_FILE "$id\n";
  close (ID_FILE);

  $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, publishName, stageOutLocation, published) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$location', $nFiles, $sizeInGB, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$publishName', '$stageOutLocation', 0)";
  $results = $db->query ($query);

  return $results;
}

sub
dbUpdate
{
  my $id = shift;
  my $status = shift;
  my $user = shift;
  my $logName = shift;

  open (LOG, "<$logName");
  my @log = <LOG>;
  close (LOG);
  my $log = join ("", @log);

  $status = addSlashes ($status);
  $user = addSlashes ($user);
  $log = addSlashes ($log);

  my $results;
  my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', status='$status', submissionLog='$log' where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}

sub
dbStatus
{
  my $id = shift;
  my $currentStatusName = shift;
  my $user = shift;

  open (CURRENT_STATUS, "<$currentStatusName");
  my @currentStatus = <CURRENT_STATUS>;
  close (CURRENT_STATUS);
  my $currentStatus = join ("", @currentStatus);

  $currentStatus = addSlashes ($currentStatus);
  $user = addSlashes ($user);

  my $results;
  my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', currentStatus='$currentStatus' where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}

sub
dbReport
{
  my $id = shift;
  my $lumiSummaryName = shift;
  my $user = shift;

  open (LUMI_SUMMARY, "<$lumiSummaryName");
  my @lumiSummary = <LUMI_SUMMARY>;
  close (LUMI_SUMMARY);
  my $lumiSummary = join ("", @lumiSummary);

  $lumiSummary = addSlashes ($lumiSummary);
  $user = addSlashes ($user);

  my $results;
  my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', lumiSummary='$lumiSummary' where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}

sub
deleteEntry
{
  my $id = shift;

  my $results;
  my $query = "delete from ntuple where id=$id";
  $db->selectdb ("ntuple");
  $results = $db->query ($query);

  return $results;
}
Revision:	1.10
Committed:	Thu Aug 1 20:32:49 2013 UTC (11 years, 9 months ago) by ahart
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.9:	+33 -11 lines
Log Message:	Added a few fields which are now in the database and created a submission log which is also uploaded.
#	User	Rev	Content
1	ahart	1.1	#!/usr/bin/env perl
2
3			use strict;
4			use Mysql;
5
6			sub getWorkingDir;
7			sub dbUpdate;
8	ahart	1.4	sub dbStatus;
9	ahart	1.1	sub dbReport;
10			sub dbInsert;
11			sub addSlashes;
12			sub getOption;
13	ahart	1.8	sub deleteEntry;
14	ahart	1.1
15	ahart	1.2	our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
16	ahart	1.1
17			my $arg = join (" ", @ARGV);
18			if ($arg =~ m/-create/)
19			{
20			my $crabCfgName = $arg;
21			$crabCfgName =~ s/.-cfg ([^ \t\n])./$1/;
22			open (CRAB_CFG, "<$crabCfgName");
23			my @crabCfg = <CRAB_CFG>;
24			close (CRAB_CFG);
25			my $dataset = getOption (\@crabCfg, "datasetpath");
26			my $psetName = getOption (\@crabCfg, "pset");
27			my $jsonFileName = getOption (\@crabCfg, "lumi_mask");
28	ahart	1.7	my $workingDir = getOption (\@crabCfg, "ui_working_dir");
29	ahart	1.10	my $publishName = "";
30			$publishName = getOption (\@crabCfg, "publish_data_name") if getOption (\@crabCfg, "publish_data") eq "1";
31			my $stageOutLocation = getOption (\@crabCfg, "storage_element");
32	ahart	1.1
33			print "Ntuple format (BEAN): ";
34			my $format = <STDIN>;
35			$format =~ s/\n//g;
36			$format = "BEAN" if !$format;
37			print "Comment for the ntuple database: ";
38			my $comment = <STDIN>;
39			$comment =~ s/\n//g;
40
41	ahart	1.10	system ("rm -f .crabSubmit.log ; crab $arg 2>&1 \| tee .crabSubmit.log");
42	ahart	1.1
43			my $status = "created";
44			$status = "submitted" if $arg =~ m/-submit/;
45	ahart	1.10	my $hostname = `/bin/hostname`;
46			$hostname =~ s/[\f\n\r]//g;
47			my $results = dbInsert ($dataset, "$ENV{'USER'}\@$hostname", $format, "", 0, 0, $status, $comment, $psetName, $crabCfgName, $jsonFileName, $workingDir, $publishName, $stageOutLocation);
48	ahart	1.1	}
49			elsif ($arg =~ m/-submit/)
50			{
51			my $workingDir = getWorkingDir (\@ARGV);
52	ahart	1.3	my $id = -1;
53			if (-e "$workingDir/log/.osuID")
54			{
55			open (ID_FILE, "<$workingDir/log/.osuID");
56			$id = <ID_FILE>;
57			close (ID_FILE);
58			$id =~ s/(.*)\n/$1/;
59			}
60	ahart	1.10	my $logFile = $workingDir . "/log/crabSubmit.log";
61	ahart	1.1
62	ahart	1.10	system ("crab $arg 2>&1 \| tee -a .crabSubmit.log ; if [ -e \"$workingDir/log/crabSubmit.log\" ] ; then rm -f .crabSubmit.log ; else mv -f .crabSubmit.log $workingDir/log/crabSubmit.log ; fi");
63	ahart	1.1
64	ahart	1.10	my $hostname = `/bin/hostname`;
65			$hostname =~ s/[\f\n\r]//g;
66			my $results = dbUpdate ($id, "submitted", "$ENV{'USER'}\@$hostname", $logFile) if $id > -1;
67	ahart	1.1	}
68			elsif ($arg =~ m/-report/)
69			{
70			my $workingDir = getWorkingDir (\@ARGV);
71	ahart	1.3	my $id = -1;
72			if (-e "$workingDir/log/.osuID")
73			{
74			open (ID_FILE, "<$workingDir/log/.osuID");
75	ahart	1.4	$id = <ID_FILE>;
76	ahart	1.3	close (ID_FILE);
77			$id =~ s/(.*)\n/$1/;
78			}
79	ahart	1.1	my $lumiSummaryName = $workingDir . "/res/lumiSummary.json";
80
81			system ("crab $arg");
82
83	ahart	1.10	my $hostname = `/bin/hostname`;
84			$hostname =~ s/[\f\n\r]//g;
85			my $results = dbReport ($id, $lumiSummaryName, "$ENV{'USER'}\@$hostname") if $id > -1;
86	ahart	1.4	}
87			elsif ($arg =~ m/-status/)
88			{
89			my $workingDir = getWorkingDir (\@ARGV);
90			my $id = -1;
91			if (-e "$workingDir/log/.osuID")
92			{
93			open (ID_FILE, "<$workingDir/log/.osuID");
94			$id = <ID_FILE>;
95			close (ID_FILE);
96			$id =~ s/(.*)\n/$1/;
97			}
98			my $currentStatusName = "$workingDir/log/.currentStatus";
99			unlink ($currentStatusName) if -e $currentStatusName;
100			system ("crab $arg \| tee $currentStatusName");
101	ahart	1.10	my $hostname = `/bin/hostname`;
102			$hostname =~ s/[\f\n\r]//g;
103			my $results = dbStatus ($id, $currentStatusName, "$ENV{'USER'}\@$hostname") if $id > -1;
104	ahart	1.4	unlink ($currentStatusName);
105	ahart	1.1	}
106	ahart	1.8	elsif ($arg =~ m/-kill/)
107			{
108			my $workingDir = getWorkingDir (\@ARGV);
109			my $id = -1;
110			if (-e "$workingDir/log/.osuID")
111			{
112			open (ID_FILE, "<$workingDir/log/.osuID");
113			$id = <ID_FILE>;
114			close (ID_FILE);
115			$id =~ s/(.*)\n/$1/;
116			}
117
118			print "Are you cancelling this task? (y/N): ";
119			my $response = <STDIN>;
120	ahart	1.9	$response =~ s/\n//g;
121	ahart	1.8	deleteEntry ($id) if lc ($response) eq "y";
122			system ("crab $arg");
123			}
124	ahart	1.1	else
125			{
126			system ("crab $arg");
127			}
128
129			sub
130			getWorkingDir
131			{
132			my $argv = shift;
133
134			my $workingDir;
135			my $nextToLast = 0;
136			foreach my $argument (@$argv)
137			{
138			if ($nextToLast)
139			{
140			$workingDir = $argument;
141			last;
142			}
143			$nextToLast = 1 if ($argument eq "-c" \|\| $argument eq "-continue");
144			}
145
146			return $workingDir;
147			}
148
149			sub
150			getOption
151			{
152			my $file = shift;
153			my $option = shift;
154
155			my $optionValue;
156			foreach my $line (@$file)
157			{
158	ahart	1.10	next if ($line =~ m/^[ \t\n]*#/);
159	ahart	1.1	next if (!($line =~ m/$option = [^ \t\n]*/));
160			$optionValue = $line;
161			$optionValue =~ s/.$option = ([^ \t\n]).*\n/$1/;
162			}
163
164			return $optionValue;
165			}
166
167			sub
168			addSlashes
169			{
170			my $string = shift;
171
172			$string =~ s/\\/\\\\/g;
173			$string =~ s/'/\\'/g;
174			$string =~ s/"/\\"/g;
175			$string =~ s/\\0/\\\\0/g;
176
177			return $string;
178			}
179
180			sub
181			dbInsert
182			{
183			my $dataset = shift;
184	ahart	1.6	my $userAndHost = shift;
185	ahart	1.1	my $format = shift;
186			my $location = shift;
187			my $nFiles = shift;
188			my $sizeInGB = shift;
189			my $status = shift;
190			my $comment = shift;
191			my $psetName = shift;
192			my $crabCfgName = shift;
193			my $jsonFileName = shift;
194	ahart	1.7	my $workingDir = shift;
195	ahart	1.10	my $publishName = shift;
196			my $stageOutLocation = shift;
197	ahart	1.1
198	ahart	1.6	my $user = $userAndHost;
199			$user =~ s/@.*$//;
200
201	ahart	1.1	open (PY_CONFIG, "<$psetName");
202			my @pset = <PY_CONFIG>;
203			close (PY_CONFIG);
204			my $pset = join ("", @pset);
205			open (CRAB_CONFIG, "<$crabCfgName");
206			my @crabCfg = <CRAB_CONFIG>;
207			close (CRAB_CONFIG);
208			my $crabCfg = join ("", @crabCfg);
209			open (JSON_FILE, "<$jsonFileName");
210			my @jsonFile = <JSON_FILE>;
211			close (JSON_FILE);
212			my $jsonFile = join ("", @jsonFile);
213
214			$dataset = addSlashes ($dataset);
215			$user = addSlashes ($user);
216			$format = addSlashes ($format);
217			$pset = addSlashes ($pset);
218			$crabCfg = addSlashes ($crabCfg);
219			$jsonFile = addSlashes ($jsonFile);
220			$location = addSlashes ($location);
221			$status = addSlashes ($status);
222			$comment = addSlashes ($comment);
223	ahart	1.10	$publishName = addSlashes ($publishName);
224			$stageOutLocation = addSlashes ($stageOutLocation);
225	ahart	1.1
226			my $results;
227			my $query = "select max(id) from ntuple";
228			$db->selectdb ("ntuple");
229			$results = $db->query ($query);
230			my @row = $results->fetchrow ();
231			my $id = 1;
232			$id = $row[0] + 1 if $results->numrows ();
233
234			open (ID_FILE, ">$workingDir/log/.osuID");
235			print ID_FILE "$id\n";
236			close (ID_FILE);
237
238	ahart	1.10	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, publishName, stageOutLocation, published) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$location', $nFiles, $sizeInGB, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$publishName', '$stageOutLocation', 0)";
239	ahart	1.1	$results = $db->query ($query);
240
241			return $results;
242			}
243
244			sub
245			dbUpdate
246			{
247			my $id = shift;
248			my $status = shift;
249	ahart	1.4	my $user = shift;
250	ahart	1.10	my $logName = shift;
251
252			open (LOG, "<$logName");
253			my @log = <LOG>;
254			close (LOG);
255			my $log = join ("", @log);
256	ahart	1.1
257			$status = addSlashes ($status);
258	ahart	1.4	$user = addSlashes ($user);
259	ahart	1.10	$log = addSlashes ($log);
260	ahart	1.1
261			my $results;
262	ahart	1.10	my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', status='$status', submissionLog='$log' where id=$id";
263	ahart	1.4	$db->selectdb ("ntuple");
264			$results = $db->query ($query);
265
266			return $results;
267			}
268
269			sub
270			dbStatus
271			{
272			my $id = shift;
273			my $currentStatusName = shift;
274			my $user = shift;
275
276			open (CURRENT_STATUS, "<$currentStatusName");
277			my @currentStatus = <CURRENT_STATUS>;
278			close (CURRENT_STATUS);
279			my $currentStatus = join ("", @currentStatus);
280
281			$currentStatus = addSlashes ($currentStatus);
282			$user = addSlashes ($user);
283
284			my $results;
285			my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', currentStatus='$currentStatus' where id=$id";
286	ahart	1.1	$db->selectdb ("ntuple");
287			$results = $db->query ($query);
288
289			return $results;
290			}
291
292			sub
293			dbReport
294			{
295			my $id = shift;
296			my $lumiSummaryName = shift;
297	ahart	1.4	my $user = shift;
298	ahart	1.1
299			open (LUMI_SUMMARY, "<$lumiSummaryName");
300			my @lumiSummary = <LUMI_SUMMARY>;
301			close (LUMI_SUMMARY);
302			my $lumiSummary = join ("", @lumiSummary);
303
304			$lumiSummary = addSlashes ($lumiSummary);
305	ahart	1.4	$user = addSlashes ($user);
306	ahart	1.1
307			my $results;
308	ahart	1.10	my $query = "update ntuple set lastUpdateTime=now(), lastUpdateUser='$user', lumiSummary='$lumiSummary' where id=$id";
309	ahart	1.1	$db->selectdb ("ntuple");
310			$results = $db->query ($query);
311
312			return $results;
313			}
314	ahart	1.8
315			sub
316			deleteEntry
317			{
318			my $id = shift;
319
320			my $results;
321			my $query = "delete from ntuple where id=$id";
322			$db->selectdb ("ntuple");
323			$results = $db->query ($query);
324
325			return $results;
326			}