ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb
(Generate patch)

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.4 by ahart, Tue Jul 31 14:12:19 2012 UTC vs.
Revision 1.12 by ahart, Mon Oct 22 12:50:07 2012 UTC

# Line 7 | Line 7 | use File::Copy;
7  
8   sub getDataset;
9   sub dbUpdate;
10 + sub dbUpdateOther;
11   sub dbDelete;
12 + sub dbDeleteOther;
13   sub addSlashes;
14   sub uploadRelease;
15 + sub downloadRelease;
16 + sub datasetExists;
17  
18   our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
19  
20   my %opt;
21   Getopt::Long::Configure ("bundling");
22 < GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "help|h");
22 > GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "other|o", "help|h");
23   my $argc = @ARGV;
24  
25   printHelp ($ARGV[0]) if $opt{"help"};
26 < printHelp () if $argc != 2;
27 < printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "deleteEntry";
26 > printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
27 > printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry";
28   if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
29    {
30      print "The directory containing the ntuples must be given!\n";
# Line 31 | Line 35 | if ($ARGV[0] eq "uploadRelease" && !$opt
35      print "Both the Python and the CMSSW release must be given!\n";
36      exit;
37    }
38 < if ($ARGV[0] ne "uploadRelease")
35 <  {
36 <    my $id = -1;
37 <    $id = getDataset ($ARGV[0], $ARGV[1]) if $ARGV[0] ne "create";
38 <    $ARGV[0] = "create" if $id < 0;
39 <    my $status = "present";
40 <    $status = "" if $ARGV[0] eq "update";
41 <    $status = "deprecated" if $ARGV[0] eq "deprecate";
42 <    $opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
43 <    $opt{"location"} = "$ENV{'PWD'}/$opt{'location'}" if !($opt{"location"} =~ m/^\//);
44 <    dbUpdate ($id, $ARGV[1], $ENV{"USER"}, $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"});
45 <    dbDelete ($id) if $ARGV[0] eq "deleteEntry";
46 <  }
47 < else
38 > if ($ARGV[0] eq "uploadRelease")
39    {
40      my $parentDir = "./$opt{'release'}";
41      $parentDir =~ s/CMSSW_[^\/]*//g;
# Line 53 | Line 44 | else
44      $cmsswRelease =~ s/^.*CMSSW_([^\/]*).*$/CMSSW_$1/;
45      `tar -C $parentDir -czf $ARGV[1].tar.gz $opt{"release"}`;
46      $opt{"format"} = "BEAN" if !$opt{"format"};
47 <    uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], $ENV{"USER"}, $opt{"comment"});
47 >    uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], "$ENV{'USER'}", $opt{"comment"});
48 >  }
49 > elsif ($ARGV[0] eq "downloadRelease")
50 >  {
51 >    downloadRelease ($ARGV[1], $opt{"pyConfig"});
52 >  }
53 > elsif ($ARGV[0] eq "createOther")
54 >  {
55 >    my @listOfFiles = @ARGV;
56 >    @listOfFiles = reverse (@listOfFiles);
57 >    pop (@listOfFiles);
58 >    @listOfFiles = reverse (@listOfFiles);
59 >    dbOtherCreate ("$ENV{'USER'}", \@listOfFiles, $opt{"comment"});
60 >  }
61 > else
62 >  {
63 >    my $id = -1;
64 >    my $fullDataset;
65 >    ($id, $fullDataset) = getDataset ($ARGV[0], $ARGV[1]) if $ARGV[0] ne "create" && ($ARGV[0] ne "update" || !$opt{"other"}) && ($ARGV[0] ne "deleteEntry" || !$opt{"other"});
66 >    $id = $ARGV[1] if ($ARGV[0] eq "update" && $opt{"other"}) || ($ARGV[0] eq "deleteEntry" && $opt{"other"});
67 >    $fullDataset = $ARGV[1] if $id < 0;
68 >    $ARGV[0] = "create" if $id < 0;
69 >    my $status = "present";
70 >    $status = "" if $ARGV[0] eq "update";
71 >    $status = "deprecated" if $ARGV[0] eq "deprecate";
72 >    $opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
73 >    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}) if !$opt{"other"};
74 >    dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
75 >    dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
76 >    dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
77    }
78  
79   sub
# Line 79 | Line 99 | getDataset
99    my $queryDataset = $dataset;
100    $queryDataset =~ s/\*/%/g;
101    $queryDataset =~ s/(.*)/%$1%/g;
102 <  my $query = "select id,dataset,user,creationTime from ntuple where dataset like '$queryDataset' order by lastUpdateTime";
102 >  my $query = "select id,dataset,user,creationTime from ntuple where dataset like '$queryDataset' order by creationTime";
103    $db->selectdb ("ntuple");
104    $results = $db->query ($query);
105    if ($results->numrows () == 1)
106      {
107        my @row = $results->fetchrow ();
108 <      return $row[0];
108 >      return ($row[0], $row[1]);
109      }
110    if ($results->numrows () == 0)
111      {
# Line 96 | Line 116 | getDataset
116            $response =~ s/\n//g;
117            $response = "y" if !$response;
118            exit if substr (lc ($response), 0, 1) ne 'y';
119 <          return -1;
119 >          return (-1, "");
120          }
121        else
122          {
# Line 104 | Line 124 | getDataset
124            exit;
125          }
126      }
127 <  my %map;
127 >  my %id;
128 >  my %fullDataset;
129    print "Found multiple database entries matching\n";
130    print "\"$dataset\":\n";
131    print "( 0) new\n" if $command ne "deleteEntry" && $command ne "deprecate";
132    for (my $i = 1; $i <= $results->numrows (); $i++)
133      {
134        my @row = $results->fetchrow ();
135 <      $map{"$i"} = $row[0];
135 >      $id{"$i"} = $row[0];
136 >      $fullDataset{"$i"} = $row[1];
137        printf "(%2d) $row[1]\n", $i;
138        print "     created by $row[2] on $row[3]\n";
139      }
# Line 126 | Line 148 | getDataset
148      }
149    my $response = <STDIN>;
150    $response =~ s/[ \t\n]//g;
151 <  return -1 if $response == 0 && $command ne "deleteEntry" && $command ne "deprecate";
152 <  if (!(exists $map{$response}))
151 >  return (-1, "") if $response == 0 && $command ne "deleteEntry" && $command ne "deprecate";
152 >  if (!(exists $id{$response}))
153      {
154        print "Your selection \"$response\" was not a valid option! Quitting.\n";
155        exit;
156      }
157  
158 <  return $map{$response};
158 >  return ($id{$response}, $fullDataset{$response});
159   }
160  
161   sub
# Line 151 | Line 173 | dbUpdate
173    my $crabCfgName = shift;
174    my $jsonFileName = shift;
175    my $lumiSummaryName = shift;
176 +  my $release = shift;
177  
178    my $fileList;
179    my $pset;
# Line 219 | Line 242 | dbUpdate
242        close (LUMI_SUMMARY);
243        $lumiSummary = join ("", @lumiSummary);
244      }
245 +  if ($release)
246 +    {
247 +      my $query = "select id from ntupleRelease where name='$release'";
248 +      $db->selectdb ("ntuple");
249 +      my $results = $db->query ($query);
250 +      if ($results->numrows () != 1)
251 +        {
252 +          print "Ntuple release \"$release\" not found!\n";
253 +          exit;
254 +        }
255 +    }
256    ($nFiles, $size) = sizeOfDataset ($dataset, $location, $fileList);
257 +  my $fullLocation = $location;
258 +  $fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
259  
260    $dataset = addSlashes ($dataset);
261    $user = addSlashes ($user);
# Line 228 | Line 264 | dbUpdate
264    $crabCfg = addSlashes ($crabCfg);
265    $jsonFile = addSlashes ($jsonFile);
266    $lumiSummary = addSlashes ($lumiSummary);
267 <  $location = addSlashes ($location);
267 >  $fullLocation = addSlashes ($fullLocation);
268    $fileList = addSlashes ($fileList);
269    $status = addSlashes ($status);
270    $comment = addSlashes ($comment);
271 +  $release = addSlashes ($release);
272  
273    my $results;
274    my $query;
# Line 244 | Line 281 | dbUpdate
281        my $id = 1;
282        $id = $row[0] + 1 if $results->numrows ();
283  
284 <      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary) values ($id, '$dataset', now(), now(), '$user', '$format', '$location', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary')";
284 >      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version) values ($id, '$dataset', now(), now(), '$user', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release')";
285      }
286    if ($id > 0)
287      {
# Line 253 | Line 290 | dbUpdate
290  
291        $values .= ", dataset='$dataset'" if $dataset;
292        $values .= ", lastUpdateTime=now()";
293 +      $values .= ", lastUpdateUser='$user'";
294        $values .= ", format='$format'" if $format;
295 <      $values .= ", location='$location'" if $location;
295 >      $values .= ", location='$fullLocation'" if $location;
296        $values .= ", fileList='$fileList'" if $fileList;
297        $values .= ", nFiles=$nFiles" if $nFiles;
298        $values .= ", sizeInGB=$size" if $size;
# Line 264 | Line 302 | dbUpdate
302        $values .= ", crabCfg='$crabCfg'" if $crabCfg;
303        $values .= ", jsonFile='$jsonFile'" if $jsonFile;
304        $values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
305 +      $values .= ", version='$release'" if $release;
306  
307        $values =~ s/^, //;
308        $query = "update ntuple set $values where id=$id";
# Line 274 | Line 313 | dbUpdate
313   }
314  
315   sub
316 + dbUpdateOther
317 + {
318 +  my $id = shift;
319 +  my $comment = shift;
320 +  my $user = shift;
321 +
322 +  $comment = addSlashes ($comment);
323 +
324 +  my $query = "update other set lastUpdateTime=now(), lastUpdateUser='$user', comment='$comment'  where id=$id";
325 +  $db->selectdb ("ntuple");
326 +  my $results = $db->query ($query);
327 +
328 +  return $results;
329 + }
330 +
331 + sub
332   dbDelete
333   {
334    my $id = shift;
335  
281  return if $id < 0;
336    my $query = "delete from ntuple where id=$id";
337    $db->selectdb ("ntuple");
338    my $results = $db->query ($query);
339   }
340  
341   sub
342 + dbDeleteOther
343 + {
344 +  my $id = shift;
345 +
346 +  my $query = "delete from other where id=$id";
347 +  $db->selectdb ("ntuple");
348 +  my $results = $db->query ($query);
349 + }
350 +
351 + sub
352   printHelp
353   {
354    my $command = shift;
# Line 305 | Line 369 | printHelp
369        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
370        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
371        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
372 +      printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
373        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
374      }
375 +  elsif ($command eq "createOther")
376 +    {
377 +      print "Usage: $exeName [OPTION]... createOther DIRECTORIES_AND_FILES\n";
378 +      print "Creates an entry in the database for non-ntuple data.\n";
379 +      print "\n";
380 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
381 +      printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
382 +    }
383    elsif ($command eq "update")
384      {
385        print "Usage: $exeName [OPTION]... update NAME\n";
# Line 318 | Line 391 | printHelp
391        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
392        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
393        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
394 +      printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
395        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
396 +      printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
397        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
398      }
399    elsif ($command eq "finish")
# Line 334 | Line 409 | printHelp
409      }
410    elsif ($command eq "deleteEntry")
411      {
412 <      print "Usage: $exeName deleteEntry NAME\n";
412 >      print "Usage: $exeName [OPTION]... deleteEntry NAME\n";
413        print "Deletes the database entry for dataset NAME. This is intended primarily for\n";
414        print "mistaken database entries. If you wish to actually delete a set of ntuples,\n";
415        print "please use the \"deprecate\" command instead.\n";
416 +      print "\n";
417 +      printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
418      }
419    elsif ($command eq "deprecate")
420      {
# Line 360 | Line 437 | printHelp
437        printf "%-29s%s\n", "  -p, --pyConfig FILE", "default Python config for the release";
438        printf "%-29s%s\n", "  -r, --release DIRECTORY", "CMSSW release containing ntuple packages";
439      }
440 +  elsif ($command eq "downloadRelease")
441 +    {
442 +      print "Usage: $exeName [OPTION]... downloadRelease NAME\n";
443 +      print "Copies an ntuple release to the current directory on the Tier 3. Optionally\n";
444 +      print "copies the corresponding Python configuration file registered in the database.\n";
445 +      print "\n";
446 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
447 +      printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
448 +    }
449    else
450      {
451        print "Usage: $exeName [OPTION]... COMMAND NAME\n";
# Line 371 | Line 457 | printHelp
457        print "\n";
458        print "COMMAND may be one of the following:\n";
459        printf "%-29s%s\n", "  create", "creates the entry";
460 +      printf "%-29s%s\n", "  createOther", "creates an entry for non-ntuple data";
461        printf "%-29s%s\n", "  update", "updates the entry";
462        printf "%-29s%s\n", "  finish", "finalizes the database entry";
463        printf "%-29s%s\n", "  deleteEntry", "removes the database entry";
464        printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
465        printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
466 +      printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
467      }
468  
469    exit;
# Line 388 | Line 476 | sizeOfDataset
476    my $location = shift;
477    my $fileList = shift;
478  
391  my $query = "select nFiles from ntuple where dataset='$dataset'";
392  $db->selectdb ("ntuple");
393  my $results = $db->query ($query);
394  my @row = $results->fetchrow ();
395  my $previousNFiles = 0;
396  $previousNFiles = $row[0] if $results->numrows ();
397
479    my $size = 0.0;
480    my $nFiles = 0;
481 <  if ($location && !$fileList && $previousNFiles < 1)
481 >  if ($location && !$fileList)
482      {
483        if (!(-e $location))
484          {
# Line 466 | Line 547 | uploadRelease
547    $db->selectdb ("ntuple");
548    my $results = $db->query ($query);
549   }
550 +
551 + sub
552 + downloadRelease
553 + {
554 +  my $release = shift;
555 +  my $pyConfigName = shift;
556 +
557 +  my $query = "select id,pset from ntupleRelease where name='$release'";
558 +  $db->selectdb ("ntuple");
559 +  my $results = $db->query ($query);
560 +  if ($results->numrows () != 1)
561 +    {
562 +      print "Ntuple release \"$release\" not found!\n";
563 +      exit;
564 +    }
565 +  if (!(-e "/home/hart/public_html/releases/$release.tar.gz"))
566 +    {
567 +      print "Release is in the database but no package exists!\n";
568 +      exit;
569 +    }
570 +  if ($pyConfigName)
571 +    {
572 +      my @row = $results->fetchrow ();
573 +      open (PY_CONFIG, ">$pyConfigName");
574 +      print PY_CONFIG $row[1];
575 +      close (PY_CONFIG);
576 +    }
577 +  copy ("/home/hart/public_html/releases/$release.tar.gz", "$release.tar.gz");
578 + }
579 +
580 + sub
581 + dbOtherCreate
582 + {
583 +  my $user = shift;
584 +  my $listOfFiles = shift;
585 +  my $comment = shift;
586 +
587 +  my $size = 0.0;
588 +  my $nFiles = 0.0;
589 +  for (my $i = 0; $i < @$listOfFiles; $i++)
590 +    {
591 +      if (!(-e $$listOfFiles[$i]))
592 +        {
593 +          print "$$listOfFiles[$i] does not exist!\n";
594 +          exit;
595 +        }
596 +      $nFiles += `ls -R $$listOfFiles[$i] | grep -v ':\$' | grep -v '^\$' | wc -l`;
597 +      my $fileSize = `du -s $$listOfFiles[$i]`;
598 +      $fileSize =~ s/([^ ]*) .*/$1/;
599 +      $fileSize /= 1024 * 1024;
600 +      $size += $fileSize;
601 +      $$listOfFiles[$i] = "$ENV{'PWD'}/$$listOfFiles[$i]" if !($$listOfFiles[$i] =~ m/^\//);
602 +    }
603 +  $size = sprintf "%.2f", $size;
604 +  my $location = join ("<br />", @$listOfFiles);
605 +
606 +  my $query = "select max(id) from other";
607 +  $db->selectdb ("ntuple");
608 +  my $results = $db->query ($query);
609 +  my @row = $results->fetchrow ();
610 +  my $id = 1;
611 +  $id = $row[0] + 1 if $results->numrows ();
612 +
613 +  $user = addSlashes ($user);
614 +  $location = addSlashes ($location);
615 +  $comment = addSlashes ($comment);
616 +
617 +  my $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$user', '$user', '$location', $nFiles, $size, '$comment')";
618 +  $db->selectdb ("ntuple");
619 +  my $results = $db->query ($query);
620 +
621 +  print "ID $id\n";
622 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines