ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb
(Generate patch)

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.13 by ahart, Mon Oct 22 13:50:25 2012 UTC vs.
Revision 1.25 by ahart, Fri May 10 00:21:12 2013 UTC

# Line 14 | Line 14 | sub addSlashes;
14   sub uploadRelease;
15   sub downloadRelease;
16   sub datasetExists;
17 + sub uploadConfig;
18 + sub locationExists;
19 + sub getLocation;
20  
21   our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
22  
23   my %opt;
24   Getopt::Long::Configure ("bundling");
25 < GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "help|h");
25 > GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "configName|i=s", "xSection|x=s", "higherOrderXSec|y=s", "help|h");
26   my $argc = @ARGV;
27  
28   printHelp ($ARGV[0]) if $opt{"help"};
29   printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
30 < printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry";
30 > printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
31   if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
32    {
33      print "The directory containing the ntuples must be given!\n";
# Line 35 | Line 38 | if ($ARGV[0] eq "uploadRelease" && (!$op
38      print "The Python config, CMSSW release directory, and recipe file must be given!\n";
39      exit;
40    }
41 + if ($ARGV[0] eq "uploadConfig" && (!$opt{"pyConfig"} || !$opt{"release"}))
42 +  {
43 +    print "The Python config and ntuple release must be given!\n";
44 +    exit;
45 +  }
46   if ($ARGV[0] eq "uploadRelease")
47    {
48      my $parentDir = "./$opt{'release'}";
# Line 50 | Line 58 | elsif ($ARGV[0] eq "downloadRelease")
58    {
59      downloadRelease ($ARGV[1], $opt{"pyConfig"});
60    }
61 + elsif ($ARGV[0] eq "uploadConfig")
62 +  {
63 +    uploadConfig ($ARGV[1], $ENV{"USER"}, $opt{"pyConfig"}, $opt{"release"}, $opt{"comment"});
64 +  }
65   elsif ($ARGV[0] eq "createOther")
66    {
67      my @listOfFiles = @ARGV;
68      @listOfFiles = reverse (@listOfFiles);
69      pop (@listOfFiles);
70      @listOfFiles = reverse (@listOfFiles);
71 <    dbOtherCreate ("$ENV{'USER'}", \@listOfFiles, $opt{"comment"});
71 >    dbOtherCreate ("$ENV{'USER'}\@$ENV{'HOSTNAME'}", \@listOfFiles, $opt{"comment"});
72    }
73   else
74    {
# Line 70 | Line 82 | else
82      $status = "" if $ARGV[0] eq "update";
83      $status = "deprecated" if $ARGV[0] eq "deprecate";
84      $opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
85 <    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}) if !$opt{"other"};
85 >    if ($opt{"location"} && locationExists ($id, $opt{"location"}))
86 >      {
87 >        print "This location is already registered in the database!\n";
88 >        exit;
89 >      }
90 >    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}, $opt{"configName"}, $opt{"xSection"}, $opt{"higherOrderXSec"}) if !$opt{"other"};
91      dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
92      dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
93      dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
# Line 163 | Line 180 | dbUpdate
180   {
181    my $id = shift;
182    my $dataset = shift;
183 <  my $user = shift;
183 >  my $userAndHost = shift;
184    my $format = shift;
185    my $location = shift;
186 +  my $name = shift;
187    my $fileListName = shift;
188    my $status = shift;
189    my $comment = shift;
# Line 174 | Line 192 | dbUpdate
192    my $jsonFileName = shift;
193    my $lumiSummaryName = shift;
194    my $release = shift;
195 +  my $globalTag = shift;
196 +  my $configName = shift;
197 +  my $xSection = shift;
198 +  my $ySection = shift;
199 +
200 +  $location = getLocation ($id) if $id > 0
201 +                               && !$crabCfgName
202 +                               && !$comment
203 +                               && !$format
204 +                               && !$globalTag
205 +                               && !$configName
206 +                               && !$jsonFileName
207 +                               && !$location
208 +                               && !$name
209 +                               && !$psetName
210 +                               && !$release
211 +                               && !$lumiSummaryName
212 +                               && !$xSection
213 +                               && !$ySection;
214 +
215 +  my $user = $userAndHost;
216 +  $user =~ s/@.*$//g;
217  
218    my $fileList;
219    my $pset;
# Line 242 | Line 282 | dbUpdate
282        close (LUMI_SUMMARY);
283        $lumiSummary = join ("", @lumiSummary);
284      }
285 <  if ($release)
286 <    {
247 <      my $query = "select id from ntupleRelease where name='$release'";
248 <      $db->selectdb ("ntuple");
249 <      my $results = $db->query ($query);
250 <      if ($results->numrows () != 1)
251 <        {
252 <          print "Ntuple release \"$release\" not found!\n";
253 <          exit;
254 <        }
255 <    }
256 <  ($nFiles, $size) = sizeOfDataset ($dataset, $location, $fileList);
285 >  $release .= ".$configName" if $configName;
286 >  ($nFiles, $size) = sizeOfDataset ($location, $fileList);
287    my $fullLocation = $location;
288    $fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
289  
# Line 265 | Line 295 | dbUpdate
295    $jsonFile = addSlashes ($jsonFile);
296    $lumiSummary = addSlashes ($lumiSummary);
297    $fullLocation = addSlashes ($fullLocation);
298 +  $name = addSlashes ($name);
299    $fileList = addSlashes ($fileList);
300    $status = addSlashes ($status);
301    $comment = addSlashes ($comment);
302    $release = addSlashes ($release);
303 +  $globalTag = addSlashes ($globalTag);
304  
305    my $results;
306    my $query;
307    if ($id < 0)
308      {
309 +      $query = "select id from ntuple where dataset='$dataset'";
310 +      $db->selectdb ("ntuple");
311 +      $results = $db->query ($query);
312 +      if ($results->numrows ())
313 +        {
314 +          print "This dataset already exists in the database. Create anyway? (y/N): ";
315 +          my $response = <STDIN>;
316 +          $response =~ s/\n//g;
317 +          $response = "n" if !$response;
318 +          exit if substr (lc ($response), 0, 1) ne 'y';
319 +        }
320 +
321        $query = "select max(id) from ntuple";
322        $db->selectdb ("ntuple");
323        $results = $db->query ($query);
# Line 281 | Line 325 | dbUpdate
325        my $id = 1;
326        $id = $row[0] + 1 if $results->numrows ();
327  
328 <      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version) values ($id, '$dataset', now(), now(), '$user', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release')";
328 >      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag, crossSectionInPicobarn, higherOrderCrossSectionInPicobarn) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag', $xSection, $ySection)";
329      }
330    if ($id > 0)
331      {
332        my $fields;
333        my $values;
334  
335 <      $values .= ", dataset='$dataset'" if $dataset;
335 >      if ($name)
336 >        {
337 >          $query = "select id from ntuple where dataset='$name'";
338 >          $db->selectdb ("ntuple");
339 >          $results = $db->query ($query);
340 >          if ($results->numrows ())
341 >            {
342 >              print "This dataset already exists in the database. Use this name anyway? (y/N): ";
343 >              my $response = <STDIN>;
344 >              $response =~ s/\n//g;
345 >              $response = "n" if !$response;
346 >              exit if substr (lc ($response), 0, 1) ne 'y';
347 >            }
348 >          $values .= ", dataset='$name'";
349 >        }
350 >
351        $values .= ", lastUpdateTime=now()";
352        $values .= ", lastUpdateUser='$user'";
353        $values .= ", format='$format'" if $format;
# Line 303 | Line 362 | dbUpdate
362        $values .= ", jsonFile='$jsonFile'" if $jsonFile;
363        $values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
364        $values .= ", version='$release'" if $release;
365 +      $values .= ", globalTag='$globalTag'" if $globalTag;
366 +      $values .= ", crossSectionInPicobarn=$xSection" if $xSection;
367 +      $values .= ", higherOrderCrossSectionInPicobarn=$ySection" if $ySection;
368  
369        $values =~ s/^, //;
370        $query = "update ntuple set $values where id=$id";
# Line 366 | Line 428 | printHelp
428        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
429        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
430        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
431 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
432 +      printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
433        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
434        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
435        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
436        printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
437        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
438 +      printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
439 +      printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns for the";
440 +      printf "%-29s%s\n", "                            ", "sample";
441      }
442    elsif ($command eq "createOther")
443      {
# Line 383 | Line 450 | printHelp
450    elsif ($command eq "update")
451      {
452        print "Usage: $exeName [OPTION]... update NAME\n";
453 <      print "Updates an existing database entry for dataset NAME.\n";
453 >      print "Updates an existing database entry for dataset NAME. If no options are given,\n";
454 >      print "updates the number of files and the size on disk of the dataset using the\n";
455 >      print "location in the database.\n";
456        print "\n";
457        print "Mandatory arguments to long options are mandatory for short options too.\n";
458        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
459        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
460        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
461 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
462 +      printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
463        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
464        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
465 +      printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
466        printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
467        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
468        printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
469        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
470 +      printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
471 +      printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns for the";
472 +      printf "%-29s%s\n", "                            ", "sample";
473      }
474    elsif ($command eq "finish")
475      {
476        print "Usage: $exeName -l DIRECTORY finish NAME\n";
402
477        print "Finalizes the database entry for dataset NAME, changing its status to\n";
478        print "\"present\".  This is intended to be the final step in command-line based ntuple\n";
479        print "production.\n";
# Line 447 | Line 521 | printHelp
521        print "Mandatory arguments to long options are mandatory for short options too.\n";
522        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
523      }
524 +  elsif ($command eq "uploadConfig")
525 +    {
526 +      print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
527 +      print "Creates an entry in the database for a customized Python configuration file.\n";
528 +      print "\n";
529 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
530 +      printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
531 +      printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config file";
532 +      printf "%-29s%s\n", "  -r, --release NAME", "ntuple release with which to use this config";
533 +    }
534    else
535      {
536        print "Usage: $exeName [OPTION]... COMMAND NAME\n";
# Line 465 | Line 549 | printHelp
549        printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
550        printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
551        printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
552 +      printf "%-29s%s\n", "  uploadConfig", "upload an ntuple config";
553      }
554  
555    exit;
# Line 473 | Line 558 | printHelp
558   sub
559   sizeOfDataset
560   {
476  my $dataset = shift;
561    my $location = shift;
562    my $fileList = shift;
563  
# Line 530 | Line 614 | uploadRelease
614        print "$recipeName does not exist!\n";
615        exit;
616      }
617 <  move ($release, "/home/hart/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
617 >  move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
618    open (PY_CONFIG, "<$psetName");
619    my @pset = <PY_CONFIG>;
620    close (PY_CONFIG);
# Line 538 | Line 622 | uploadRelease
622    open (RECIPE, "<$recipeName");
623    my @recipe = <RECIPE>;
624    close (RECIPE);
625 <  my $recipe = join ("\n", @recipe);
625 >  my $recipe = join ("", @recipe);
626  
627    my $query = "select max(id) from ntupleRelease";
628    $db->selectdb ("ntuple");
# Line 574 | Line 658 | downloadRelease
658        print "Ntuple release \"$release\" not found!\n";
659        exit;
660      }
577  if (!(-e "/home/hart/public_html/releases/$release.tar.gz"))
578    {
579      print "Release is in the database but no package exists!\n";
580      exit;
581    }
661    if ($pyConfigName)
662      {
663        my @row = $results->fetchrow ();
# Line 586 | Line 665 | downloadRelease
665        print PY_CONFIG $row[1];
666        close (PY_CONFIG);
667      }
668 <  copy ("/home/hart/public_html/releases/$release.tar.gz", "$release.tar.gz");
668 >  `wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
669   }
670  
671   sub
672   dbOtherCreate
673   {
674 <  my $user = shift;
674 >  my $userAndHost = shift;
675    my $listOfFiles = shift;
676    my $comment = shift;
677  
678 +  my $user = $userAndHost;
679 +  $user =~ s/@.*$//g;
680 +
681    my $size = 0.0;
682    my $nFiles = 0.0;
683    for (my $i = 0; $i < @$listOfFiles; $i++)
# Line 626 | Line 708 | dbOtherCreate
708    $location = addSlashes ($location);
709    $comment = addSlashes ($comment);
710  
711 <  my $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$user', '$user', '$location', $nFiles, $size, '$comment')";
711 >  $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$userAndHost', '$user', '$location', $nFiles, $size, '$comment')";
712    $db->selectdb ("ntuple");
713 <  my $results = $db->query ($query);
713 >  $results = $db->query ($query);
714  
715    print "ID $id\n";
716   }
717 +
718 + sub
719 + uploadConfig
720 + {
721 +  my $name = shift;
722 +  my $user = shift;
723 +  my $psetName = shift;
724 +  my $release = shift;
725 +  my $comment = shift;
726 +
727 +  if (!(-e $psetName))
728 +    {
729 +      print "$psetName does not exist!\n";
730 +      exit;
731 +    }
732 +  if ($release)
733 +    {
734 +      my $query = "select id from ntupleRelease where name='$release'";
735 +      $db->selectdb ("ntuple");
736 +      my $results = $db->query ($query);
737 +      if ($results->numrows () != 1)
738 +        {
739 +          print "Ntuple release \"$release\" not found!\n";
740 +          exit;
741 +        }
742 +    }
743 +  open (PSET, "<$psetName");
744 +  my @pset = <PSET>;
745 +  close (PSET);
746 +  my $pset = join ("", @pset);
747 +
748 +  my $query = "select max(id) from ntupleConfig";
749 +  $db->selectdb ("ntuple");
750 +  my $results = $db->query ($query);
751 +  my @row = $results->fetchrow ();
752 +  my $id = 1;
753 +  $id = $row[0] + 1 if $results->numrows ();
754 +
755 +  $name = addSlashes ($name);
756 +  $user = addSlashes ($user);
757 +  $pset = addSlashes ($pset);
758 +  $release = addSlashes ($release);
759 +  $comment = addSlashes ($comment);
760 +
761 +  my $query = "insert into ntupleConfig (id, name, pset, user, pending, ntupleRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$release', '$comment')";
762 +  $db->selectdb ("ntuple");
763 +  $results = $db->query ($query);
764 + }
765 +
766 + sub
767 + locationExists
768 + {
769 +  my $id = shift;
770 +  my $location = shift;
771 +
772 +  $location =~ s/^(.*)\/+$/$1/;
773 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
774 +  $db->selectdb ("ntuple");
775 +  my $results = $db->query ($query);
776 +  return 1 if $results->numrows ();
777 +
778 +  if ($location =~ m/^\/store\/.*$/)
779 +    {
780 +      $location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
781 +    }
782 +  elsif ($location =~ m/^\/data\/se\/store\/.*$/)
783 +    {
784 +      $location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
785 +    }
786 +
787 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
788 +  $db->selectdb ("ntuple");
789 +  my $results = $db->query ($query);
790 +  return 1 if $results->numrows ();
791 +
792 +  return 0;
793 + }
794 +
795 + sub
796 + getLocation
797 + {
798 +  my $id = shift;
799 +
800 +  my $query = "select location from ntuple where id=$id";
801 +  $db->selectdb ("ntuple");
802 +  my $results = $db->query ($query);
803 +  my @row = $results->fetchrow ();
804 +
805 +  return $row[0];
806 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines