ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb
(Generate patch)

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.9 by ahart, Fri Aug 24 13:50:50 2012 UTC vs.
Revision 1.21 by ahart, Sun Feb 3 00:14:20 2013 UTC

# Line 12 | Line 12 | sub dbDelete;
12   sub dbDeleteOther;
13   sub addSlashes;
14   sub uploadRelease;
15 + sub downloadRelease;
16   sub datasetExists;
17 + sub uploadConfig;
18 + sub locationExists;
19  
20   our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
21  
22   my %opt;
23   Getopt::Long::Configure ("bundling");
24 < GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "other|o", "help|h");
24 > GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "help|h");
25   my $argc = @ARGV;
26  
27   printHelp ($ARGV[0]) if $opt{"help"};
28   printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
29 < printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "deleteEntry";
29 > printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
30   if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
31    {
32      print "The directory containing the ntuples must be given!\n";
33      exit;
34    }
35 < if ($ARGV[0] eq "uploadRelease" && !$opt{"pyConfig"} && !$opt{"release"})
35 > if ($ARGV[0] eq "uploadRelease" && (!$opt{"pyConfig"} || !$opt{"release"} || !$opt{"recipe"}))
36    {
37 <    print "Both the Python and the CMSSW release must be given!\n";
37 >    print "The Python config, CMSSW release directory, and recipe file must be given!\n";
38 >    exit;
39 >  }
40 > if ($ARGV[0] eq "uploadConfig" && (!$opt{"pyConfig"} || !$opt{"release"}))
41 >  {
42 >    print "The Python config and ntuple release must be given!\n";
43      exit;
44    }
45   if ($ARGV[0] eq "uploadRelease")
# Line 43 | Line 51 | if ($ARGV[0] eq "uploadRelease")
51      $cmsswRelease =~ s/^.*CMSSW_([^\/]*).*$/CMSSW_$1/;
52      `tar -C $parentDir -czf $ARGV[1].tar.gz $opt{"release"}`;
53      $opt{"format"} = "BEAN" if !$opt{"format"};
54 <    uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], $ENV{"USER"}, $opt{"comment"});
54 >    uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], "$ENV{'USER'}", $opt{"comment"}, $opt{"recipe"});
55 >  }
56 > elsif ($ARGV[0] eq "downloadRelease")
57 >  {
58 >    downloadRelease ($ARGV[1], $opt{"pyConfig"});
59 >  }
60 > elsif ($ARGV[0] eq "uploadConfig")
61 >  {
62 >    uploadConfig ($ARGV[1], $ENV{"USER"}, $opt{"pyConfig"}, $opt{"release"}, $opt{"comment"});
63    }
64   elsif ($ARGV[0] eq "createOther")
65    {
# Line 51 | Line 67 | elsif ($ARGV[0] eq "createOther")
67      @listOfFiles = reverse (@listOfFiles);
68      pop (@listOfFiles);
69      @listOfFiles = reverse (@listOfFiles);
70 <    dbOtherCreate ($ENV{"USER"}, \@listOfFiles, $opt{"comment"});
70 >    dbOtherCreate ("$ENV{'USER'}\@$ENV{'HOSTNAME'}", \@listOfFiles, $opt{"comment"});
71    }
72   else
73    {
# Line 65 | Line 81 | else
81      $status = "" if $ARGV[0] eq "update";
82      $status = "deprecated" if $ARGV[0] eq "deprecate";
83      $opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
84 <    dbUpdate ($id, $fullDataset, $ENV{"USER"}, $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}) if !$opt{"other"};
85 <    dbUpdateOther ($id, $opt{"comment"}, $ENV{"USER"}) if $opt{"other"};
84 >    if ($opt{"location"} && locationExists ($id, $opt{"location"}))
85 >      {
86 >        print "This location is already registered in the database!\n";
87 >        exit;
88 >      }
89 >    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}) if !$opt{"other"};
90 >    dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
91      dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
92      dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
93    }
# Line 158 | Line 179 | dbUpdate
179   {
180    my $id = shift;
181    my $dataset = shift;
182 <  my $user = shift;
182 >  my $userAndHost = shift;
183    my $format = shift;
184    my $location = shift;
185 +  my $name = shift;
186    my $fileListName = shift;
187    my $status = shift;
188    my $comment = shift;
# Line 169 | Line 191 | dbUpdate
191    my $jsonFileName = shift;
192    my $lumiSummaryName = shift;
193    my $release = shift;
194 +  my $globalTag = shift;
195 +
196 +  my $user = $userAndHost;
197 +  $user =~ s/@.*$//g;
198  
199    my $fileList;
200    my $pset;
# Line 260 | Line 286 | dbUpdate
286    $jsonFile = addSlashes ($jsonFile);
287    $lumiSummary = addSlashes ($lumiSummary);
288    $fullLocation = addSlashes ($fullLocation);
289 +  $name = addSlashes ($name);
290    $fileList = addSlashes ($fileList);
291    $status = addSlashes ($status);
292    $comment = addSlashes ($comment);
293    $release = addSlashes ($release);
294 +  $globalTag = addSlashes ($globalTag);
295  
296    my $results;
297    my $query;
298    if ($id < 0)
299      {
300 +      $query = "select id from ntuple where dataset='$dataset'";
301 +      $db->selectdb ("ntuple");
302 +      $results = $db->query ($query);
303 +      if ($results->numrows ())
304 +        {
305 +          print "This dataset already exists in the database. Create anyway? (y/N): ";
306 +          my $response = <STDIN>;
307 +          $response =~ s/\n//g;
308 +          $response = "n" if !$response;
309 +          exit if substr (lc ($response), 0, 1) ne 'y';
310 +        }
311 +
312        $query = "select max(id) from ntuple";
313        $db->selectdb ("ntuple");
314        $results = $db->query ($query);
# Line 276 | Line 316 | dbUpdate
316        my $id = 1;
317        $id = $row[0] + 1 if $results->numrows ();
318  
319 <      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version) values ($id, '$dataset', now(), now(), '$user', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release')";
319 >      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag')";
320      }
321    if ($id > 0)
322      {
323        my $fields;
324        my $values;
325  
326 <      $values .= ", dataset='$dataset'" if $dataset;
326 >      if ($name)
327 >        {
328 >          $query = "select id from ntuple where dataset='$name'";
329 >          $db->selectdb ("ntuple");
330 >          $results = $db->query ($query);
331 >          if ($results->numrows ())
332 >            {
333 >              print "This dataset already exists in the database. Create anyway? (y/N): ";
334 >              my $response = <STDIN>;
335 >              $response =~ s/\n//g;
336 >              $response = "n" if !$response;
337 >              exit if substr (lc ($response), 0, 1) ne 'y';
338 >            }
339 >          $values .= ", dataset='$name'";
340 >        }
341 >
342        $values .= ", lastUpdateTime=now()";
343        $values .= ", lastUpdateUser='$user'";
344        $values .= ", format='$format'" if $format;
# Line 298 | Line 353 | dbUpdate
353        $values .= ", jsonFile='$jsonFile'" if $jsonFile;
354        $values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
355        $values .= ", version='$release'" if $release;
356 +      $values .= ", globalTag='$globalTag'" if $globalTag;
357  
358        $values =~ s/^, //;
359        $query = "update ntuple set $values where id=$id";
# Line 361 | Line 417 | printHelp
417        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
418        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
419        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
420 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
421        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
422        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
423        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
# Line 384 | Line 441 | printHelp
441        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
442        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
443        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
444 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
445        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
446        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
447 +      printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
448        printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
449        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
450        printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
# Line 394 | Line 453 | printHelp
453    elsif ($command eq "finish")
454      {
455        print "Usage: $exeName -l DIRECTORY finish NAME\n";
397
456        print "Finalizes the database entry for dataset NAME, changing its status to\n";
457        print "\"present\".  This is intended to be the final step in command-line based ntuple\n";
458        print "production.\n";
# Line 420 | Line 478 | printHelp
478      }
479    elsif ($command eq "uploadRelease")
480      {
481 <      print "Usage: $exeName -p FILE -r DIRECTORY [OPTION]... uploadRelease NAME\n";
481 >      print "Usage: $exeName -e FILE -p FILE -r DIRECTORY [OPTION]... uploadRelease NAME\n";
482        print "Copies an ntuple release to the appropriate area on the Tier 3, and creates a\n";
483        print "database entry for it, with NAME being the name of the release. An ntuple\n";
484        print "release is a CMSSW release with all the necessary packages added for creating\n";
# Line 428 | Line 486 | printHelp
486        print "\n";
487        print "Mandatory arguments to long options are mandatory for short options too.\n";
488        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
489 +      printf "%-29s%s\n", "  -e, --recipe FILE", "file containing the recipe for the release";
490        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
491        printf "%-29s%s\n", "  -p, --pyConfig FILE", "default Python config for the release";
492        printf "%-29s%s\n", "  -r, --release DIRECTORY", "CMSSW release containing ntuple packages";
493      }
494 +  elsif ($command eq "downloadRelease")
495 +    {
496 +      print "Usage: $exeName [OPTION]... downloadRelease NAME\n";
497 +      print "Copies an ntuple release to the current directory on the Tier 3. Optionally\n";
498 +      print "copies the corresponding Python configuration file registered in the database.\n";
499 +      print "\n";
500 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
501 +      printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
502 +    }
503 +  elsif ($command eq "uploadConfig")
504 +    {
505 +      print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
506 +      print "Creates an entry in the database for a customized Python configuration file.\n";
507 +      print "\n";
508 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
509 +      printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
510 +      printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config file";
511 +      printf "%-29s%s\n", "  -r, --release NAME", "ntuple release with which to use this config";
512 +    }
513    else
514      {
515        print "Usage: $exeName [OPTION]... COMMAND NAME\n";
# Line 443 | Line 521 | printHelp
521        print "\n";
522        print "COMMAND may be one of the following:\n";
523        printf "%-29s%s\n", "  create", "creates the entry";
524 <      printf "%-29s%s\n", "  create", "creates an entry for non-ntuple data";
524 >      printf "%-29s%s\n", "  createOther", "creates an entry for non-ntuple data";
525        printf "%-29s%s\n", "  update", "updates the entry";
526        printf "%-29s%s\n", "  finish", "finalizes the database entry";
527        printf "%-29s%s\n", "  deleteEntry", "removes the database entry";
528        printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
529        printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
530 +      printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
531 +      printf "%-29s%s\n", "  uploadConfig", "upload an ntuple config";
532      }
533  
534    exit;
# Line 502 | Line 582 | uploadRelease
582    my $name = shift;
583    my $user = shift;
584    my $comment = shift;
585 +  my $recipeName = shift;
586  
506  move ($release, "/home/hart/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
587    if (!(-e $psetName))
588      {
589        print "$psetName does not exist!\n";
590        exit;
591      }
592 +  if (!(-e $recipeName))
593 +    {
594 +      print "$recipeName does not exist!\n";
595 +      exit;
596 +    }
597 +  move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
598    open (PY_CONFIG, "<$psetName");
599    my @pset = <PY_CONFIG>;
600    close (PY_CONFIG);
601    my $pset = join ("", @pset);
602 +  open (RECIPE, "<$recipeName");
603 +  my @recipe = <RECIPE>;
604 +  close (RECIPE);
605 +  my $recipe = join ("\n", @recipe);
606  
607    my $query = "select max(id) from ntupleRelease";
608    $db->selectdb ("ntuple");
# Line 527 | Line 617 | uploadRelease
617    $format = addSlashes ($format);
618    $cmsswRelease = addSlashes ($cmsswRelease);
619    $comment = addSlashes ($comment);
620 +  $recipe = addSlashes ($recipe);
621 +
622 +  my $query = "insert into ntupleRelease (id, name, pset, user, pending, format, cmsswRelease, comment, recipe) values ($id, '$name', '$pset', '$user', 1, '$format', '$cmsswRelease', '$comment', '$recipe')";
623 +  $db->selectdb ("ntuple");
624 +  my $results = $db->query ($query);
625 + }
626 +
627 + sub
628 + downloadRelease
629 + {
630 +  my $release = shift;
631 +  my $pyConfigName = shift;
632  
633 <  my $query = "insert into ntupleRelease (id, name, pset, user, pending, format, cmsswRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$format', '$cmsswRelease', '$comment')";
633 >  my $query = "select id,pset from ntupleRelease where name='$release'";
634    $db->selectdb ("ntuple");
635    my $results = $db->query ($query);
636 +  if ($results->numrows () != 1)
637 +    {
638 +      print "Ntuple release \"$release\" not found!\n";
639 +      exit;
640 +    }
641 +  if ($pyConfigName)
642 +    {
643 +      my @row = $results->fetchrow ();
644 +      open (PY_CONFIG, ">$pyConfigName");
645 +      print PY_CONFIG $row[1];
646 +      close (PY_CONFIG);
647 +    }
648 +  `wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
649   }
650  
651   sub
652   dbOtherCreate
653   {
654 <  my $user = shift;
654 >  my $userAndHost = shift;
655    my $listOfFiles = shift;
656    my $comment = shift;
657  
658 +  my $user = $userAndHost;
659 +  $user =~ s/@.*$//g;
660 +
661    my $size = 0.0;
662    my $nFiles = 0.0;
663    for (my $i = 0; $i < @$listOfFiles; $i++)
# Line 570 | Line 688 | dbOtherCreate
688    $location = addSlashes ($location);
689    $comment = addSlashes ($comment);
690  
691 <  my $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$user', '$user', '$location', $nFiles, $size, '$comment')";
691 >  $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$userAndHost', '$user', '$location', $nFiles, $size, '$comment')";
692    $db->selectdb ("ntuple");
693 <  my $results = $db->query ($query);
693 >  $results = $db->query ($query);
694  
695    print "ID $id\n";
696   }
697 +
698 + sub
699 + uploadConfig
700 + {
701 +  my $name = shift;
702 +  my $user = shift;
703 +  my $psetName = shift;
704 +  my $release = shift;
705 +  my $comment = shift;
706 +
707 +  if (!(-e $psetName))
708 +    {
709 +      print "$psetName does not exist!\n";
710 +      exit;
711 +    }
712 +  if ($release)
713 +    {
714 +      my $query = "select id from ntupleRelease where name='$release'";
715 +      $db->selectdb ("ntuple");
716 +      my $results = $db->query ($query);
717 +      if ($results->numrows () != 1)
718 +        {
719 +          print "Ntuple release \"$release\" not found!\n";
720 +          exit;
721 +        }
722 +    }
723 +  open (PSET, "<$psetName");
724 +  my @pset = <PSET>;
725 +  close (PSET);
726 +  my $pset = join ("", @pset);
727 +
728 +  my $query = "select max(id) from ntupleConfig";
729 +  $db->selectdb ("ntuple");
730 +  my $results = $db->query ($query);
731 +  my @row = $results->fetchrow ();
732 +  my $id = 1;
733 +  $id = $row[0] + 1 if $results->numrows ();
734 +
735 +  $name = addSlashes ($name);
736 +  $user = addSlashes ($user);
737 +  $pset = addSlashes ($pset);
738 +  $release = addSlashes ($release);
739 +  $comment = addSlashes ($comment);
740 +
741 +  my $query = "insert into ntupleConfig (id, name, pset, user, pending, ntupleRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$release', '$comment')";
742 +  $db->selectdb ("ntuple");
743 +  $results = $db->query ($query);
744 + }
745 +
746 + sub
747 + locationExists
748 + {
749 +  my $id = shift;
750 +  my $location = shift;
751 +
752 +  $location =~ s/^(.*)\/+$/$1/;
753 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
754 +  $db->selectdb ("ntuple");
755 +  my $results = $db->query ($query);
756 +  return 1 if $results->numrows ();
757 +
758 +  if ($location =~ m/^\/store\/.*$/)
759 +    {
760 +      $location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
761 +    }
762 +  elsif ($location =~ m/^\/data\/se\/store\/.*$/)
763 +    {
764 +      $location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
765 +    }
766 +
767 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
768 +  $db->selectdb ("ntuple");
769 +  my $results = $db->query ($query);
770 +  return 1 if $results->numrows ();
771 +
772 +  return 0;
773 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines