ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb
(Generate patch)

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.13 by ahart, Mon Oct 22 13:50:25 2012 UTC vs.
Revision 1.24 by ahart, Thu Mar 28 17:22:28 2013 UTC

# Line 14 | Line 14 | sub addSlashes;
14   sub uploadRelease;
15   sub downloadRelease;
16   sub datasetExists;
17 + sub uploadConfig;
18 + sub locationExists;
19  
20   our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
21  
22   my %opt;
23   Getopt::Long::Configure ("bundling");
24 < GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "help|h");
24 > GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "configName|i=s", "xSection|x=s", "help|h");
25   my $argc = @ARGV;
26  
27   printHelp ($ARGV[0]) if $opt{"help"};
28   printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
29 < printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry";
29 > printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
30   if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
31    {
32      print "The directory containing the ntuples must be given!\n";
# Line 35 | Line 37 | if ($ARGV[0] eq "uploadRelease" && (!$op
37      print "The Python config, CMSSW release directory, and recipe file must be given!\n";
38      exit;
39    }
40 + if ($ARGV[0] eq "uploadConfig" && (!$opt{"pyConfig"} || !$opt{"release"}))
41 +  {
42 +    print "The Python config and ntuple release must be given!\n";
43 +    exit;
44 +  }
45   if ($ARGV[0] eq "uploadRelease")
46    {
47      my $parentDir = "./$opt{'release'}";
# Line 50 | Line 57 | elsif ($ARGV[0] eq "downloadRelease")
57    {
58      downloadRelease ($ARGV[1], $opt{"pyConfig"});
59    }
60 + elsif ($ARGV[0] eq "uploadConfig")
61 +  {
62 +    uploadConfig ($ARGV[1], $ENV{"USER"}, $opt{"pyConfig"}, $opt{"release"}, $opt{"comment"});
63 +  }
64   elsif ($ARGV[0] eq "createOther")
65    {
66      my @listOfFiles = @ARGV;
67      @listOfFiles = reverse (@listOfFiles);
68      pop (@listOfFiles);
69      @listOfFiles = reverse (@listOfFiles);
70 <    dbOtherCreate ("$ENV{'USER'}", \@listOfFiles, $opt{"comment"});
70 >    dbOtherCreate ("$ENV{'USER'}\@$ENV{'HOSTNAME'}", \@listOfFiles, $opt{"comment"});
71    }
72   else
73    {
# Line 70 | Line 81 | else
81      $status = "" if $ARGV[0] eq "update";
82      $status = "deprecated" if $ARGV[0] eq "deprecate";
83      $opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
84 <    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}) if !$opt{"other"};
84 >    if ($opt{"location"} && locationExists ($id, $opt{"location"}))
85 >      {
86 >        print "This location is already registered in the database!\n";
87 >        exit;
88 >      }
89 >    dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}, $opt{"configName"}, $opt{"xSection"}) if !$opt{"other"};
90      dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
91      dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
92      dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
# Line 163 | Line 179 | dbUpdate
179   {
180    my $id = shift;
181    my $dataset = shift;
182 <  my $user = shift;
182 >  my $userAndHost = shift;
183    my $format = shift;
184    my $location = shift;
185 +  my $name = shift;
186    my $fileListName = shift;
187    my $status = shift;
188    my $comment = shift;
# Line 174 | Line 191 | dbUpdate
191    my $jsonFileName = shift;
192    my $lumiSummaryName = shift;
193    my $release = shift;
194 +  my $globalTag = shift;
195 +  my $configName = shift;
196 +  my $xSection = shift;
197 +
198 +  my $user = $userAndHost;
199 +  $user =~ s/@.*$//g;
200  
201    my $fileList;
202    my $pset;
# Line 242 | Line 265 | dbUpdate
265        close (LUMI_SUMMARY);
266        $lumiSummary = join ("", @lumiSummary);
267      }
268 <  if ($release)
246 <    {
247 <      my $query = "select id from ntupleRelease where name='$release'";
248 <      $db->selectdb ("ntuple");
249 <      my $results = $db->query ($query);
250 <      if ($results->numrows () != 1)
251 <        {
252 <          print "Ntuple release \"$release\" not found!\n";
253 <          exit;
254 <        }
255 <    }
268 >  $release .= ".$configName" if $configName;
269    ($nFiles, $size) = sizeOfDataset ($dataset, $location, $fileList);
270    my $fullLocation = $location;
271    $fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
# Line 265 | Line 278 | dbUpdate
278    $jsonFile = addSlashes ($jsonFile);
279    $lumiSummary = addSlashes ($lumiSummary);
280    $fullLocation = addSlashes ($fullLocation);
281 +  $name = addSlashes ($name);
282    $fileList = addSlashes ($fileList);
283    $status = addSlashes ($status);
284    $comment = addSlashes ($comment);
285    $release = addSlashes ($release);
286 +  $globalTag = addSlashes ($globalTag);
287  
288    my $results;
289    my $query;
290    if ($id < 0)
291      {
292 +      $query = "select id from ntuple where dataset='$dataset'";
293 +      $db->selectdb ("ntuple");
294 +      $results = $db->query ($query);
295 +      if ($results->numrows ())
296 +        {
297 +          print "This dataset already exists in the database. Create anyway? (y/N): ";
298 +          my $response = <STDIN>;
299 +          $response =~ s/\n//g;
300 +          $response = "n" if !$response;
301 +          exit if substr (lc ($response), 0, 1) ne 'y';
302 +        }
303 +
304        $query = "select max(id) from ntuple";
305        $db->selectdb ("ntuple");
306        $results = $db->query ($query);
# Line 281 | Line 308 | dbUpdate
308        my $id = 1;
309        $id = $row[0] + 1 if $results->numrows ();
310  
311 <      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version) values ($id, '$dataset', now(), now(), '$user', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release')";
311 >      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag, crossSectionInPicobarn) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag', $xSection)" if $xSection;
312 >      $query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag')" if !$xSection;
313      }
314    if ($id > 0)
315      {
316        my $fields;
317        my $values;
318  
319 <      $values .= ", dataset='$dataset'" if $dataset;
319 >      if ($name)
320 >        {
321 >          $query = "select id from ntuple where dataset='$name'";
322 >          $db->selectdb ("ntuple");
323 >          $results = $db->query ($query);
324 >          if ($results->numrows ())
325 >            {
326 >              print "This dataset already exists in the database. Create anyway? (y/N): ";
327 >              my $response = <STDIN>;
328 >              $response =~ s/\n//g;
329 >              $response = "n" if !$response;
330 >              exit if substr (lc ($response), 0, 1) ne 'y';
331 >            }
332 >          $values .= ", dataset='$name'";
333 >        }
334 >
335        $values .= ", lastUpdateTime=now()";
336        $values .= ", lastUpdateUser='$user'";
337        $values .= ", format='$format'" if $format;
# Line 303 | Line 346 | dbUpdate
346        $values .= ", jsonFile='$jsonFile'" if $jsonFile;
347        $values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
348        $values .= ", version='$release'" if $release;
349 +      $values .= ", globalTag='$globalTag'" if $globalTag;
350 +      $values .= ", crossSectionInPicobarn=$xSection" if $xSection;
351  
352        $values =~ s/^, //;
353        $query = "update ntuple set $values where id=$id";
# Line 366 | Line 411 | printHelp
411        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
412        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
413        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
414 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
415 +      printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
416        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
417        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
418        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
419        printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
420        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
421 +      printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
422      }
423    elsif ($command eq "createOther")
424      {
# Line 389 | Line 437 | printHelp
437        printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
438        printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
439        printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
440 +      printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
441 +      printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
442        printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
443        printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
444 +      printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
445        printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
446        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
447        printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
448        printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
449 +      printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
450      }
451    elsif ($command eq "finish")
452      {
453        print "Usage: $exeName -l DIRECTORY finish NAME\n";
402
454        print "Finalizes the database entry for dataset NAME, changing its status to\n";
455        print "\"present\".  This is intended to be the final step in command-line based ntuple\n";
456        print "production.\n";
# Line 447 | Line 498 | printHelp
498        print "Mandatory arguments to long options are mandatory for short options too.\n";
499        printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
500      }
501 +  elsif ($command eq "uploadConfig")
502 +    {
503 +      print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
504 +      print "Creates an entry in the database for a customized Python configuration file.\n";
505 +      print "\n";
506 +      print "Mandatory arguments to long options are mandatory for short options too.\n";
507 +      printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
508 +      printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config file";
509 +      printf "%-29s%s\n", "  -r, --release NAME", "ntuple release with which to use this config";
510 +    }
511    else
512      {
513        print "Usage: $exeName [OPTION]... COMMAND NAME\n";
# Line 465 | Line 526 | printHelp
526        printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
527        printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
528        printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
529 +      printf "%-29s%s\n", "  uploadConfig", "upload an ntuple config";
530      }
531  
532    exit;
# Line 530 | Line 592 | uploadRelease
592        print "$recipeName does not exist!\n";
593        exit;
594      }
595 <  move ($release, "/home/hart/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
595 >  move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
596    open (PY_CONFIG, "<$psetName");
597    my @pset = <PY_CONFIG>;
598    close (PY_CONFIG);
# Line 574 | Line 636 | downloadRelease
636        print "Ntuple release \"$release\" not found!\n";
637        exit;
638      }
577  if (!(-e "/home/hart/public_html/releases/$release.tar.gz"))
578    {
579      print "Release is in the database but no package exists!\n";
580      exit;
581    }
639    if ($pyConfigName)
640      {
641        my @row = $results->fetchrow ();
# Line 586 | Line 643 | downloadRelease
643        print PY_CONFIG $row[1];
644        close (PY_CONFIG);
645      }
646 <  copy ("/home/hart/public_html/releases/$release.tar.gz", "$release.tar.gz");
646 >  `wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
647   }
648  
649   sub
650   dbOtherCreate
651   {
652 <  my $user = shift;
652 >  my $userAndHost = shift;
653    my $listOfFiles = shift;
654    my $comment = shift;
655  
656 +  my $user = $userAndHost;
657 +  $user =~ s/@.*$//g;
658 +
659    my $size = 0.0;
660    my $nFiles = 0.0;
661    for (my $i = 0; $i < @$listOfFiles; $i++)
# Line 626 | Line 686 | dbOtherCreate
686    $location = addSlashes ($location);
687    $comment = addSlashes ($comment);
688  
689 <  my $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$user', '$user', '$location', $nFiles, $size, '$comment')";
689 >  $query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$userAndHost', '$user', '$location', $nFiles, $size, '$comment')";
690    $db->selectdb ("ntuple");
691 <  my $results = $db->query ($query);
691 >  $results = $db->query ($query);
692  
693    print "ID $id\n";
694   }
695 +
696 + sub
697 + uploadConfig
698 + {
699 +  my $name = shift;
700 +  my $user = shift;
701 +  my $psetName = shift;
702 +  my $release = shift;
703 +  my $comment = shift;
704 +
705 +  if (!(-e $psetName))
706 +    {
707 +      print "$psetName does not exist!\n";
708 +      exit;
709 +    }
710 +  if ($release)
711 +    {
712 +      my $query = "select id from ntupleRelease where name='$release'";
713 +      $db->selectdb ("ntuple");
714 +      my $results = $db->query ($query);
715 +      if ($results->numrows () != 1)
716 +        {
717 +          print "Ntuple release \"$release\" not found!\n";
718 +          exit;
719 +        }
720 +    }
721 +  open (PSET, "<$psetName");
722 +  my @pset = <PSET>;
723 +  close (PSET);
724 +  my $pset = join ("", @pset);
725 +
726 +  my $query = "select max(id) from ntupleConfig";
727 +  $db->selectdb ("ntuple");
728 +  my $results = $db->query ($query);
729 +  my @row = $results->fetchrow ();
730 +  my $id = 1;
731 +  $id = $row[0] + 1 if $results->numrows ();
732 +
733 +  $name = addSlashes ($name);
734 +  $user = addSlashes ($user);
735 +  $pset = addSlashes ($pset);
736 +  $release = addSlashes ($release);
737 +  $comment = addSlashes ($comment);
738 +
739 +  my $query = "insert into ntupleConfig (id, name, pset, user, pending, ntupleRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$release', '$comment')";
740 +  $db->selectdb ("ntuple");
741 +  $results = $db->query ($query);
742 + }
743 +
744 + sub
745 + locationExists
746 + {
747 +  my $id = shift;
748 +  my $location = shift;
749 +
750 +  $location =~ s/^(.*)\/+$/$1/;
751 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
752 +  $db->selectdb ("ntuple");
753 +  my $results = $db->query ($query);
754 +  return 1 if $results->numrows ();
755 +
756 +  if ($location =~ m/^\/store\/.*$/)
757 +    {
758 +      $location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
759 +    }
760 +  elsif ($location =~ m/^\/data\/se\/store\/.*$/)
761 +    {
762 +      $location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
763 +    }
764 +
765 +  my $query = "select id from ntuple where location like '$location%' and id!=$id";
766 +  $db->selectdb ("ntuple");
767 +  my $results = $db->query ($query);
768 +  return 1 if $results->numrows ();
769 +
770 +  return 0;
771 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines