ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.15 by ahart, Fri Nov 30 18:20:59 2012 UTC vs.
Revision 1.28 by ahart, Fri Aug 30 19:54:49 2013 UTC

#	Line 4 | Line 4 | use strict;
4		use Mysql;
5		use Getopt::Long;
6		use File::Copy;
7	+	use Term::ANSIColor;
8
9		sub getDataset;
10		sub dbUpdate;
#	Line 15 | Line 16 | sub uploadRelease;
16		sub downloadRelease;
17		sub datasetExists;
18		sub uploadConfig;
19	+	sub locationExists;
20	+	sub getLocation;
21
22		our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
23
24		my %opt;
25		Getopt::Long::Configure ("bundling");
26	<	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "help|h");
26	>	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "configName|i=s", "xSection|x=s", "higherOrderXSec|y=s", "help|h");
27		my $argc = @ARGV;
28
29		printHelp ($ARGV[0]) if $opt{"help"};
30		printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
31		printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
32	<	if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
32	>	if ($ARGV[0] eq "create" && !$opt{"location"})
33		{
34		print "The directory containing the ntuples must be given!\n";
35		exit;
#	Line 80 | Line 83 | else
83		$status = "" if $ARGV[0] eq "update";
84		$status = "deprecated" if $ARGV[0] eq "deprecate";
85		$opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
86	<	dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}) if !$opt{"other"};
87	<	dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
86	>	$opt{"lumiSummary"} = "/dev/null" if $ARGV[0] eq "finish" && !$opt{"lumiSummary"};
87	>	if ($ARGV[0] eq "finish" && (!$opt{"location"} && !getLocation ($id)))
88	>	{
89	>	print "The directory containing the ntuples must be given!\n";
90	>	exit;
91	>	}
92	>	if ($opt{"location"} && locationExists ($id, $opt{"location"}))
93	>	{
94	>	print "This location is already registered in the database!\n";
95	>	exit;
96	>	}
97	>	dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}, $opt{"configName"}, $opt{"xSection"}, $opt{"higherOrderXSec"}) if $ARGV[0] ne "deleteEntry" && !$opt{"other"};
98	>	dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $ARGV[0] ne "deleteEntry" && $opt{"other"};
99		dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
100		dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
101		}
#	Line 109 | Line 123 | getDataset
123		my $queryDataset = $dataset;
124		$queryDataset =~ s/\*/%/g;
125		$queryDataset =~ s/(.*)/%$1%/g;
126	<	my $query = "select id,dataset,user,creationTime from ntuple where dataset like '$queryDataset' order by creationTime";
126	>	my $query = "select id,dataset,user,creationTime,status from ntuple where dataset like '$queryDataset' order by creationTime";
127		$db->selectdb ("ntuple");
128		$results = $db->query ($query);
129		if ($results->numrows () == 1)
#	Line 142 | Line 156 | getDataset
156		for (my $i = 1; $i <= $results->numrows (); $i++)
157		{
158		my @row = $results->fetchrow ();
159	+	$row[3] =~ s/([^ ]*) [^ ]*/$1/g;
160		$id{"$i"} = $row[0];
161		$fullDataset{"$i"} = $row[1];
162		printf "(%2d) $row[1]\n", $i;
163	<	print "     created by $row[2] on $row[3]\n";
163	>	print "     (";
164	>	print color "green" if $row[4] eq "present";
165	>	print color "bold yellow" if $row[4] eq "submitted";
166	>	print color "bold red" if $row[4] eq "created" or $row[4] eq "cancelled" or $row[4] eq "deprecated";
167	>	print $row[4];
168	>	print color "reset";
169	>	print ") created by $row[2] on $row[3]\n";
170		}
171		print "\nWhich entry would you like to modify?";
172		if ($command ne "deleteEntry" && $command ne "deprecate")
#	Line 176 | Line 197 | dbUpdate
197		my $userAndHost = shift;
198		my $format = shift;
199		my $location = shift;
200	+	my $name = shift;
201		my $fileListName = shift;
202		my $status = shift;
203		my $comment = shift;
#	Line 184 | Line 206 | dbUpdate
206		my $jsonFileName = shift;
207		my $lumiSummaryName = shift;
208		my $release = shift;
209	+	my $globalTag = shift;
210	+	my $configName = shift;
211	+	my $xSection = shift;
212	+	my $ySection = shift;
213	+
214	+	$location = getLocation ($id) if $id > 0
215	+	&& !$crabCfgName
216	+	&& !$comment
217	+	&& !$format
218	+	&& !$globalTag
219	+	&& !$configName
220	+	&& !$jsonFileName
221	+	&& !$location
222	+	&& !$name
223	+	&& !$psetName
224	+	&& !$release
225	+	&& !$lumiSummaryName
226	+	&& !$xSection
227	+	&& !$ySection;
228
229		my $user = $userAndHost;
230		$user =~ s/@.*$//g;
#	Line 255 | Line 296 | dbUpdate
296		close (LUMI_SUMMARY);
297		$lumiSummary = join ("", @lumiSummary);
298		}
299	<	if ($release)
300	<	{
260	<	my $query = "select id from ntupleRelease where name='$release'";
261	<	$db->selectdb ("ntuple");
262	<	my $results = $db->query ($query);
263	<	if ($results->numrows () != 1)
264	<	{
265	<	print "Ntuple release \"$release\" not found!\n";
266	<	exit;
267	<	}
268	<	}
269	<	($nFiles, $size) = sizeOfDataset ($dataset, $location, $fileList);
299	>	$release .= ".$configName" if $configName;
300	>	($nFiles, $size) = sizeOfDataset ($location, $fileList);
301		my $fullLocation = $location;
302		$fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
303
#	Line 278 | Line 309 | dbUpdate
309		$jsonFile = addSlashes ($jsonFile);
310		$lumiSummary = addSlashes ($lumiSummary);
311		$fullLocation = addSlashes ($fullLocation);
312	+	$name = addSlashes ($name);
313		$fileList = addSlashes ($fileList);
314		$status = addSlashes ($status);
315		$comment = addSlashes ($comment);
316		$release = addSlashes ($release);
317	+	$globalTag = addSlashes ($globalTag);
318
319		my $results;
320		my $query;
321		if ($id < 0)
322		{
323	+	$query = "select id from ntuple where dataset='$dataset'";
324	+	$db->selectdb ("ntuple");
325	+	$results = $db->query ($query);
326	+	if ($results->numrows ())
327	+	{
328	+	print "This dataset already exists in the database. Create anyway? (y/N): ";
329	+	my $response = <STDIN>;
330	+	$response =~ s/\n//g;
331	+	$response = "n" if !$response;
332	+	exit if substr (lc ($response), 0, 1) ne 'y';
333	+	}
334	+
335		$query = "select max(id) from ntuple";
336		$db->selectdb ("ntuple");
337		$results = $db->query ($query);
#	Line 294 | Line 339 | dbUpdate
339		my $id = 1;
340		$id = $row[0] + 1 if $results->numrows ();
341
342	<	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release')";
342	>	my $columns = "(id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag";
343	>	$columns .= ", crossSectionInPicobarn" if $xSection;
344	>	$columns .= ", higherOrderCrossSectionInPicobarn" if $ySection;
345	>	$columns .= ")";
346	>	my $values = "($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag'";
347	>	$values .= ", $xSection" if $xSection;
348	>	$values .= ", $ySection" if $ySection;
349	>	$values .= ")";
350	>	$query = "insert into ntuple $columns values $values";
351		}
352		if ($id > 0)
353		{
354		my $fields;
355		my $values;
356
357	<	$values .= ", dataset='$dataset'" if $dataset;
357	>	if ($name)
358	>	{
359	>	$query = "select id from ntuple where dataset='$name'";
360	>	$db->selectdb ("ntuple");
361	>	$results = $db->query ($query);
362	>	if ($results->numrows ())
363	>	{
364	>	print "This dataset already exists in the database. Use this name anyway? (y/N): ";
365	>	my $response = <STDIN>;
366	>	$response =~ s/\n//g;
367	>	$response = "n" if !$response;
368	>	exit if substr (lc ($response), 0, 1) ne 'y';
369	>	}
370	>	$values .= ", dataset='$name'";
371	>	}
372	>
373		$values .= ", lastUpdateTime=now()";
374		$values .= ", lastUpdateUser='$user'";
375		$values .= ", format='$format'" if $format;
376		$values .= ", location='$fullLocation'" if $location;
377	<	$values .= ", fileList='$fileList'" if $fileList;
377	>	$values .= ", fileList='$fileList'" if $fileListName;
378		$values .= ", nFiles=$nFiles" if $nFiles;
379		$values .= ", sizeInGB=$size" if $size;
380		$values .= ", status='$status'" if $status;
381		$values .= ", comment='$comment'" if $comment;
382	<	$values .= ", pset='$pset'" if $pset;
383	<	$values .= ", crabCfg='$crabCfg'" if $crabCfg;
384	<	$values .= ", jsonFile='$jsonFile'" if $jsonFile;
385	<	$values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
382	>	$values .= ", pset='$pset'" if $psetName;
383	>	$values .= ", crabCfg='$crabCfg'" if $crabCfgName;
384	>	$values .= ", jsonFile='$jsonFile'" if $jsonFileName;
385	>	$values .= ", lumiSummary='$lumiSummary'" if $lumiSummaryName;
386		$values .= ", version='$release'" if $release;
387	+	$values .= ", globalTag='$globalTag'" if $globalTag;
388	+	$values .= ", crossSectionInPicobarn=$xSection" if $xSection;
389	+	$values .= ", higherOrderCrossSectionInPicobarn=$ySection" if $ySection;
390
391		$values =~ s/^, //;
392		$query = "update ntuple set $values where id=$id";
#	Line 379 | Line 450 | printHelp
450		printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
451		printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
452		printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
453	+	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
454	+	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
455		printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
456		printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
457		printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
458		printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
459		printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
460	+	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns from PREP";
461	+	printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns";
462		}
463		elsif ($command eq "createOther")
464		{
#	Line 396 | Line 471 | printHelp
471		elsif ($command eq "update")
472		{
473		print "Usage: $exeName [OPTION]... update NAME\n";
474	<	print "Updates an existing database entry for dataset NAME.\n";
474	>	print "Updates an existing database entry for dataset NAME. If no options are given,\n";
475	>	print "updates the number of files and the size on disk of the dataset using the\n";
476	>	print "location in the database.\n";
477		print "\n";
478		print "Mandatory arguments to long options are mandatory for short options too.\n";
479		printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
480		printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
481		printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
482	+	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
483	+	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
484		printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
485		printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
486	+	printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
487		printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
488		printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
489		printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
490		printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
491	+	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns from PREP";
492	+	printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns";
493		}
494		elsif ($command eq "finish")
495		{
#	Line 461 | Line 543 | printHelp
543		}
544		elsif ($command eq "uploadConfig")
545		{
546	<	print "Usage: $exeName -p FILE -r DIRECTORY [OPTION]... uploadConfig NAME\n";
546	>	print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
547		print "Creates an entry in the database for a customized Python configuration file.\n";
548		print "\n";
549		print "Mandatory arguments to long options are mandatory for short options too.\n";
#	Line 496 | Line 578 | printHelp
578		sub
579		sizeOfDataset
580		{
499	–	my $dataset = shift;
581		my $location = shift;
582		my $fileList = shift;
583
#	Line 553 | Line 634 | uploadRelease
634		print "$recipeName does not exist!\n";
635		exit;
636		}
637	<	move ($release, "/home/hart/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
637	>	move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
638		open (PY_CONFIG, "<$psetName");
639		my @pset = <PY_CONFIG>;
640		close (PY_CONFIG);
#	Line 561 | Line 642 | uploadRelease
642		open (RECIPE, "<$recipeName");
643		my @recipe = <RECIPE>;
644		close (RECIPE);
645	<	my $recipe = join ("\n", @recipe);
645	>	my $recipe = join ("", @recipe);
646
647		my $query = "select max(id) from ntupleRelease";
648		$db->selectdb ("ntuple");
#	Line 597 | Line 678 | downloadRelease
678		print "Ntuple release \"$release\" not found!\n";
679		exit;
680		}
600	–	if (!(-e "/home/hart/public_html/releases/$release.tar.gz"))
601	–	{
602	–	print "Release is in the database but no package exists!\n";
603	–	exit;
604	–	}
681		if ($pyConfigName)
682		{
683		my @row = $results->fetchrow ();
#	Line 609 | Line 685 | downloadRelease
685		print PY_CONFIG $row[1];
686		close (PY_CONFIG);
687		}
688	<	copy ("/home/hart/public_html/releases/$release.tar.gz", "$release.tar.gz");
688	>	`wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
689		}
690
691		sub
#	Line 706 | Line 782 | uploadConfig
782		$db->selectdb ("ntuple");
783		$results = $db->query ($query);
784		}
785	+
786	+	sub
787	+	locationExists
788	+	{
789	+	my $id = shift;
790	+	my $location = shift;
791	+
792	+	$location =~ s/^(.*)\/+$/$1/;
793	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
794	+	$db->selectdb ("ntuple");
795	+	my $results = $db->query ($query);
796	+	return 1 if $results->numrows ();
797	+
798	+	if ($location =~ m/^\/store\/.*$/)
799	+	{
800	+	$location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
801	+	}
802	+	elsif ($location =~ m/^\/data\/se\/store\/.*$/)
803	+	{
804	+	$location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
805	+	}
806	+
807	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
808	+	$db->selectdb ("ntuple");
809	+	my $results = $db->query ($query);
810	+	return 1 if $results->numrows ();
811	+
812	+	return 0;
813	+	}
814	+
815	+	sub
816	+	getLocation
817	+	{
818	+	my $id = shift;
819	+
820	+	my $query = "select location from ntuple where id=$id";
821	+	$db->selectdb ("ntuple");
822	+	my $results = $db->query ($query);
823	+	my @row = $results->fetchrow ();
824	+
825	+	return $row[0];
826	+	}

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines