ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.3 by ahart, Thu Jun 14 17:42:30 2012 UTC vs.
Revision 1.28 by ahart, Fri Aug 30 19:54:49 2013 UTC

#	Line 3 | Line 3
3		use strict;
4		use Mysql;
5		use Getopt::Long;
6	+	use File::Copy;
7	+	use Term::ANSIColor;
8
9		sub getDataset;
10		sub dbUpdate;
11	+	sub dbUpdateOther;
12	+	sub dbDelete;
13	+	sub dbDeleteOther;
14		sub addSlashes;
15	+	sub uploadRelease;
16	+	sub downloadRelease;
17	+	sub datasetExists;
18	+	sub uploadConfig;
19	+	sub locationExists;
20	+	sub getLocation;
21
22	<	our $db;
22	>	our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
23
24		my %opt;
25		Getopt::Long::Configure ("bundling");
26	<	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "remoteLocation|r=s", "lumiSummary|s=s", "location|l=s", "help|h");
26	>	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "configName|i=s", "xSection|x=s", "higherOrderXSec|y=s", "help|h");
27		my $argc = @ARGV;
28
29	<	printHelp () if $opt{"help"};
30	<	printHelp () if $argc != 2;
31	<	printHelp () if $ARGV[0] != "create" && $ARGV[0] != "update" && $ARGV[0] != "finish" && $ARGV[0] != "deprecate";
32	<	my $id = -1;
33	<	$id = getDataset ($ARGV[1]) if $ARGV[0] ne "create";
34	<	$ARGV[0] = "create" if $id < 0;
35	<	my $status = "present";
36	<	$status = "" if $ARGV[0] eq "update";
37	<	$status = "deprecated" if $ARGV[0] eq "deprecate";
38	<	$opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
39	<	dbUpdate ($id, $ARGV[1], $ENV{"USER"}, $opt{"format"}, $opt{"location"}, $opt{"remoteLocation"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"});
29	>	printHelp ($ARGV[0]) if $opt{"help"};
30	>	printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
31	>	printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
32	>	if ($ARGV[0] eq "create" && !$opt{"location"})
33	>	{
34	>	print "The directory containing the ntuples must be given!\n";
35	>	exit;
36	>	}
37	>	if ($ARGV[0] eq "uploadRelease" && (!$opt{"pyConfig"} || !$opt{"release"} || !$opt{"recipe"}))
38	>	{
39	>	print "The Python config, CMSSW release directory, and recipe file must be given!\n";
40	>	exit;
41	>	}
42	>	if ($ARGV[0] eq "uploadConfig" && (!$opt{"pyConfig"} || !$opt{"release"}))
43	>	{
44	>	print "The Python config and ntuple release must be given!\n";
45	>	exit;
46	>	}
47	>	if ($ARGV[0] eq "uploadRelease")
48	>	{
49	>	my $parentDir = "./$opt{'release'}";
50	>	$parentDir =~ s/CMSSW_[^\/]*//g;
51	>	$parentDir =~ s/\/\/*/\//g;
52	>	my $cmsswRelease = $opt{'release'};
53	>	$cmsswRelease =~ s/^.*CMSSW_([^\/]*).*$/CMSSW_$1/;
54	>	`tar -C $parentDir -czf $ARGV[1].tar.gz $opt{"release"}`;
55	>	$opt{"format"} = "BEAN" if !$opt{"format"};
56	>	uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], "$ENV{'USER'}", $opt{"comment"}, $opt{"recipe"});
57	>	}
58	>	elsif ($ARGV[0] eq "downloadRelease")
59	>	{
60	>	downloadRelease ($ARGV[1], $opt{"pyConfig"});
61	>	}
62	>	elsif ($ARGV[0] eq "uploadConfig")
63	>	{
64	>	uploadConfig ($ARGV[1], $ENV{"USER"}, $opt{"pyConfig"}, $opt{"release"}, $opt{"comment"});
65	>	}
66	>	elsif ($ARGV[0] eq "createOther")
67	>	{
68	>	my @listOfFiles = @ARGV;
69	>	@listOfFiles = reverse (@listOfFiles);
70	>	pop (@listOfFiles);
71	>	@listOfFiles = reverse (@listOfFiles);
72	>	dbOtherCreate ("$ENV{'USER'}\@$ENV{'HOSTNAME'}", \@listOfFiles, $opt{"comment"});
73	>	}
74	>	else
75	>	{
76	>	my $id = -1;
77	>	my $fullDataset;
78	>	($id, $fullDataset) = getDataset ($ARGV[0], $ARGV[1]) if $ARGV[0] ne "create" && ($ARGV[0] ne "update" || !$opt{"other"}) && ($ARGV[0] ne "deleteEntry" || !$opt{"other"});
79	>	$id = $ARGV[1] if ($ARGV[0] eq "update" && $opt{"other"}) || ($ARGV[0] eq "deleteEntry" && $opt{"other"});
80	>	$fullDataset = $ARGV[1] if $id < 0;
81	>	$ARGV[0] = "create" if $id < 0;
82	>	my $status = "present";
83	>	$status = "" if $ARGV[0] eq "update";
84	>	$status = "deprecated" if $ARGV[0] eq "deprecate";
85	>	$opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
86	>	$opt{"lumiSummary"} = "/dev/null" if $ARGV[0] eq "finish" && !$opt{"lumiSummary"};
87	>	if ($ARGV[0] eq "finish" && (!$opt{"location"} && !getLocation ($id)))
88	>	{
89	>	print "The directory containing the ntuples must be given!\n";
90	>	exit;
91	>	}
92	>	if ($opt{"location"} && locationExists ($id, $opt{"location"}))
93	>	{
94	>	print "This location is already registered in the database!\n";
95	>	exit;
96	>	}
97	>	dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}, $opt{"configName"}, $opt{"xSection"}, $opt{"higherOrderXSec"}) if $ARGV[0] ne "deleteEntry" && !$opt{"other"};
98	>	dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $ARGV[0] ne "deleteEntry" && $opt{"other"};
99	>	dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
100	>	dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
101	>	}
102
103		sub
104		addSlashes
#	Line 43 | Line 116 | addSlashes
116		sub
117		getDataset
118		{
119	+	my $command = shift;
120		my $dataset = shift;
121
122		my $results;
123		my $queryDataset = $dataset;
124		$queryDataset =~ s/\*/%/g;
125		$queryDataset =~ s/(.*)/%$1%/g;
126	<	$db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User");
53	<	my $query = "select id,dataset,user,creationTime from ntuple where dataset like '$queryDataset' order by lastUpdateTime";
126	>	my $query = "select id,dataset,user,creationTime,status from ntuple where dataset like '$queryDataset' order by creationTime";
127		$db->selectdb ("ntuple");
128		$results = $db->query ($query);
129		if ($results->numrows () == 1)
130		{
131		my @row = $results->fetchrow ();
132	<	return $row[0];
132	>	return ($row[0], $row[1]);
133		}
134		if ($results->numrows () == 0)
135		{
136	<	print "Database entry does not exist. Create it? (Y/n): ";
137	<	my $response = <STDIN>;
138	<	$response =~ s/\n//g;
139	<	$response = "y" if !$response;
140	<	exit if substr (lc ($response), 0, 1) ne 'y';
141	<	return -1;
136	>	if ($command ne "deleteEntry" && $command ne "deprecate")
137	>	{
138	>	print "Database entry does not exist. Create it? (Y/n): ";
139	>	my $response = <STDIN>;
140	>	$response =~ s/\n//g;
141	>	$response = "y" if !$response;
142	>	exit if substr (lc ($response), 0, 1) ne 'y';
143	>	return (-1, "");
144	>	}
145	>	else
146	>	{
147	>	print "Database entry does not exist.\n";
148	>	exit;
149	>	}
150		}
151	<	my %map;
151	>	my %id;
152	>	my %fullDataset;
153		print "Found multiple database entries matching\n";
154		print "\"$dataset\":\n";
155	<	print "( 0) new\n";
155	>	print "( 0) new\n" if $command ne "deleteEntry" && $command ne "deprecate";
156		for (my $i = 1; $i <= $results->numrows (); $i++)
157		{
158		my @row = $results->fetchrow ();
159	<	$map{"$i"} = $row[0];
159	>	$row[3] =~ s/([^ ]*) [^ ]*/$1/g;
160	>	$id{"$i"} = $row[0];
161	>	$fullDataset{"$i"} = $row[1];
162		printf "(%2d) $row[1]\n", $i;
163	<	print "     created by $row[2] on $row[3]\n";
163	>	print "     (";
164	>	print color "green" if $row[4] eq "present";
165	>	print color "bold yellow" if $row[4] eq "submitted";
166	>	print color "bold red" if $row[4] eq "created" or $row[4] eq "cancelled" or $row[4] eq "deprecated";
167	>	print $row[4];
168	>	print color "reset";
169	>	print ") created by $row[2] on $row[3]\n";
170	>	}
171	>	print "\nWhich entry would you like to modify?";
172	>	if ($command ne "deleteEntry" && $command ne "deprecate")
173	>	{
174	>	print " (Select 0 to create a new entry): "
175	>	}
176	>	else
177	>	{
178	>	print ": ";
179		}
81	–	print "\nWhich entry would you like to update? (Select 0 to create a new entry): ";
180		my $response = <STDIN>;
181		$response =~ s/[ \t\n]//g;
182	<	return -1 if $response == 0;
183	<	if (!(exists $map{$response}))
182	>	return (-1, "") if $response == 0 && $command ne "deleteEntry" && $command ne "deprecate";
183	>	if (!(exists $id{$response}))
184		{
185		print "Your selection \"$response\" was not a valid option! Quitting.\n";
186		exit;
187		}
188
189	<	return $map{$response};
189	>	return ($id{$response}, $fullDataset{$response});
190		}
191
192		sub
#	Line 96 | Line 194 | dbUpdate
194		{
195		my $id = shift;
196		my $dataset = shift;
197	<	my $user = shift;
197	>	my $userAndHost = shift;
198		my $format = shift;
199		my $location = shift;
200	<	my $remoteLocation = shift;
200	>	my $name = shift;
201	>	my $fileListName = shift;
202		my $status = shift;
203		my $comment = shift;
204		my $psetName = shift;
205		my $crabCfgName = shift;
206		my $jsonFileName = shift;
207		my $lumiSummaryName = shift;
208	+	my $release = shift;
209	+	my $globalTag = shift;
210	+	my $configName = shift;
211	+	my $xSection = shift;
212	+	my $ySection = shift;
213	+
214	+	$location = getLocation ($id) if $id > 0
215	+	&& !$crabCfgName
216	+	&& !$comment
217	+	&& !$format
218	+	&& !$globalTag
219	+	&& !$configName
220	+	&& !$jsonFileName
221	+	&& !$location
222	+	&& !$name
223	+	&& !$psetName
224	+	&& !$release
225	+	&& !$lumiSummaryName
226	+	&& !$xSection
227	+	&& !$ySection;
228	+
229	+	my $user = $userAndHost;
230	+	$user =~ s/@.*$//g;
231
232	+	my $fileList;
233		my $pset;
234		my $crabCfg;
235		my $jsonFile;
236		my $lumiSummary;
237		my $size = 0;
238		my $nFiles = 0;
239	+	if ($fileListName)
240	+	{
241	+	if (!(-e $fileListName))
242	+	{
243	+	print "$fileListName does not exist!\n";
244	+	exit;
245	+	}
246	+	open (FILE_LIST, "<$fileListName");
247	+	my @fileList = <FILE_LIST>;
248	+	close (FILE_LIST);
249	+	$fileList = join ("", @fileList);
250	+	}
251		if ($psetName)
252		{
253		if (!(-e $psetName))
#	Line 161 | Line 296 | dbUpdate
296		close (LUMI_SUMMARY);
297		$lumiSummary = join ("", @lumiSummary);
298		}
299	<	if ($location)
300	<	{
301	<	if (!(-e $location))
302	<	{
168	<	print "$location does not exist!\n";
169	<	exit;
170	<	}
171	<	$nFiles = `ls $location | wc -l`;
172	<	$size = `du -s $location`;
173	<	$size =~ s/([^ ]*) .*/$1/;
174	<	$size /= 1024 * 1024;
175	<	$size = sprintf "%.2f", $size;
176	<	}
299	>	$release .= ".$configName" if $configName;
300	>	($nFiles, $size) = sizeOfDataset ($location, $fileList);
301	>	my $fullLocation = $location;
302	>	$fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
303
304		$dataset = addSlashes ($dataset);
305		$user = addSlashes ($user);
#	Line 182 | Line 308 | dbUpdate
308		$crabCfg = addSlashes ($crabCfg);
309		$jsonFile = addSlashes ($jsonFile);
310		$lumiSummary = addSlashes ($lumiSummary);
311	<	$location = addSlashes ($location);
312	<	$remoteLocation = addSlashes ($remoteLocation);
311	>	$fullLocation = addSlashes ($fullLocation);
312	>	$name = addSlashes ($name);
313	>	$fileList = addSlashes ($fileList);
314		$status = addSlashes ($status);
315		$comment = addSlashes ($comment);
316	+	$release = addSlashes ($release);
317	+	$globalTag = addSlashes ($globalTag);
318
319		my $results;
320		my $query;
321		if ($id < 0)
322		{
323	<	$db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User");
323	>	$query = "select id from ntuple where dataset='$dataset'";
324	>	$db->selectdb ("ntuple");
325	>	$results = $db->query ($query);
326	>	if ($results->numrows ())
327	>	{
328	>	print "This dataset already exists in the database. Create anyway? (y/N): ";
329	>	my $response = <STDIN>;
330	>	$response =~ s/\n//g;
331	>	$response = "n" if !$response;
332	>	exit if substr (lc ($response), 0, 1) ne 'y';
333	>	}
334	>
335		$query = "select max(id) from ntuple";
336		$db->selectdb ("ntuple");
337		$results = $db->query ($query);
#	Line 199 | Line 339 | dbUpdate
339		my $id = 1;
340		$id = $row[0] + 1 if $results->numrows ();
341
342	<	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, user, format, location, remoteLocation, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary) values ($id, '$dataset', now(), now(), '$user', '$format', '$location', '$remoteLocation', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary')";
342	>	my $columns = "(id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag";
343	>	$columns .= ", crossSectionInPicobarn" if $xSection;
344	>	$columns .= ", higherOrderCrossSectionInPicobarn" if $ySection;
345	>	$columns .= ")";
346	>	my $values = "($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag'";
347	>	$values .= ", $xSection" if $xSection;
348	>	$values .= ", $ySection" if $ySection;
349	>	$values .= ")";
350	>	$query = "insert into ntuple $columns values $values";
351		}
352		if ($id > 0)
353		{
354		my $fields;
355		my $values;
356
357	<	$values .= ", dataset='$dataset'" if $dataset;
357	>	if ($name)
358	>	{
359	>	$query = "select id from ntuple where dataset='$name'";
360	>	$db->selectdb ("ntuple");
361	>	$results = $db->query ($query);
362	>	if ($results->numrows ())
363	>	{
364	>	print "This dataset already exists in the database. Use this name anyway? (y/N): ";
365	>	my $response = <STDIN>;
366	>	$response =~ s/\n//g;
367	>	$response = "n" if !$response;
368	>	exit if substr (lc ($response), 0, 1) ne 'y';
369	>	}
370	>	$values .= ", dataset='$name'";
371	>	}
372	>
373		$values .= ", lastUpdateTime=now()";
374	+	$values .= ", lastUpdateUser='$user'";
375		$values .= ", format='$format'" if $format;
376	<	$values .= ", location='$location'" if $location;
377	<	$values .= ", remoteLocation='$remoteLocation'" if $remoteLocation;
378	<	$values .= ", nFiles=$nFiles" if $location;
379	<	$values .= ", sizeInGB=$size" if $location;
376	>	$values .= ", location='$fullLocation'" if $location;
377	>	$values .= ", fileList='$fileList'" if $fileListName;
378	>	$values .= ", nFiles=$nFiles" if $nFiles;
379	>	$values .= ", sizeInGB=$size" if $size;
380		$values .= ", status='$status'" if $status;
381		$values .= ", comment='$comment'" if $comment;
382	<	$values .= ", pset='$pset'" if $pset;
383	<	$values .= ", crabCfg='$crabCfg'" if $crabCfg;
384	<	$values .= ", jsonFile='$jsonFile'" if $jsonFile;
385	<	$values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
382	>	$values .= ", pset='$pset'" if $psetName;
383	>	$values .= ", crabCfg='$crabCfg'" if $crabCfgName;
384	>	$values .= ", jsonFile='$jsonFile'" if $jsonFileName;
385	>	$values .= ", lumiSummary='$lumiSummary'" if $lumiSummaryName;
386	>	$values .= ", version='$release'" if $release;
387	>	$values .= ", globalTag='$globalTag'" if $globalTag;
388	>	$values .= ", crossSectionInPicobarn=$xSection" if $xSection;
389	>	$values .= ", higherOrderCrossSectionInPicobarn=$ySection" if $ySection;
390
391		$values =~ s/^, //;
392		$query = "update ntuple set $values where id=$id";
#	Line 229 | Line 397 | dbUpdate
397		}
398
399		sub
400	+	dbUpdateOther
401	+	{
402	+	my $id = shift;
403	+	my $comment = shift;
404	+	my $user = shift;
405	+
406	+	$comment = addSlashes ($comment);
407	+
408	+	my $query = "update other set lastUpdateTime=now(), lastUpdateUser='$user', comment='$comment'  where id=$id";
409	+	$db->selectdb ("ntuple");
410	+	my $results = $db->query ($query);
411	+
412	+	return $results;
413	+	}
414	+
415	+	sub
416	+	dbDelete
417	+	{
418	+	my $id = shift;
419	+
420	+	my $query = "delete from ntuple where id=$id";
421	+	$db->selectdb ("ntuple");
422	+	my $results = $db->query ($query);
423	+	}
424	+
425	+	sub
426	+	dbDeleteOther
427	+	{
428	+	my $id = shift;
429	+
430	+	my $query = "delete from other where id=$id";
431	+	$db->selectdb ("ntuple");
432	+	my $results = $db->query ($query);
433	+	}
434	+
435	+	sub
436		printHelp
437		{
438	+	my $command = shift;
439	+
440		my $exeName = $0;
441		$exeName =~ s/^.*\/([^\/]*)$/$1/;
442
443	<	print "Usage: $exeName [OPTION]... COMMAND DATASET\n";
444	<	print "Manipulates entries in the OSU Tier 3 ntuple database.\n";
445	<	print "\n";
446	<	print "Mandatory arguments to long options are mandatory for short options too.\n";
447	<	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config file used to submit the ntuple jobs";
448	<	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for this database entry";
449	<	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
450	<	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used to ntuplize the dataset";
451	<	printf "%-29s%s\n", "  -h, --help", "print this help message";
452	<	printf "%-29s%s\n", "  -l, --location DIRECTORY", "location on the Tier 3";
453	<	printf "%-29s%s\n", "  -p, --pyConfig FILE", "python config used to ntuplize the dataset";
454	<	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json returned by CRAB";
455	<	print "\n";
456	<	print "COMMAND may be one of the following:\n";
457	<	printf "%-29s%s\n", "  create", "creates the entry (assumes the dataset is present)";
458	<	printf "%-29s%s\n", "  update", "updates the entry";
459	<	printf "%-29s%s\n", "  finish", "finalizes the database entry";
460	<	printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
443	>	if ($command eq "create")
444	>	{
445	>	print "Usage: $exeName -l DIRECTORY [OPTION]... create NAME\n";
446	>	print "Creates an entry in the database for dataset NAME. The ntuples are assumed to\n";
447	>	print "be present on the Tier 3, with the status being set accordingly.\n";
448	>	print "\n";
449	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
450	>	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
451	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
452	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
453	>	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
454	>	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
455	>	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
456	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
457	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
458	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
459	>	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
460	>	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns from PREP";
461	>	printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns";
462	>	}
463	>	elsif ($command eq "createOther")
464	>	{
465	>	print "Usage: $exeName [OPTION]... createOther DIRECTORIES_AND_FILES\n";
466	>	print "Creates an entry in the database for non-ntuple data.\n";
467	>	print "\n";
468	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
469	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
470	>	}
471	>	elsif ($command eq "update")
472	>	{
473	>	print "Usage: $exeName [OPTION]... update NAME\n";
474	>	print "Updates an existing database entry for dataset NAME. If no options are given,\n";
475	>	print "updates the number of files and the size on disk of the dataset using the\n";
476	>	print "location in the database.\n";
477	>	print "\n";
478	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
479	>	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
480	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
481	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
482	>	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
483	>	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
484	>	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
485	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
486	>	printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
487	>	printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
488	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
489	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
490	>	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
491	>	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns from PREP";
492	>	printf "%-29s%s\n", "  -y, --higherOrderXSec XSEC", "higher-order cross section in picobarns";
493	>	}
494	>	elsif ($command eq "finish")
495	>	{
496	>	print "Usage: $exeName -l DIRECTORY finish NAME\n";
497	>	print "Finalizes the database entry for dataset NAME, changing its status to\n";
498	>	print "\"present\".  This is intended to be the final step in command-line based ntuple\n";
499	>	print "production.\n";
500	>	print "\n";
501	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
502	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
503	>	}
504	>	elsif ($command eq "deleteEntry")
505	>	{
506	>	print "Usage: $exeName [OPTION]... deleteEntry NAME\n";
507	>	print "Deletes the database entry for dataset NAME. This is intended primarily for\n";
508	>	print "mistaken database entries. If you wish to actually delete a set of ntuples,\n";
509	>	print "please use the \"deprecate\" command instead.\n";
510	>	print "\n";
511	>	printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
512	>	}
513	>	elsif ($command eq "deprecate")
514	>	{
515	>	print "Usage: $exeName deprecate NAME\n";
516	>	print "Marks the dataset NAME for deletion. WARNING: The dataset will be deleted from\n";
517	>	print "the Tier 3 within one week of this action. If you wish to simply delete a\n";
518	>	print "mistaken database entry, use the \"deleteEntry\" command instead.\n";
519	>	}
520	>	elsif ($command eq "uploadRelease")
521	>	{
522	>	print "Usage: $exeName -e FILE -p FILE -r DIRECTORY [OPTION]... uploadRelease NAME\n";
523	>	print "Copies an ntuple release to the appropriate area on the Tier 3, and creates a\n";
524	>	print "database entry for it, with NAME being the name of the release. An ntuple\n";
525	>	print "release is a CMSSW release with all the necessary packages added for creating\n";
526	>	print "ntuples, along with a default Python config file.\n";
527	>	print "\n";
528	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
529	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
530	>	printf "%-29s%s\n", "  -e, --recipe FILE", "file containing the recipe for the release";
531	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
532	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "default Python config for the release";
533	>	printf "%-29s%s\n", "  -r, --release DIRECTORY", "CMSSW release containing ntuple packages";
534	>	}
535	>	elsif ($command eq "downloadRelease")
536	>	{
537	>	print "Usage: $exeName [OPTION]... downloadRelease NAME\n";
538	>	print "Copies an ntuple release to the current directory on the Tier 3. Optionally\n";
539	>	print "copies the corresponding Python configuration file registered in the database.\n";
540	>	print "\n";
541	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
542	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
543	>	}
544	>	elsif ($command eq "uploadConfig")
545	>	{
546	>	print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
547	>	print "Creates an entry in the database for a customized Python configuration file.\n";
548	>	print "\n";
549	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
550	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
551	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config file";
552	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release with which to use this config";
553	>	}
554	>	else
555	>	{
556	>	print "Usage: $exeName [OPTION]... COMMAND NAME\n";
557	>	print "Manipulates entry in the OSU Tier 3 ntuple database given by NAME.\n";
558	>	print "\n";
559	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
560	>	printf "%-29s%s\n", "  -h, --help", "print help. If COMMAND is present, print help";
561	>	printf "%-29s%s\n", " ", "specific to COMMAND.";
562	>	print "\n";
563	>	print "COMMAND may be one of the following:\n";
564	>	printf "%-29s%s\n", "  create", "creates the entry";
565	>	printf "%-29s%s\n", "  createOther", "creates an entry for non-ntuple data";
566	>	printf "%-29s%s\n", "  update", "updates the entry";
567	>	printf "%-29s%s\n", "  finish", "finalizes the database entry";
568	>	printf "%-29s%s\n", "  deleteEntry", "removes the database entry";
569	>	printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
570	>	printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
571	>	printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
572	>	printf "%-29s%s\n", "  uploadConfig", "upload an ntuple config";
573	>	}
574
575		exit;
576		}
577	+
578	+	sub
579	+	sizeOfDataset
580	+	{
581	+	my $location = shift;
582	+	my $fileList = shift;
583	+
584	+	my $size = 0.0;
585	+	my $nFiles = 0;
586	+	if ($location && !$fileList)
587	+	{
588	+	if (!(-e $location))
589	+	{
590	+	print "$location does not exist!\n";
591	+	exit;
592	+	}
593	+	$nFiles = `ls $location | wc -l`;
594	+	$size = `du -s $location`;
595	+	$size =~ s/([^ ]*) .*/$1/;
596	+	$size /= 1024 * 1024;
597	+	$size = sprintf "%.2f", $size;
598	+	}
599	+	elsif ($fileList)
600	+	{
601	+	foreach my $file (split (/\n/, $fileList))
602	+	{
603	+	$nFiles++;
604	+	my $fileSize = `du -s $file`;
605	+	$fileSize =~ s/([^ ]*) .*/$1/;
606	+	$size += $fileSize;
607	+	}
608	+	$size /= 1024 * 1024;
609	+	$size = sprintf "%.2f", $size;
610	+	}
611	+
612	+	return ($nFiles, $size);
613	+	}
614	+
615	+	sub
616	+	uploadRelease
617	+	{
618	+	my $format = shift;
619	+	my $cmsswRelease = shift;
620	+	my $release = shift;
621	+	my $psetName = shift;
622	+	my $name = shift;
623	+	my $user = shift;
624	+	my $comment = shift;
625	+	my $recipeName = shift;
626	+
627	+	if (!(-e $psetName))
628	+	{
629	+	print "$psetName does not exist!\n";
630	+	exit;
631	+	}
632	+	if (!(-e $recipeName))
633	+	{
634	+	print "$recipeName does not exist!\n";
635	+	exit;
636	+	}
637	+	move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
638	+	open (PY_CONFIG, "<$psetName");
639	+	my @pset = <PY_CONFIG>;
640	+	close (PY_CONFIG);
641	+	my $pset = join ("", @pset);
642	+	open (RECIPE, "<$recipeName");
643	+	my @recipe = <RECIPE>;
644	+	close (RECIPE);
645	+	my $recipe = join ("", @recipe);
646	+
647	+	my $query = "select max(id) from ntupleRelease";
648	+	$db->selectdb ("ntuple");
649	+	my $results = $db->query ($query);
650	+	my @row = $results->fetchrow ();
651	+	my $id = 1;
652	+	$id = $row[0] + 1 if $results->numrows ();
653	+
654	+	$name = addSlashes ($name);
655	+	$pset = addSlashes ($pset);
656	+	$user = addSlashes ($user);
657	+	$format = addSlashes ($format);
658	+	$cmsswRelease = addSlashes ($cmsswRelease);
659	+	$comment = addSlashes ($comment);
660	+	$recipe = addSlashes ($recipe);
661	+
662	+	my $query = "insert into ntupleRelease (id, name, pset, user, pending, format, cmsswRelease, comment, recipe) values ($id, '$name', '$pset', '$user', 1, '$format', '$cmsswRelease', '$comment', '$recipe')";
663	+	$db->selectdb ("ntuple");
664	+	my $results = $db->query ($query);
665	+	}
666	+
667	+	sub
668	+	downloadRelease
669	+	{
670	+	my $release = shift;
671	+	my $pyConfigName = shift;
672	+
673	+	my $query = "select id,pset from ntupleRelease where name='$release'";
674	+	$db->selectdb ("ntuple");
675	+	my $results = $db->query ($query);
676	+	if ($results->numrows () != 1)
677	+	{
678	+	print "Ntuple release \"$release\" not found!\n";
679	+	exit;
680	+	}
681	+	if ($pyConfigName)
682	+	{
683	+	my @row = $results->fetchrow ();
684	+	open (PY_CONFIG, ">$pyConfigName");
685	+	print PY_CONFIG $row[1];
686	+	close (PY_CONFIG);
687	+	}
688	+	`wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
689	+	}
690	+
691	+	sub
692	+	dbOtherCreate
693	+	{
694	+	my $userAndHost = shift;
695	+	my $listOfFiles = shift;
696	+	my $comment = shift;
697	+
698	+	my $user = $userAndHost;
699	+	$user =~ s/@.*$//g;
700	+
701	+	my $size = 0.0;
702	+	my $nFiles = 0.0;
703	+	for (my $i = 0; $i < @$listOfFiles; $i++)
704	+	{
705	+	if (!(-e $$listOfFiles[$i]))
706	+	{
707	+	print "$$listOfFiles[$i] does not exist!\n";
708	+	exit;
709	+	}
710	+	$nFiles += `ls -R $$listOfFiles[$i] | grep -v ':\$' | grep -v '^\$' | wc -l`;
711	+	my $fileSize = `du -s $$listOfFiles[$i]`;
712	+	$fileSize =~ s/([^ ]*) .*/$1/;
713	+	$fileSize /= 1024 * 1024;
714	+	$size += $fileSize;
715	+	$$listOfFiles[$i] = "$ENV{'PWD'}/$$listOfFiles[$i]" if !($$listOfFiles[$i] =~ m/^\//);
716	+	}
717	+	$size = sprintf "%.2f", $size;
718	+	my $location = join ("<br />", @$listOfFiles);
719	+
720	+	my $query = "select max(id) from other";
721	+	$db->selectdb ("ntuple");
722	+	my $results = $db->query ($query);
723	+	my @row = $results->fetchrow ();
724	+	my $id = 1;
725	+	$id = $row[0] + 1 if $results->numrows ();
726	+
727	+	$user = addSlashes ($user);
728	+	$location = addSlashes ($location);
729	+	$comment = addSlashes ($comment);
730	+
731	+	$query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$userAndHost', '$user', '$location', $nFiles, $size, '$comment')";
732	+	$db->selectdb ("ntuple");
733	+	$results = $db->query ($query);
734	+
735	+	print "ID $id\n";
736	+	}
737	+
738	+	sub
739	+	uploadConfig
740	+	{
741	+	my $name = shift;
742	+	my $user = shift;
743	+	my $psetName = shift;
744	+	my $release = shift;
745	+	my $comment = shift;
746	+
747	+	if (!(-e $psetName))
748	+	{
749	+	print "$psetName does not exist!\n";
750	+	exit;
751	+	}
752	+	if ($release)
753	+	{
754	+	my $query = "select id from ntupleRelease where name='$release'";
755	+	$db->selectdb ("ntuple");
756	+	my $results = $db->query ($query);
757	+	if ($results->numrows () != 1)
758	+	{
759	+	print "Ntuple release \"$release\" not found!\n";
760	+	exit;
761	+	}
762	+	}
763	+	open (PSET, "<$psetName");
764	+	my @pset = <PSET>;
765	+	close (PSET);
766	+	my $pset = join ("", @pset);
767	+
768	+	my $query = "select max(id) from ntupleConfig";
769	+	$db->selectdb ("ntuple");
770	+	my $results = $db->query ($query);
771	+	my @row = $results->fetchrow ();
772	+	my $id = 1;
773	+	$id = $row[0] + 1 if $results->numrows ();
774	+
775	+	$name = addSlashes ($name);
776	+	$user = addSlashes ($user);
777	+	$pset = addSlashes ($pset);
778	+	$release = addSlashes ($release);
779	+	$comment = addSlashes ($comment);
780	+
781	+	my $query = "insert into ntupleConfig (id, name, pset, user, pending, ntupleRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$release', '$comment')";
782	+	$db->selectdb ("ntuple");
783	+	$results = $db->query ($query);
784	+	}
785	+
786	+	sub
787	+	locationExists
788	+	{
789	+	my $id = shift;
790	+	my $location = shift;
791	+
792	+	$location =~ s/^(.*)\/+$/$1/;
793	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
794	+	$db->selectdb ("ntuple");
795	+	my $results = $db->query ($query);
796	+	return 1 if $results->numrows ();
797	+
798	+	if ($location =~ m/^\/store\/.*$/)
799	+	{
800	+	$location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
801	+	}
802	+	elsif ($location =~ m/^\/data\/se\/store\/.*$/)
803	+	{
804	+	$location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
805	+	}
806	+
807	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
808	+	$db->selectdb ("ntuple");
809	+	my $results = $db->query ($query);
810	+	return 1 if $results->numrows ();
811	+
812	+	return 0;
813	+	}
814	+
815	+	sub
816	+	getLocation
817	+	{
818	+	my $id = shift;
819	+
820	+	my $query = "select location from ntuple where id=$id";
821	+	$db->selectdb ("ntuple");
822	+	my $results = $db->query ($query);
823	+	my @row = $results->fetchrow ();
824	+
825	+	return $row[0];
826	+	}

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines