ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/UserCode/OSUT3Analysis/DBTools/scripts/osudb

Comparing UserCode/OSUT3Analysis/DBTools/scripts/osudb (file contents):
Revision 1.1 by ahart, Wed Jun 6 12:14:37 2012 UTC vs.
Revision 1.24 by ahart, Thu Mar 28 17:22:28 2013 UTC

#	Line 3 | Line 3
3		use strict;
4		use Mysql;
5		use Getopt::Long;
6	+	use File::Copy;
7
8		sub getDataset;
9		sub dbUpdate;
10	+	sub dbUpdateOther;
11	+	sub dbDelete;
12	+	sub dbDeleteOther;
13		sub addSlashes;
14	+	sub uploadRelease;
15	+	sub downloadRelease;
16	+	sub datasetExists;
17	+	sub uploadConfig;
18	+	sub locationExists;
19
20	<	our $db;
20	>	our $db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User") or die "Failed to connect to Tier 3, stopped";
21
22		my %opt;
23		Getopt::Long::Configure ("bundling");
24	<	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "remoteLocation|r=s", "lumiSummary|s=s", "location|l=s", "help|h");
24	>	GetOptions (\%opt, "comment|c=s", "format|f=s", "pyConfig|p=s", "crabCfg|b=s", "jsonFile|j=s", "lumiSummary|s=s", "location|l=s", "name|n=s", "fileList|t=s", "release|r=s", "other|o", "recipe|e=s", "globalTag|g=s", "configName|i=s", "xSection|x=s", "help|h");
25		my $argc = @ARGV;
26
27	<	printHelp () if $opt{"help"};
28	<	printHelp () if $argc != 2;
29	<	printHelp () if $ARGV[0] != "create" && $ARGV[0] != "update" && $ARGV[0] != "finish" && $ARGV[0] != "deprecate";
30	<	my $id = -1;
31	<	$id = getDataset ($ARGV[1]) if $ARGV[0] ne "create";
32	<	$ARGV[0] = "create" if $id < 0;
33	<	my $status = "present";
34	<	$status = "" if $ARGV[0] eq "update";
35	<	$status = "deprecated" if $ARGV[0] eq "deprecate";
36	<	$opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
37	<	dbUpdate ($id, $ARGV[1], $ENV{"USER"}, $opt{"format"}, $opt{"location"}, $opt{"remoteLocation"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"});
27	>	printHelp ($ARGV[0]) if $opt{"help"};
28	>	printHelp () if $argc != 2 && $ARGV[0] ne "createOther";
29	>	printHelp () if $ARGV[0] ne "create" && $ARGV[0] ne "createOther" && $ARGV[0] ne "update" && $ARGV[0] ne "finish" && $ARGV[0] ne "deprecate" && $ARGV[0] ne "uploadRelease" && $ARGV[0] ne "downloadRelease" && $ARGV[0] ne "deleteEntry" && $ARGV[0] ne "uploadConfig";
30	>	if (($ARGV[0] eq "create" || $ARGV[0] eq "finish") && !$opt{"location"})
31	>	{
32	>	print "The directory containing the ntuples must be given!\n";
33	>	exit;
34	>	}
35	>	if ($ARGV[0] eq "uploadRelease" && (!$opt{"pyConfig"} || !$opt{"release"} || !$opt{"recipe"}))
36	>	{
37	>	print "The Python config, CMSSW release directory, and recipe file must be given!\n";
38	>	exit;
39	>	}
40	>	if ($ARGV[0] eq "uploadConfig" && (!$opt{"pyConfig"} || !$opt{"release"}))
41	>	{
42	>	print "The Python config and ntuple release must be given!\n";
43	>	exit;
44	>	}
45	>	if ($ARGV[0] eq "uploadRelease")
46	>	{
47	>	my $parentDir = "./$opt{'release'}";
48	>	$parentDir =~ s/CMSSW_[^\/]*//g;
49	>	$parentDir =~ s/\/\/*/\//g;
50	>	my $cmsswRelease = $opt{'release'};
51	>	$cmsswRelease =~ s/^.*CMSSW_([^\/]*).*$/CMSSW_$1/;
52	>	`tar -C $parentDir -czf $ARGV[1].tar.gz $opt{"release"}`;
53	>	$opt{"format"} = "BEAN" if !$opt{"format"};
54	>	uploadRelease ($opt{"format"}, $cmsswRelease, "$ARGV[1].tar.gz", $opt{"pyConfig"}, $ARGV[1], "$ENV{'USER'}", $opt{"comment"}, $opt{"recipe"});
55	>	}
56	>	elsif ($ARGV[0] eq "downloadRelease")
57	>	{
58	>	downloadRelease ($ARGV[1], $opt{"pyConfig"});
59	>	}
60	>	elsif ($ARGV[0] eq "uploadConfig")
61	>	{
62	>	uploadConfig ($ARGV[1], $ENV{"USER"}, $opt{"pyConfig"}, $opt{"release"}, $opt{"comment"});
63	>	}
64	>	elsif ($ARGV[0] eq "createOther")
65	>	{
66	>	my @listOfFiles = @ARGV;
67	>	@listOfFiles = reverse (@listOfFiles);
68	>	pop (@listOfFiles);
69	>	@listOfFiles = reverse (@listOfFiles);
70	>	dbOtherCreate ("$ENV{'USER'}\@$ENV{'HOSTNAME'}", \@listOfFiles, $opt{"comment"});
71	>	}
72	>	else
73	>	{
74	>	my $id = -1;
75	>	my $fullDataset;
76	>	($id, $fullDataset) = getDataset ($ARGV[0], $ARGV[1]) if $ARGV[0] ne "create" && ($ARGV[0] ne "update" || !$opt{"other"}) && ($ARGV[0] ne "deleteEntry" || !$opt{"other"});
77	>	$id = $ARGV[1] if ($ARGV[0] eq "update" && $opt{"other"}) || ($ARGV[0] eq "deleteEntry" && $opt{"other"});
78	>	$fullDataset = $ARGV[1] if $id < 0;
79	>	$ARGV[0] = "create" if $id < 0;
80	>	my $status = "present";
81	>	$status = "" if $ARGV[0] eq "update";
82	>	$status = "deprecated" if $ARGV[0] eq "deprecate";
83	>	$opt{"format"} = "BEAN" if $ARGV[0] eq "create" && !$opt{"format"};
84	>	if ($opt{"location"} && locationExists ($id, $opt{"location"}))
85	>	{
86	>	print "This location is already registered in the database!\n";
87	>	exit;
88	>	}
89	>	dbUpdate ($id, $fullDataset, "$ENV{'USER'}\@$ENV{'HOSTNAME'}", $opt{"format"}, $opt{"location"}, $opt{"name"}, $opt{"fileList"}, $status, $opt{"comment"}, $opt{"pyConfig"}, $opt{"crabCfg"}, $opt{"jsonFile"}, $opt{"lumiSummary"}, $opt{"release"}, $opt{"globalTag"}, $opt{"configName"}, $opt{"xSection"}) if !$opt{"other"};
90	>	dbUpdateOther ($id, $opt{"comment"}, "$ENV{'USER'}\@$ENV{'HOSTNAME'}") if $opt{"other"};
91	>	dbDelete ($id) if $ARGV[0] eq "deleteEntry" && !$opt{"other"};
92	>	dbDeleteOther ($id) if $ARGV[0] eq "deleteEntry" && $opt{"other"};
93	>	}
94
95		sub
96		addSlashes
97	<	{t
97	>	{
98		my $string = shift;
99
100		$string =~ s/\\/\\\\/g;
#	Line 43 | Line 108 | addSlashes
108		sub
109		getDataset
110		{
111	+	my $command = shift;
112		my $dataset = shift;
113
114		my $results;
115	<	$db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User");
116	<	my $query = "select id,user,creationTime from ntuple where dataset='$dataset' order by lastUpdateTime";
115	>	my $queryDataset = $dataset;
116	>	$queryDataset =~ s/\*/%/g;
117	>	$queryDataset =~ s/(.*)/%$1%/g;
118	>	my $query = "select id,dataset,user,creationTime from ntuple where dataset like '$queryDataset' order by creationTime";
119		$db->selectdb ("ntuple");
120		$results = $db->query ($query);
121		if ($results->numrows () == 1)
122		{
123		my @row = $results->fetchrow ();
124	<	return $row[0];
124	>	return ($row[0], $row[1]);
125		}
126		if ($results->numrows () == 0)
127		{
128	<	print "Database entry does not exist. Create it? (Y/n): ";
129	<	my $response = <STDIN>;
130	<	$response =~ s/\n//g;
131	<	$response = "y" if !$response;
132	<	exit if substr (lc ($response), 0, 1) ne 'y';
133	<	return -1;
128	>	if ($command ne "deleteEntry" && $command ne "deprecate")
129	>	{
130	>	print "Database entry does not exist. Create it? (Y/n): ";
131	>	my $response = <STDIN>;
132	>	$response =~ s/\n//g;
133	>	$response = "y" if !$response;
134	>	exit if substr (lc ($response), 0, 1) ne 'y';
135	>	return (-1, "");
136	>	}
137	>	else
138	>	{
139	>	print "Database entry does not exist.\n";
140	>	exit;
141	>	}
142		}
143	<	my %map;
143	>	my %id;
144	>	my %fullDataset;
145		print "Found multiple database entries matching\n";
146		print "\"$dataset\":\n";
147	<	print "(0) new\n";
147	>	print "( 0) new\n" if $command ne "deleteEntry" && $command ne "deprecate";
148		for (my $i = 1; $i <= $results->numrows (); $i++)
149		{
150		my @row = $results->fetchrow ();
151	<	$map{"$i"} = $row[0];
152	<	print "($i) created by $row[1] on $row[2]\n";
151	>	$id{"$i"} = $row[0];
152	>	$fullDataset{"$i"} = $row[1];
153	>	printf "(%2d) $row[1]\n", $i;
154	>	print "     created by $row[2] on $row[3]\n";
155	>	}
156	>	print "\nWhich entry would you like to modify?";
157	>	if ($command ne "deleteEntry" && $command ne "deprecate")
158	>	{
159	>	print " (Select 0 to create a new entry): "
160	>	}
161	>	else
162	>	{
163	>	print ": ";
164		}
77	–	print "\nWhich entry would you like to update? (Select 0 to create a new entry): ";
165		my $response = <STDIN>;
166		$response =~ s/[ \t\n]//g;
167	<	return -1 if $response == 0;
168	<	if (!(exists $map{$response}))
167	>	return (-1, "") if $response == 0 && $command ne "deleteEntry" && $command ne "deprecate";
168	>	if (!(exists $id{$response}))
169		{
170		print "Your selection \"$response\" was not a valid option! Quitting.\n";
171		exit;
172		}
173
174	<	return $map{$response};
174	>	return ($id{$response}, $fullDataset{$response});
175		}
176
177		sub
#	Line 92 | Line 179 | dbUpdate
179		{
180		my $id = shift;
181		my $dataset = shift;
182	<	my $user = shift;
182	>	my $userAndHost = shift;
183		my $format = shift;
184		my $location = shift;
185	<	my $remoteLocation = shift;
185	>	my $name = shift;
186	>	my $fileListName = shift;
187		my $status = shift;
188		my $comment = shift;
189		my $psetName = shift;
190		my $crabCfgName = shift;
191		my $jsonFileName = shift;
192		my $lumiSummaryName = shift;
193	+	my $release = shift;
194	+	my $globalTag = shift;
195	+	my $configName = shift;
196	+	my $xSection = shift;
197
198	+	my $user = $userAndHost;
199	+	$user =~ s/@.*$//g;
200	+
201	+	my $fileList;
202		my $pset;
203		my $crabCfg;
204		my $jsonFile;
205		my $lumiSummary;
206		my $size = 0;
207		my $nFiles = 0;
208	+	if ($fileListName)
209	+	{
210	+	if (!(-e $fileListName))
211	+	{
212	+	print "$fileListName does not exist!\n";
213	+	exit;
214	+	}
215	+	open (FILE_LIST, "<$fileListName");
216	+	my @fileList = <FILE_LIST>;
217	+	close (FILE_LIST);
218	+	$fileList = join ("", @fileList);
219	+	}
220		if ($psetName)
221		{
222		if (!(-e $psetName))
#	Line 157 | Line 265 | dbUpdate
265		close (LUMI_SUMMARY);
266		$lumiSummary = join ("", @lumiSummary);
267		}
268	<	if ($location)
269	<	{
270	<	if (!(-e $location))
271	<	{
164	<	print "$location does not exist!\n";
165	<	exit;
166	<	}
167	<	$nFiles = `ls $location | wc -l`;
168	<	$size = `du -s $location`;
169	<	$size =~ s/([^ ]*) .*/$1/;
170	<	$size /= 1024 * 1024;
171	<	$size = sprintf "%.2f", $size;
172	<	}
268	>	$release .= ".$configName" if $configName;
269	>	($nFiles, $size) = sizeOfDataset ($dataset, $location, $fileList);
270	>	my $fullLocation = $location;
271	>	$fullLocation = "$ENV{'PWD'}/$location" if !($location =~ m/^\//);
272
273		$dataset = addSlashes ($dataset);
274		$user = addSlashes ($user);
#	Line 178 | Line 277 | dbUpdate
277		$crabCfg = addSlashes ($crabCfg);
278		$jsonFile = addSlashes ($jsonFile);
279		$lumiSummary = addSlashes ($lumiSummary);
280	<	$location = addSlashes ($location);
281	<	$remoteLocation = addSlashes ($remoteLocation);
280	>	$fullLocation = addSlashes ($fullLocation);
281	>	$name = addSlashes ($name);
282	>	$fileList = addSlashes ($fileList);
283		$status = addSlashes ($status);
284		$comment = addSlashes ($comment);
285	+	$release = addSlashes ($release);
286	+	$globalTag = addSlashes ($globalTag);
287
288		my $results;
289		my $query;
290		if ($id < 0)
291		{
292	<	$db = Mysql->connect ("cmshead.mps.ohio-state.edu", "ntuple", "osuT3User");
292	>	$query = "select id from ntuple where dataset='$dataset'";
293	>	$db->selectdb ("ntuple");
294	>	$results = $db->query ($query);
295	>	if ($results->numrows ())
296	>	{
297	>	print "This dataset already exists in the database. Create anyway? (y/N): ";
298	>	my $response = <STDIN>;
299	>	$response =~ s/\n//g;
300	>	$response = "n" if !$response;
301	>	exit if substr (lc ($response), 0, 1) ne 'y';
302	>	}
303	>
304		$query = "select max(id) from ntuple";
305		$db->selectdb ("ntuple");
306		$results = $db->query ($query);
#	Line 195 | Line 308 | dbUpdate
308		my $id = 1;
309		$id = $row[0] + 1 if $results->numrows ();
310
311	<	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, user, format, location, remoteLocation, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary) values ($id, '$dataset', now(), now(), '$user', '$format', '$location', '$remoteLocation', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary')";
311	>	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag, crossSectionInPicobarn) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag', $xSection)" if $xSection;
312	>	$query = "insert into ntuple (id, dataset, creationTime, lastUpdateTime, lastUpdateUser, user, format, location, fileList, nFiles, sizeInGB, status, comment, pset, crabCfg, jsonFile, lumiSummary, version, globalTag) values ($id, '$dataset', now(), now(), '$userAndHost', '$user', '$format', '$fullLocation', '$fileList', $nFiles, $size, '$status', '$comment', '$pset', '$crabCfg', '$jsonFile', '$lumiSummary', '$release', '$globalTag')" if !$xSection;
313		}
314		if ($id > 0)
315		{
316		my $fields;
317		my $values;
318
319	<	$values .= ", dataset='$dataset'" if $dataset;
319	>	if ($name)
320	>	{
321	>	$query = "select id from ntuple where dataset='$name'";
322	>	$db->selectdb ("ntuple");
323	>	$results = $db->query ($query);
324	>	if ($results->numrows ())
325	>	{
326	>	print "This dataset already exists in the database. Create anyway? (y/N): ";
327	>	my $response = <STDIN>;
328	>	$response =~ s/\n//g;
329	>	$response = "n" if !$response;
330	>	exit if substr (lc ($response), 0, 1) ne 'y';
331	>	}
332	>	$values .= ", dataset='$name'";
333	>	}
334	>
335		$values .= ", lastUpdateTime=now()";
336	+	$values .= ", lastUpdateUser='$user'";
337		$values .= ", format='$format'" if $format;
338	<	$values .= ", location='$location'" if $location;
339	<	$values .= ", remoteLocation='$remoteLocation'" if $remoteLocation;
340	<	$values .= ", nFiles=$nFiles" if $location;
341	<	$values .= ", sizeInGB=$size" if $location;
338	>	$values .= ", location='$fullLocation'" if $location;
339	>	$values .= ", fileList='$fileList'" if $fileList;
340	>	$values .= ", nFiles=$nFiles" if $nFiles;
341	>	$values .= ", sizeInGB=$size" if $size;
342		$values .= ", status='$status'" if $status;
343		$values .= ", comment='$comment'" if $comment;
344		$values .= ", pset='$pset'" if $pset;
345		$values .= ", crabCfg='$crabCfg'" if $crabCfg;
346		$values .= ", jsonFile='$jsonFile'" if $jsonFile;
347		$values .= ", lumiSummary='$lumiSummary'" if $lumiSummary;
348	+	$values .= ", version='$release'" if $release;
349	+	$values .= ", globalTag='$globalTag'" if $globalTag;
350	+	$values .= ", crossSectionInPicobarn=$xSection" if $xSection;
351
352		$values =~ s/^, //;
353		$query = "update ntuple set $values where id=$id";
#	Line 225 | Line 358 | dbUpdate
358		}
359
360		sub
361	+	dbUpdateOther
362	+	{
363	+	my $id = shift;
364	+	my $comment = shift;
365	+	my $user = shift;
366	+
367	+	$comment = addSlashes ($comment);
368	+
369	+	my $query = "update other set lastUpdateTime=now(), lastUpdateUser='$user', comment='$comment'  where id=$id";
370	+	$db->selectdb ("ntuple");
371	+	my $results = $db->query ($query);
372	+
373	+	return $results;
374	+	}
375	+
376	+	sub
377	+	dbDelete
378	+	{
379	+	my $id = shift;
380	+
381	+	my $query = "delete from ntuple where id=$id";
382	+	$db->selectdb ("ntuple");
383	+	my $results = $db->query ($query);
384	+	}
385	+
386	+	sub
387	+	dbDeleteOther
388	+	{
389	+	my $id = shift;
390	+
391	+	my $query = "delete from other where id=$id";
392	+	$db->selectdb ("ntuple");
393	+	my $results = $db->query ($query);
394	+	}
395	+
396	+	sub
397		printHelp
398		{
399	+	my $command = shift;
400	+
401		my $exeName = $0;
402		$exeName =~ s/^.*\/([^\/]*)$/$1/;
403
404	<	print "Usage: $exeName [OPTION]... COMMAND DATASET\n";
405	<	print "Manipulates entries in the OSU Tier 3 ntuple database.\n";
406	<	print "\n";
407	<	print "Mandatory arguments to long options are mandatory for short options too.\n";
408	<	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config file used to submit the ntuple jobs";
409	<	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for this database entry";
410	<	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
411	<	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used to ntuplize the dataset";
412	<	printf "%-29s%s\n", "  -h, --help", "print this help message";
413	<	printf "%-29s%s\n", "  -l, --location DIRECTORY", "location on the Tier 3";
414	<	printf "%-29s%s\n", "  -p, --pyConfig FILE", "python config used to ntuplize the dataset";
415	<	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json returned by CRAB";
416	<	print "\n";
417	<	print "COMMAND may be one of the following:\n";
418	<	printf "%-29s%s\n", "  create", "creates the entry (assumes the dataset is present)";
419	<	printf "%-29s%s\n", "  update", "updates the entry";
420	<	printf "%-29s%s\n", "  finish", "finalizes the database entry";
421	<	printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
404	>	if ($command eq "create")
405	>	{
406	>	print "Usage: $exeName -l DIRECTORY [OPTION]... create NAME\n";
407	>	print "Creates an entry in the database for dataset NAME. The ntuples are assumed to\n";
408	>	print "be present on the Tier 3, with the status being set accordingly.\n";
409	>	print "\n";
410	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
411	>	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
412	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
413	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
414	>	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
415	>	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
416	>	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
417	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
418	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
419	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
420	>	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
421	>	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
422	>	}
423	>	elsif ($command eq "createOther")
424	>	{
425	>	print "Usage: $exeName [OPTION]... createOther DIRECTORIES_AND_FILES\n";
426	>	print "Creates an entry in the database for non-ntuple data.\n";
427	>	print "\n";
428	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
429	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
430	>	}
431	>	elsif ($command eq "update")
432	>	{
433	>	print "Usage: $exeName [OPTION]... update NAME\n";
434	>	print "Updates an existing database entry for dataset NAME.\n";
435	>	print "\n";
436	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
437	>	printf "%-29s%s\n", "  -b, --crabCfg FILE", "CRAB config used to submit ntuple jobs";
438	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
439	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format";
440	>	printf "%-29s%s\n", "  -g, --globalTag TAG", "global tag";
441	>	printf "%-29s%s\n", "  -i, --configName NAME", "name of config used to produce ntuples";
442	>	printf "%-29s%s\n", "  -j, --jsonFile FILE", "JSON file used for this dataset";
443	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
444	>	printf "%-29s%s\n", "  -n, --name NAME", "rename the entry";
445	>	printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
446	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config used to produce ntuples";
447	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release used to produce ntuples";
448	>	printf "%-29s%s\n", "  -s, --lumiSummary FILE", "lumiSummary.json reported by CRAB";
449	>	printf "%-29s%s\n", "  -x, --xSection XSEC", "cross section in picobarns for the sample";
450	>	}
451	>	elsif ($command eq "finish")
452	>	{
453	>	print "Usage: $exeName -l DIRECTORY finish NAME\n";
454	>	print "Finalizes the database entry for dataset NAME, changing its status to\n";
455	>	print "\"present\".  This is intended to be the final step in command-line based ntuple\n";
456	>	print "production.\n";
457	>	print "\n";
458	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
459	>	printf "%-29s%s\n", "  -l, --location DIRECTORY", "directory containing the ntuples";
460	>	}
461	>	elsif ($command eq "deleteEntry")
462	>	{
463	>	print "Usage: $exeName [OPTION]... deleteEntry NAME\n";
464	>	print "Deletes the database entry for dataset NAME. This is intended primarily for\n";
465	>	print "mistaken database entries. If you wish to actually delete a set of ntuples,\n";
466	>	print "please use the \"deprecate\" command instead.\n";
467	>	print "\n";
468	>	printf "%-29s%s\n", "  -o, --other", "update an entry in the non-ntuple database";
469	>	}
470	>	elsif ($command eq "deprecate")
471	>	{
472	>	print "Usage: $exeName deprecate NAME\n";
473	>	print "Marks the dataset NAME for deletion. WARNING: The dataset will be deleted from\n";
474	>	print "the Tier 3 within one week of this action. If you wish to simply delete a\n";
475	>	print "mistaken database entry, use the \"deleteEntry\" command instead.\n";
476	>	}
477	>	elsif ($command eq "uploadRelease")
478	>	{
479	>	print "Usage: $exeName -e FILE -p FILE -r DIRECTORY [OPTION]... uploadRelease NAME\n";
480	>	print "Copies an ntuple release to the appropriate area on the Tier 3, and creates a\n";
481	>	print "database entry for it, with NAME being the name of the release. An ntuple\n";
482	>	print "release is a CMSSW release with all the necessary packages added for creating\n";
483	>	print "ntuples, along with a default Python config file.\n";
484	>	print "\n";
485	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
486	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
487	>	printf "%-29s%s\n", "  -e, --recipe FILE", "file containing the recipe for the release";
488	>	printf "%-29s%s\n", "  -f, --format FORMAT", "ntuple format (default: BEAN)";
489	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "default Python config for the release";
490	>	printf "%-29s%s\n", "  -r, --release DIRECTORY", "CMSSW release containing ntuple packages";
491	>	}
492	>	elsif ($command eq "downloadRelease")
493	>	{
494	>	print "Usage: $exeName [OPTION]... downloadRelease NAME\n";
495	>	print "Copies an ntuple release to the current directory on the Tier 3. Optionally\n";
496	>	print "copies the corresponding Python configuration file registered in the database.\n";
497	>	print "\n";
498	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
499	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python configuration file name";
500	>	}
501	>	elsif ($command eq "uploadConfig")
502	>	{
503	>	print "Usage: $exeName -p FILE -r RELEASE [OPTION]... uploadConfig NAME\n";
504	>	print "Creates an entry in the database for a customized Python configuration file.\n";
505	>	print "\n";
506	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
507	>	printf "%-29s%s\n", "  -c, --comment COMMENT", "comment for the database entry";
508	>	printf "%-29s%s\n", "  -p, --pyConfig FILE", "Python config file";
509	>	printf "%-29s%s\n", "  -r, --release NAME", "ntuple release with which to use this config";
510	>	}
511	>	else
512	>	{
513	>	print "Usage: $exeName [OPTION]... COMMAND NAME\n";
514	>	print "Manipulates entry in the OSU Tier 3 ntuple database given by NAME.\n";
515	>	print "\n";
516	>	print "Mandatory arguments to long options are mandatory for short options too.\n";
517	>	printf "%-29s%s\n", "  -h, --help", "print help. If COMMAND is present, print help";
518	>	printf "%-29s%s\n", " ", "specific to COMMAND.";
519	>	print "\n";
520	>	print "COMMAND may be one of the following:\n";
521	>	printf "%-29s%s\n", "  create", "creates the entry";
522	>	printf "%-29s%s\n", "  createOther", "creates an entry for non-ntuple data";
523	>	printf "%-29s%s\n", "  update", "updates the entry";
524	>	printf "%-29s%s\n", "  finish", "finalizes the database entry";
525	>	printf "%-29s%s\n", "  deleteEntry", "removes the database entry";
526	>	printf "%-29s%s\n", "  deprecate", "marks the dataset for deletion";
527	>	printf "%-29s%s\n", "  uploadRelease", "upload an ntuple release";
528	>	printf "%-29s%s\n", "  downloadRelease", "download an ntuple release";
529	>	printf "%-29s%s\n", "  uploadConfig", "upload an ntuple config";
530	>	}
531
532		exit;
533		}
534	+
535	+	sub
536	+	sizeOfDataset
537	+	{
538	+	my $dataset = shift;
539	+	my $location = shift;
540	+	my $fileList = shift;
541	+
542	+	my $size = 0.0;
543	+	my $nFiles = 0;
544	+	if ($location && !$fileList)
545	+	{
546	+	if (!(-e $location))
547	+	{
548	+	print "$location does not exist!\n";
549	+	exit;
550	+	}
551	+	$nFiles = `ls $location | wc -l`;
552	+	$size = `du -s $location`;
553	+	$size =~ s/([^ ]*) .*/$1/;
554	+	$size /= 1024 * 1024;
555	+	$size = sprintf "%.2f", $size;
556	+	}
557	+	elsif ($fileList)
558	+	{
559	+	foreach my $file (split (/\n/, $fileList))
560	+	{
561	+	$nFiles++;
562	+	my $fileSize = `du -s $file`;
563	+	$fileSize =~ s/([^ ]*) .*/$1/;
564	+	$size += $fileSize;
565	+	}
566	+	$size /= 1024 * 1024;
567	+	$size = sprintf "%.2f", $size;
568	+	}
569	+
570	+	return ($nFiles, $size);
571	+	}
572	+
573	+	sub
574	+	uploadRelease
575	+	{
576	+	my $format = shift;
577	+	my $cmsswRelease = shift;
578	+	my $release = shift;
579	+	my $psetName = shift;
580	+	my $name = shift;
581	+	my $user = shift;
582	+	my $comment = shift;
583	+	my $recipeName = shift;
584	+
585	+	if (!(-e $psetName))
586	+	{
587	+	print "$psetName does not exist!\n";
588	+	exit;
589	+	}
590	+	if (!(-e $recipeName))
591	+	{
592	+	print "$recipeName does not exist!\n";
593	+	exit;
594	+	}
595	+	move ($release, "/home/osucms/public_html/releases/$name.tar.gz") or die "Ntuple releases may only be uploaded on the Tier 3, stopped";
596	+	open (PY_CONFIG, "<$psetName");
597	+	my @pset = <PY_CONFIG>;
598	+	close (PY_CONFIG);
599	+	my $pset = join ("", @pset);
600	+	open (RECIPE, "<$recipeName");
601	+	my @recipe = <RECIPE>;
602	+	close (RECIPE);
603	+	my $recipe = join ("\n", @recipe);
604	+
605	+	my $query = "select max(id) from ntupleRelease";
606	+	$db->selectdb ("ntuple");
607	+	my $results = $db->query ($query);
608	+	my @row = $results->fetchrow ();
609	+	my $id = 1;
610	+	$id = $row[0] + 1 if $results->numrows ();
611	+
612	+	$name = addSlashes ($name);
613	+	$pset = addSlashes ($pset);
614	+	$user = addSlashes ($user);
615	+	$format = addSlashes ($format);
616	+	$cmsswRelease = addSlashes ($cmsswRelease);
617	+	$comment = addSlashes ($comment);
618	+	$recipe = addSlashes ($recipe);
619	+
620	+	my $query = "insert into ntupleRelease (id, name, pset, user, pending, format, cmsswRelease, comment, recipe) values ($id, '$name', '$pset', '$user', 1, '$format', '$cmsswRelease', '$comment', '$recipe')";
621	+	$db->selectdb ("ntuple");
622	+	my $results = $db->query ($query);
623	+	}
624	+
625	+	sub
626	+	downloadRelease
627	+	{
628	+	my $release = shift;
629	+	my $pyConfigName = shift;
630	+
631	+	my $query = "select id,pset from ntupleRelease where name='$release'";
632	+	$db->selectdb ("ntuple");
633	+	my $results = $db->query ($query);
634	+	if ($results->numrows () != 1)
635	+	{
636	+	print "Ntuple release \"$release\" not found!\n";
637	+	exit;
638	+	}
639	+	if ($pyConfigName)
640	+	{
641	+	my @row = $results->fetchrow ();
642	+	open (PY_CONFIG, ">$pyConfigName");
643	+	print PY_CONFIG $row[1];
644	+	close (PY_CONFIG);
645	+	}
646	+	`wget "http://cmshead.mps.ohio-state.edu/~osucms/releases/$release.tar.gz"`;
647	+	}
648	+
649	+	sub
650	+	dbOtherCreate
651	+	{
652	+	my $userAndHost = shift;
653	+	my $listOfFiles = shift;
654	+	my $comment = shift;
655	+
656	+	my $user = $userAndHost;
657	+	$user =~ s/@.*$//g;
658	+
659	+	my $size = 0.0;
660	+	my $nFiles = 0.0;
661	+	for (my $i = 0; $i < @$listOfFiles; $i++)
662	+	{
663	+	if (!(-e $$listOfFiles[$i]))
664	+	{
665	+	print "$$listOfFiles[$i] does not exist!\n";
666	+	exit;
667	+	}
668	+	$nFiles += `ls -R $$listOfFiles[$i] | grep -v ':\$' | grep -v '^\$' | wc -l`;
669	+	my $fileSize = `du -s $$listOfFiles[$i]`;
670	+	$fileSize =~ s/([^ ]*) .*/$1/;
671	+	$fileSize /= 1024 * 1024;
672	+	$size += $fileSize;
673	+	$$listOfFiles[$i] = "$ENV{'PWD'}/$$listOfFiles[$i]" if !($$listOfFiles[$i] =~ m/^\//);
674	+	}
675	+	$size = sprintf "%.2f", $size;
676	+	my $location = join ("<br />", @$listOfFiles);
677	+
678	+	my $query = "select max(id) from other";
679	+	$db->selectdb ("ntuple");
680	+	my $results = $db->query ($query);
681	+	my @row = $results->fetchrow ();
682	+	my $id = 1;
683	+	$id = $row[0] + 1 if $results->numrows ();
684	+
685	+	$user = addSlashes ($user);
686	+	$location = addSlashes ($location);
687	+	$comment = addSlashes ($comment);
688	+
689	+	$query = "insert into other (id, creationTime, lastUpdateTime, lastUpdateUser, user, location, nFiles, sizeInGB, comment) values ($id, now(), now(), '$userAndHost', '$user', '$location', $nFiles, $size, '$comment')";
690	+	$db->selectdb ("ntuple");
691	+	$results = $db->query ($query);
692	+
693	+	print "ID $id\n";
694	+	}
695	+
696	+	sub
697	+	uploadConfig
698	+	{
699	+	my $name = shift;
700	+	my $user = shift;
701	+	my $psetName = shift;
702	+	my $release = shift;
703	+	my $comment = shift;
704	+
705	+	if (!(-e $psetName))
706	+	{
707	+	print "$psetName does not exist!\n";
708	+	exit;
709	+	}
710	+	if ($release)
711	+	{
712	+	my $query = "select id from ntupleRelease where name='$release'";
713	+	$db->selectdb ("ntuple");
714	+	my $results = $db->query ($query);
715	+	if ($results->numrows () != 1)
716	+	{
717	+	print "Ntuple release \"$release\" not found!\n";
718	+	exit;
719	+	}
720	+	}
721	+	open (PSET, "<$psetName");
722	+	my @pset = <PSET>;
723	+	close (PSET);
724	+	my $pset = join ("", @pset);
725	+
726	+	my $query = "select max(id) from ntupleConfig";
727	+	$db->selectdb ("ntuple");
728	+	my $results = $db->query ($query);
729	+	my @row = $results->fetchrow ();
730	+	my $id = 1;
731	+	$id = $row[0] + 1 if $results->numrows ();
732	+
733	+	$name = addSlashes ($name);
734	+	$user = addSlashes ($user);
735	+	$pset = addSlashes ($pset);
736	+	$release = addSlashes ($release);
737	+	$comment = addSlashes ($comment);
738	+
739	+	my $query = "insert into ntupleConfig (id, name, pset, user, pending, ntupleRelease, comment) values ($id, '$name', '$pset', '$user', 1, '$release', '$comment')";
740	+	$db->selectdb ("ntuple");
741	+	$results = $db->query ($query);
742	+	}
743	+
744	+	sub
745	+	locationExists
746	+	{
747	+	my $id = shift;
748	+	my $location = shift;
749	+
750	+	$location =~ s/^(.*)\/+$/$1/;
751	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
752	+	$db->selectdb ("ntuple");
753	+	my $results = $db->query ($query);
754	+	return 1 if $results->numrows ();
755	+
756	+	if ($location =~ m/^\/store\/.*$/)
757	+	{
758	+	$location =~ s/^\/store\/(.*)$/\/data\/se\/store\/$1/;
759	+	}
760	+	elsif ($location =~ m/^\/data\/se\/store\/.*$/)
761	+	{
762	+	$location =~ s/^\/data\/se\/store\/(.*)$/\/store\/$1/;
763	+	}
764	+
765	+	my $query = "select id from ntuple where location like '$location%' and id!=$id";
766	+	$db->selectdb ("ntuple");
767	+	my $results = $db->query ($query);
768	+	return 1 if $results->numrows ();
769	+
770	+	return 0;
771	+	}

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines