1 |
ryzzyj |
1.1 |
import os
|
2 |
|
|
import os.path
|
3 |
|
|
import sys
|
4 |
|
|
from optparse import OptionParser
|
5 |
|
|
import urllib
|
6 |
|
|
import stat
|
7 |
|
|
import sqlite3
|
8 |
|
|
import atexit
|
9 |
|
|
import tarfile
|
10 |
|
|
import uuid
|
11 |
|
|
import shutil
|
12 |
|
|
|
13 |
|
|
datafile = "templateForDropbox.txt"
|
14 |
|
|
timetypes = ("runnumber", "lumiid", "timestamp", "hash", "userid")
|
15 |
diguida |
1.2 |
iovchecks = ("offline", "hlt", "prompt", "All")
|
16 |
ryzzyj |
1.4 |
dbfile = ""
|
17 |
ryzzyj |
1.1 |
isProd = False
|
18 |
|
|
nameOfTarball = ""
|
19 |
|
|
nameOfFileDB = ""
|
20 |
|
|
nameOfFileTXT = ""
|
21 |
|
|
rezOk = False
|
22 |
ryzzyj |
1.3 |
IOVCheckAll = False
|
23 |
|
|
duplicateEntered = False
|
24 |
|
|
tag = ""
|
25 |
ryzzyj |
1.1 |
|
26 |
|
|
def tarball(prodTag, nameOfTemplateFile, nameOfOldDBFile):
|
27 |
|
|
try:
|
28 |
|
|
if prodTag == "":
|
29 |
|
|
return ""
|
30 |
|
|
|
31 |
|
|
uuidOfFiles = str(uuid.uuid1())
|
32 |
|
|
|
33 |
|
|
global nameOfFileDB
|
34 |
|
|
global nameOfFileTXT
|
35 |
|
|
global nameOfTarball
|
36 |
|
|
nameOfFileDB = prodTag + "@" + uuidOfFiles + ".db"
|
37 |
|
|
nameOfFileTXT = prodTag + "@" + uuidOfFiles + ".txt"
|
38 |
|
|
nameOfTarball = prodTag + "@" + uuidOfFiles + ".tar.bz2"
|
39 |
|
|
|
40 |
|
|
shutil.copy(nameOfTemplateFile, nameOfFileTXT)
|
41 |
|
|
shutil.copy(nameOfOldDBFile, nameOfFileDB)
|
42 |
|
|
|
43 |
|
|
os.chmod(nameOfFileTXT, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
|
44 |
|
|
|
45 |
|
|
if isProd:
|
46 |
|
|
tar = tarfile.open(nameOfTarball,"w:bz2")
|
47 |
|
|
|
48 |
|
|
for name in [nameOfFileDB, nameOfFileTXT]:
|
49 |
|
|
tar.add(name)
|
50 |
|
|
tar.close()
|
51 |
|
|
os.chmod(nameOfTarball, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
|
52 |
|
|
|
53 |
|
|
|
54 |
|
|
except:
|
55 |
|
|
print "Failed to generate files."
|
56 |
|
|
|
57 |
|
|
|
58 |
|
|
def deleteOldFile(path):
|
59 |
|
|
if os.path.isfile(str(path)):
|
60 |
|
|
os.remove(datafile)
|
61 |
|
|
|
62 |
|
|
def downloadFile(path, where):
|
63 |
|
|
print "Downloading file %s..." % path
|
64 |
|
|
err = urllib.urlretrieve(path, where)
|
65 |
|
|
if err == 0:
|
66 |
|
|
print "Error downloading file %s! Closing program." % path
|
67 |
|
|
sys.exit()
|
68 |
|
|
|
69 |
|
|
def uploadFiles():
|
70 |
|
|
try:
|
71 |
|
|
if nameOfTarball == "":
|
72 |
|
|
raise Exception()
|
73 |
|
|
|
74 |
|
|
localfile = nameOfTarball
|
75 |
|
|
remotehost = "webcondvm.cern.ch"
|
76 |
|
|
remotefile = "/tmp"
|
77 |
|
|
|
78 |
|
|
print "Uploading files..."
|
79 |
|
|
commandBeginning = '"mv /tmp/'
|
80 |
|
|
if isProd:
|
81 |
ryzzyj |
1.5 |
commandEnding = '/DropBox"'
|
82 |
diguida |
1.6 |
if os.path.exists(nameOfTarball) and os.path.isfile(nameOfTarball) and os.path.getsize(nameOfTarball) != 0:
|
83 |
ryzzyj |
1.5 |
os.system('scp -p "%s" "%s":"%s"' % (nameOfTarball, remotehost, remotefile) )
|
84 |
|
|
#os.system('scp -p "%s" "%s":"%s"' % (nameOfTarball, remotehost, remotefile) )
|
85 |
|
|
os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfTarball, commandEnding) )
|
86 |
|
|
else:
|
87 |
|
|
print 'ERROR: Could not upload file because it has 0 size.'
|
88 |
ryzzyj |
1.1 |
else:
|
89 |
ryzzyj |
1.5 |
commandEnding = '/DropBox_test"'
|
90 |
diguida |
1.6 |
if os.path.exists(nameOfFileDB) and os.path.isfile(nameOfFileDB) and os.path.getsize(nameOfFileDB) != 0 and os.path.exists(nameOfFileTXT) and os.path.isfile(nameOfFileTXT) and os.path.getsize(nameOfFileTXT) != 0:
|
91 |
ryzzyj |
1.5 |
os.system('scp -p "%s" "%s":"%s"' % (nameOfFileDB, remotehost, remotefile) )
|
92 |
|
|
os.system('scp -p "%s" "%s":"%s"' % (nameOfFileTXT, remotehost, remotefile) )
|
93 |
|
|
os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfFileDB, commandEnding) )
|
94 |
|
|
os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfFileTXT, commandEnding) )
|
95 |
|
|
else:
|
96 |
|
|
print 'ERROR: Could not upload files because one or more files have 0 size.'
|
97 |
ryzzyj |
1.1 |
except:
|
98 |
|
|
print "Error uploading file %s" % nameOfTarball
|
99 |
|
|
|
100 |
|
|
def getDestDB(line):
|
101 |
|
|
print "Choose dropbox type:"
|
102 |
|
|
print "0) Production"
|
103 |
|
|
print "1) Preparation"
|
104 |
|
|
while True:
|
105 |
|
|
try:
|
106 |
|
|
answ = int(raw_input("Your choice: "))
|
107 |
|
|
except:
|
108 |
|
|
answ = -1
|
109 |
|
|
if (answ < 0) or (answ > 1):
|
110 |
|
|
print "Wrong number. Number has to be between %d and %d. Please try again." % (0, 1)
|
111 |
|
|
else:
|
112 |
|
|
while True:
|
113 |
|
|
answ1 = raw_input("ENTER your username: ")
|
114 |
|
|
if answ1.find(" ") != -1:
|
115 |
|
|
print "Username can't contain whitespaces. Please make sure you typed your username correctly."
|
116 |
|
|
elif len(answ1) > 0:
|
117 |
ryzzyj |
1.3 |
break
|
118 |
|
|
while True:
|
119 |
|
|
answ2 = raw_input("ENTER the Destination Database schema name: ")
|
120 |
|
|
if answ2.find(" ") != -1:
|
121 |
|
|
print "Database schema name can't contain whitespaces. Please make sure you typed the Database schema name correctly."
|
122 |
|
|
elif len(answ2) > 0:
|
123 |
|
|
break
|
124 |
ryzzyj |
1.1 |
global isProd
|
125 |
|
|
if answ == 0:
|
126 |
|
|
isProd = True
|
127 |
diguida |
1.2 |
return "oracle://cms_orcon_prod/%s" % answ2
|
128 |
ryzzyj |
1.1 |
else:
|
129 |
|
|
isProd = False
|
130 |
diguida |
1.2 |
return "oracle://cms_orcoff_prep/%s" % answ2
|
131 |
ryzzyj |
1.1 |
|
132 |
|
|
def getInputtag(line):
|
133 |
|
|
try:
|
134 |
ryzzyj |
1.3 |
connection = sqlite3.connect(dbfile)
|
135 |
ryzzyj |
1.1 |
cur = connection.cursor()
|
136 |
|
|
cur.execute("select name from metadata")
|
137 |
|
|
except ValueError:
|
138 |
|
|
print "Couldn't connect to database. Closing program."
|
139 |
|
|
sys.exit()
|
140 |
|
|
except:
|
141 |
ryzzyj |
1.3 |
print "Couldn't read input tags from %s. Closing program." % dbfile
|
142 |
ryzzyj |
1.1 |
sys.exit()
|
143 |
|
|
rows = cur.fetchall()
|
144 |
|
|
if len(rows) <= 0:
|
145 |
ryzzyj |
1.3 |
print "There are no input tags in %s. Closing program." % dbfile
|
146 |
ryzzyj |
1.1 |
sys.exit()
|
147 |
|
|
if len(rows) == 1:
|
148 |
|
|
print "Only one input tag found. Setting inputtag value as '%s'" % rows[0][0]
|
149 |
|
|
return rows[0][0]
|
150 |
|
|
else:
|
151 |
|
|
i = 0
|
152 |
|
|
for c in rows:
|
153 |
|
|
print "%d) %s" % (i, c[0])
|
154 |
|
|
i += 1
|
155 |
|
|
while True:
|
156 |
|
|
try:
|
157 |
|
|
answ = int(raw_input("Your choice: "))
|
158 |
|
|
except:
|
159 |
|
|
answ = -1
|
160 |
|
|
if (answ < 0) or (answ >= len(rows)):
|
161 |
|
|
print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(rows) - 1)
|
162 |
|
|
else:
|
163 |
|
|
return rows[answ][0]
|
164 |
|
|
|
165 |
|
|
def getTimetype(line):
|
166 |
|
|
i = 0
|
167 |
|
|
for i in range(len(timetypes)):
|
168 |
|
|
print "%d) %s" % (i, timetypes[i])
|
169 |
|
|
while True:
|
170 |
|
|
try:
|
171 |
|
|
answ = int(raw_input("Your choice: "))
|
172 |
|
|
except:
|
173 |
|
|
answ = -1
|
174 |
|
|
if (answ < 0) or (answ >= len(timetypes)):
|
175 |
|
|
print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(timetypes) - 1)
|
176 |
|
|
else:
|
177 |
|
|
return timetypes[answ]
|
178 |
|
|
|
179 |
|
|
def getIOVCheck(line):
|
180 |
|
|
i = 0
|
181 |
|
|
for i in range(len(iovchecks)):
|
182 |
|
|
print "%d) %s" % (i, iovchecks[i])
|
183 |
|
|
while True:
|
184 |
|
|
try:
|
185 |
|
|
answ = int(raw_input("Your choice: "))
|
186 |
|
|
except:
|
187 |
|
|
answ = -1
|
188 |
|
|
if (answ < 0) or (answ >= len(iovchecks)):
|
189 |
|
|
print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(iovchecks) - 1)
|
190 |
|
|
else:
|
191 |
ryzzyj |
1.3 |
if iovchecks[answ].lower() == "all":
|
192 |
|
|
global IOVCheckAll
|
193 |
|
|
IOVCheckAll = True
|
194 |
ryzzyj |
1.1 |
return iovchecks[answ]
|
195 |
|
|
|
196 |
|
|
def getTag(line):
|
197 |
|
|
answ = raw_input()
|
198 |
ryzzyj |
1.3 |
if len(answ.strip()) <= 0:
|
199 |
ryzzyj |
1.1 |
return getInput(line)
|
200 |
ryzzyj |
1.3 |
global tag
|
201 |
|
|
tag = answ
|
202 |
|
|
return answ
|
203 |
|
|
|
204 |
|
|
def getDuplicate(line):
|
205 |
|
|
answ = raw_input()
|
206 |
|
|
if len(answ.strip()) > 0:
|
207 |
|
|
global duplicateEntered
|
208 |
|
|
duplicateEntered = True
|
209 |
ryzzyj |
1.1 |
return answ
|
210 |
|
|
|
211 |
|
|
def getInput(line):
|
212 |
|
|
print "ENTER %s: " % line,
|
213 |
|
|
if line.lower() == "destdb":
|
214 |
|
|
print " "
|
215 |
|
|
return getDestDB(line)
|
216 |
|
|
elif line.lower() == "inputtag":
|
217 |
|
|
print " "
|
218 |
|
|
return getInputtag(line)
|
219 |
|
|
elif line.lower() == "timetype":
|
220 |
|
|
print " "
|
221 |
|
|
return getTimetype(line)
|
222 |
|
|
elif line.lower() == "iovcheck":
|
223 |
|
|
print " "
|
224 |
|
|
return getIOVCheck(line)
|
225 |
|
|
elif line.lower() == "tag":
|
226 |
|
|
return getTag(line)
|
227 |
ryzzyj |
1.3 |
elif line.lower().find("duplicate") == 0:
|
228 |
|
|
return getDuplicate(line)
|
229 |
ryzzyj |
1.1 |
else:
|
230 |
|
|
return raw_input()
|
231 |
|
|
|
232 |
|
|
def populateFile():
|
233 |
|
|
os.chmod(datafile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
|
234 |
|
|
tf = open(datafile, "r")
|
235 |
ryzzyj |
1.3 |
values = {}
|
236 |
|
|
donotify = True
|
237 |
|
|
duplicateTags = []
|
238 |
ryzzyj |
1.1 |
for line in tf:
|
239 |
|
|
value = line[0:line.find(" ")]
|
240 |
ryzzyj |
1.3 |
if value.lower().find("duplicate") == 0 and IOVCheckAll:
|
241 |
|
|
if donotify:
|
242 |
|
|
print "INFO: Value 'All' for 'IOVCheck' property requires you to enter at least 1 duplicate tag."
|
243 |
|
|
donotify = False
|
244 |
|
|
duplicateTags.append(value)
|
245 |
|
|
#value = value + " " + getInput(value) + "\n"
|
246 |
|
|
values[value] = getInput(value)
|
247 |
|
|
while not duplicateEntered and IOVCheckAll:
|
248 |
|
|
print "ERROR: Can't submit the metadata file while none of duplicate tags are filled."
|
249 |
|
|
for dt in duplicateTags:
|
250 |
|
|
print "ENTER %s: " % dt,
|
251 |
|
|
values[dt] = getDuplicate(dt)
|
252 |
ryzzyj |
1.1 |
global rezOk
|
253 |
|
|
rezOk = True
|
254 |
|
|
tf.close()
|
255 |
|
|
df = open(datafile, "w")
|
256 |
ryzzyj |
1.3 |
for key in values.keys():
|
257 |
|
|
df.write("%s %s\n" % (key, values[key]))
|
258 |
ryzzyj |
1.1 |
df.close()
|
259 |
|
|
|
260 |
|
|
def checkPkg():
|
261 |
|
|
if os.path.isfile("$CMSSW_BASE/lib/$SCRAM_ARCH/libCondFormats*.so"):
|
262 |
|
|
print "You have compiled version of CondFormats. Please make sure, that you haven't compiled the package. If you don't know how to do it, please contact system administrator."
|
263 |
|
|
|
264 |
|
|
def validateArgs():
|
265 |
ryzzyj |
1.3 |
global dbfile
|
266 |
ryzzyj |
1.1 |
if options.dbfile == None:
|
267 |
|
|
cont = True
|
268 |
|
|
while cont:
|
269 |
|
|
print "You didn't specify DB file. Would you like to do it now (if not, program will close)? (y/n): ",
|
270 |
|
|
answ = raw_input()
|
271 |
|
|
if answ.lower() == "y":
|
272 |
|
|
answ1 = raw_input("ENTER name of DB file: ")
|
273 |
|
|
if not os.path.isfile(answ1):
|
274 |
|
|
print "File %s not found." % answ1
|
275 |
|
|
else:
|
276 |
ryzzyj |
1.3 |
dbfile = answ1
|
277 |
ryzzyj |
1.1 |
cont = False
|
278 |
|
|
elif answ.lower() == "n":
|
279 |
|
|
sys.exit()
|
280 |
|
|
else:
|
281 |
|
|
print "Answer has to be Y or N."
|
282 |
|
|
elif not os.path.isfile(options.dbfile):
|
283 |
|
|
print "File %s not found. Closing program." % options.dbfile
|
284 |
|
|
sys.exit()
|
285 |
ryzzyj |
1.3 |
elif os.path.isfile(options.dbfile):
|
286 |
|
|
dbfile = options.dbfile
|
287 |
ryzzyj |
1.1 |
|
288 |
|
|
def prepare_exit():
|
289 |
|
|
if not rezOk:
|
290 |
|
|
sys.exit()
|
291 |
|
|
if os.path.isfile(datafile):
|
292 |
|
|
os.remove(datafile)
|
293 |
|
|
if os.path.isfile(nameOfTarball):
|
294 |
|
|
os.remove(nameOfTarball)
|
295 |
|
|
if os.path.isfile(nameOfFileDB):
|
296 |
|
|
os.remove(nameOfFileDB)
|
297 |
|
|
if os.path.isfile(nameOfFileTXT):
|
298 |
|
|
os.remove(nameOfFileTXT)
|
299 |
ryzzyj |
1.3 |
|
300 |
ryzzyj |
1.1 |
#======= Main script
|
301 |
|
|
parser = OptionParser()
|
302 |
|
|
parser.add_option("-f", "--force", dest = "force", default = False, action = "store_true", help = "forces script to skip CondFormats and CVS tag check", )
|
303 |
ryzzyj |
1.3 |
parser.add_option("-s", "--sql", dest = "dbfile", type="string", help = "name of the DB file")
|
304 |
ryzzyj |
1.1 |
(options, args) = parser.parse_args()
|
305 |
|
|
|
306 |
|
|
atexit.register(prepare_exit)
|
307 |
|
|
validateArgs()
|
308 |
|
|
deleteOldFile("templateForDropbox.txt")
|
309 |
|
|
downloadFile("http://condb.web.cern.ch/condb/DropBoxOffline/templateForDropbox.txt", datafile)
|
310 |
|
|
populateFile()
|
311 |
|
|
if not options.force:
|
312 |
|
|
checkPkg()
|
313 |
ryzzyj |
1.3 |
tarball(tag, datafile, dbfile)
|
314 |
ryzzyj |
1.1 |
uploadFiles()
|
315 |
|
|
#======= End of script
|
316 |
|
|
|
317 |
|
|
|
318 |
|
|
|
319 |
|
|
|
320 |
|
|
|
321 |
|
|
|
322 |
diguida |
1.2 |
|