ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/ryzzyj/uploadDropboxFiles.py
Revision: 1.6
Committed: Wed Sep 8 09:26:32 2010 UTC (14 years, 7 months ago) by diguida
Content type: text/x-python
Branch: MAIN
Changes since 1.5: +2 -3 lines
Log Message:
bug fixing: still timetype to be implemented

File Contents

# User Rev Content
1 ryzzyj 1.1 import os
2     import os.path
3     import sys
4     from optparse import OptionParser
5     import urllib
6     import stat
7     import sqlite3
8     import atexit
9     import tarfile
10     import uuid
11     import shutil
12    
13     datafile = "templateForDropbox.txt"
14     timetypes = ("runnumber", "lumiid", "timestamp", "hash", "userid")
15 diguida 1.2 iovchecks = ("offline", "hlt", "prompt", "All")
16 ryzzyj 1.4 dbfile = ""
17 ryzzyj 1.1 isProd = False
18     nameOfTarball = ""
19     nameOfFileDB = ""
20     nameOfFileTXT = ""
21     rezOk = False
22 ryzzyj 1.3 IOVCheckAll = False
23     duplicateEntered = False
24     tag = ""
25 ryzzyj 1.1
26     def tarball(prodTag, nameOfTemplateFile, nameOfOldDBFile):
27     try:
28     if prodTag == "":
29     return ""
30    
31     uuidOfFiles = str(uuid.uuid1())
32    
33     global nameOfFileDB
34     global nameOfFileTXT
35     global nameOfTarball
36     nameOfFileDB = prodTag + "@" + uuidOfFiles + ".db"
37     nameOfFileTXT = prodTag + "@" + uuidOfFiles + ".txt"
38     nameOfTarball = prodTag + "@" + uuidOfFiles + ".tar.bz2"
39    
40     shutil.copy(nameOfTemplateFile, nameOfFileTXT)
41     shutil.copy(nameOfOldDBFile, nameOfFileDB)
42    
43     os.chmod(nameOfFileTXT, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
44    
45     if isProd:
46     tar = tarfile.open(nameOfTarball,"w:bz2")
47    
48     for name in [nameOfFileDB, nameOfFileTXT]:
49     tar.add(name)
50     tar.close()
51     os.chmod(nameOfTarball, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
52    
53    
54     except:
55     print "Failed to generate files."
56    
57    
58     def deleteOldFile(path):
59     if os.path.isfile(str(path)):
60     os.remove(datafile)
61    
62     def downloadFile(path, where):
63     print "Downloading file %s..." % path
64     err = urllib.urlretrieve(path, where)
65     if err == 0:
66     print "Error downloading file %s! Closing program." % path
67     sys.exit()
68    
69     def uploadFiles():
70     try:
71     if nameOfTarball == "":
72     raise Exception()
73    
74     localfile = nameOfTarball
75     remotehost = "webcondvm.cern.ch"
76     remotefile = "/tmp"
77    
78     print "Uploading files..."
79     commandBeginning = '"mv /tmp/'
80     if isProd:
81 ryzzyj 1.5 commandEnding = '/DropBox"'
82 diguida 1.6 if os.path.exists(nameOfTarball) and os.path.isfile(nameOfTarball) and os.path.getsize(nameOfTarball) != 0:
83 ryzzyj 1.5 os.system('scp -p "%s" "%s":"%s"' % (nameOfTarball, remotehost, remotefile) )
84     #os.system('scp -p "%s" "%s":"%s"' % (nameOfTarball, remotehost, remotefile) )
85     os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfTarball, commandEnding) )
86     else:
87     print 'ERROR: Could not upload file because it has 0 size.'
88 ryzzyj 1.1 else:
89 ryzzyj 1.5 commandEnding = '/DropBox_test"'
90 diguida 1.6 if os.path.exists(nameOfFileDB) and os.path.isfile(nameOfFileDB) and os.path.getsize(nameOfFileDB) != 0 and os.path.exists(nameOfFileTXT) and os.path.isfile(nameOfFileTXT) and os.path.getsize(nameOfFileTXT) != 0:
91 ryzzyj 1.5 os.system('scp -p "%s" "%s":"%s"' % (nameOfFileDB, remotehost, remotefile) )
92     os.system('scp -p "%s" "%s":"%s"' % (nameOfFileTXT, remotehost, remotefile) )
93     os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfFileDB, commandEnding) )
94     os.system('ssh "%s" "%s""%s" "%s" ' % (remotehost, commandBeginning, nameOfFileTXT, commandEnding) )
95     else:
96     print 'ERROR: Could not upload files because one or more files have 0 size.'
97 ryzzyj 1.1 except:
98     print "Error uploading file %s" % nameOfTarball
99    
100     def getDestDB(line):
101     print "Choose dropbox type:"
102     print "0) Production"
103     print "1) Preparation"
104     while True:
105     try:
106     answ = int(raw_input("Your choice: "))
107     except:
108     answ = -1
109     if (answ < 0) or (answ > 1):
110     print "Wrong number. Number has to be between %d and %d. Please try again." % (0, 1)
111     else:
112     while True:
113     answ1 = raw_input("ENTER your username: ")
114     if answ1.find(" ") != -1:
115     print "Username can't contain whitespaces. Please make sure you typed your username correctly."
116     elif len(answ1) > 0:
117 ryzzyj 1.3 break
118     while True:
119     answ2 = raw_input("ENTER the Destination Database schema name: ")
120     if answ2.find(" ") != -1:
121     print "Database schema name can't contain whitespaces. Please make sure you typed the Database schema name correctly."
122     elif len(answ2) > 0:
123     break
124 ryzzyj 1.1 global isProd
125     if answ == 0:
126     isProd = True
127 diguida 1.2 return "oracle://cms_orcon_prod/%s" % answ2
128 ryzzyj 1.1 else:
129     isProd = False
130 diguida 1.2 return "oracle://cms_orcoff_prep/%s" % answ2
131 ryzzyj 1.1
132     def getInputtag(line):
133     try:
134 ryzzyj 1.3 connection = sqlite3.connect(dbfile)
135 ryzzyj 1.1 cur = connection.cursor()
136     cur.execute("select name from metadata")
137     except ValueError:
138     print "Couldn't connect to database. Closing program."
139     sys.exit()
140     except:
141 ryzzyj 1.3 print "Couldn't read input tags from %s. Closing program." % dbfile
142 ryzzyj 1.1 sys.exit()
143     rows = cur.fetchall()
144     if len(rows) <= 0:
145 ryzzyj 1.3 print "There are no input tags in %s. Closing program." % dbfile
146 ryzzyj 1.1 sys.exit()
147     if len(rows) == 1:
148     print "Only one input tag found. Setting inputtag value as '%s'" % rows[0][0]
149     return rows[0][0]
150     else:
151     i = 0
152     for c in rows:
153     print "%d) %s" % (i, c[0])
154     i += 1
155     while True:
156     try:
157     answ = int(raw_input("Your choice: "))
158     except:
159     answ = -1
160     if (answ < 0) or (answ >= len(rows)):
161     print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(rows) - 1)
162     else:
163     return rows[answ][0]
164    
165     def getTimetype(line):
166     i = 0
167     for i in range(len(timetypes)):
168     print "%d) %s" % (i, timetypes[i])
169     while True:
170     try:
171     answ = int(raw_input("Your choice: "))
172     except:
173     answ = -1
174     if (answ < 0) or (answ >= len(timetypes)):
175     print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(timetypes) - 1)
176     else:
177     return timetypes[answ]
178    
179     def getIOVCheck(line):
180     i = 0
181     for i in range(len(iovchecks)):
182     print "%d) %s" % (i, iovchecks[i])
183     while True:
184     try:
185     answ = int(raw_input("Your choice: "))
186     except:
187     answ = -1
188     if (answ < 0) or (answ >= len(iovchecks)):
189     print "Wrong number. Number has to be between %d and %d. Please try again." % (0, len(iovchecks) - 1)
190     else:
191 ryzzyj 1.3 if iovchecks[answ].lower() == "all":
192     global IOVCheckAll
193     IOVCheckAll = True
194 ryzzyj 1.1 return iovchecks[answ]
195    
196     def getTag(line):
197     answ = raw_input()
198 ryzzyj 1.3 if len(answ.strip()) <= 0:
199 ryzzyj 1.1 return getInput(line)
200 ryzzyj 1.3 global tag
201     tag = answ
202     return answ
203    
204     def getDuplicate(line):
205     answ = raw_input()
206     if len(answ.strip()) > 0:
207     global duplicateEntered
208     duplicateEntered = True
209 ryzzyj 1.1 return answ
210    
211     def getInput(line):
212     print "ENTER %s: " % line,
213     if line.lower() == "destdb":
214     print " "
215     return getDestDB(line)
216     elif line.lower() == "inputtag":
217     print " "
218     return getInputtag(line)
219     elif line.lower() == "timetype":
220     print " "
221     return getTimetype(line)
222     elif line.lower() == "iovcheck":
223     print " "
224     return getIOVCheck(line)
225     elif line.lower() == "tag":
226     return getTag(line)
227 ryzzyj 1.3 elif line.lower().find("duplicate") == 0:
228     return getDuplicate(line)
229 ryzzyj 1.1 else:
230     return raw_input()
231    
232     def populateFile():
233     os.chmod(datafile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
234     tf = open(datafile, "r")
235 ryzzyj 1.3 values = {}
236     donotify = True
237     duplicateTags = []
238 ryzzyj 1.1 for line in tf:
239     value = line[0:line.find(" ")]
240 ryzzyj 1.3 if value.lower().find("duplicate") == 0 and IOVCheckAll:
241     if donotify:
242     print "INFO: Value 'All' for 'IOVCheck' property requires you to enter at least 1 duplicate tag."
243     donotify = False
244     duplicateTags.append(value)
245     #value = value + " " + getInput(value) + "\n"
246     values[value] = getInput(value)
247     while not duplicateEntered and IOVCheckAll:
248     print "ERROR: Can't submit the metadata file while none of duplicate tags are filled."
249     for dt in duplicateTags:
250     print "ENTER %s: " % dt,
251     values[dt] = getDuplicate(dt)
252 ryzzyj 1.1 global rezOk
253     rezOk = True
254     tf.close()
255     df = open(datafile, "w")
256 ryzzyj 1.3 for key in values.keys():
257     df.write("%s %s\n" % (key, values[key]))
258 ryzzyj 1.1 df.close()
259    
260     def checkPkg():
261     if os.path.isfile("$CMSSW_BASE/lib/$SCRAM_ARCH/libCondFormats*.so"):
262     print "You have compiled version of CondFormats. Please make sure, that you haven't compiled the package. If you don't know how to do it, please contact system administrator."
263    
264     def validateArgs():
265 ryzzyj 1.3 global dbfile
266 ryzzyj 1.1 if options.dbfile == None:
267     cont = True
268     while cont:
269     print "You didn't specify DB file. Would you like to do it now (if not, program will close)? (y/n): ",
270     answ = raw_input()
271     if answ.lower() == "y":
272     answ1 = raw_input("ENTER name of DB file: ")
273     if not os.path.isfile(answ1):
274     print "File %s not found." % answ1
275     else:
276 ryzzyj 1.3 dbfile = answ1
277 ryzzyj 1.1 cont = False
278     elif answ.lower() == "n":
279     sys.exit()
280     else:
281     print "Answer has to be Y or N."
282     elif not os.path.isfile(options.dbfile):
283     print "File %s not found. Closing program." % options.dbfile
284     sys.exit()
285 ryzzyj 1.3 elif os.path.isfile(options.dbfile):
286     dbfile = options.dbfile
287 ryzzyj 1.1
288     def prepare_exit():
289     if not rezOk:
290     sys.exit()
291     if os.path.isfile(datafile):
292     os.remove(datafile)
293     if os.path.isfile(nameOfTarball):
294     os.remove(nameOfTarball)
295     if os.path.isfile(nameOfFileDB):
296     os.remove(nameOfFileDB)
297     if os.path.isfile(nameOfFileTXT):
298     os.remove(nameOfFileTXT)
299 ryzzyj 1.3
300 ryzzyj 1.1 #======= Main script
301     parser = OptionParser()
302     parser.add_option("-f", "--force", dest = "force", default = False, action = "store_true", help = "forces script to skip CondFormats and CVS tag check", )
303 ryzzyj 1.3 parser.add_option("-s", "--sql", dest = "dbfile", type="string", help = "name of the DB file")
304 ryzzyj 1.1 (options, args) = parser.parse_args()
305    
306     atexit.register(prepare_exit)
307     validateArgs()
308     deleteOldFile("templateForDropbox.txt")
309     downloadFile("http://condb.web.cern.ch/condb/DropBoxOffline/templateForDropbox.txt", datafile)
310     populateFile()
311     if not options.force:
312     checkPkg()
313 ryzzyj 1.3 tarball(tag, datafile, dbfile)
314 ryzzyj 1.1 uploadFiles()
315     #======= End of script
316    
317    
318    
319    
320    
321    
322 diguida 1.2