← Back to branch summary

~ubuntu-branches/ubuntu/feisty/aptoncd/feisty

« back to all changes in this revision

Viewing changes to RepDownload.py

  • Committer: Bazaar Package Importer
  • Author(s): Rafael Proença
  • Date: 2007-02-02 14:00:59 UTC
  • Revision ID: james.westby@ubuntu.com-20070202140059-7p801vwa0h1yymq9
Tags: upstream-0.0.99+svn20070202
Import upstream version 0.0.99+svn20070202

Show diffs side-by-side

added added

removed removed

Lines of Context:
RepDownload.py
 
1
#!/usr/bin/env python
 
2
# -*- coding: utf-8 -*-
 
3
######################################################
 
4
##
 
5
#  This program is free software; you can redistribute it and/or modify
 
6
#  it under the terms of the GNU General Public License as published
 
7
#  by the Free Software Foundation; version 2 only.
 
8
#
 
9
#  This program is distributed in the hope that it will be useful,
 
10
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
11
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
12
#  GNU General Public License for more details.
 
13
##
 
14
######################################################
 
15
## 
 
16
#  Project: AptOnCd
 
17
#  File: RepDownload.py
 
18
#  Author: Laudeci Oliveira <laudeci@gmail.com>
 
19
#  Creation: 07/12/2006
 
20
#  Changed: 
 
21
#  Purpose: Class that will control repositoy downloads
 
22
##
 
23
######################################################
 
24
import CreateAptOncd
 
25
import utils
 
26
import os
 
27
import xmlfile
 
28
import parsegz
 
29
import config
 
30
import ProgressWindow
 
31
import config
 
32
from messageBox import MessageBox
 
33
import time
 
34
import gtk
 
35
import gtk.glade
 
36
import pygtk
 
37
import DownloadGUI
 
38
import download
 
39
import msg
 
40
import gzip
 
41
import MediaSplitter
 
42
 
 
43
class RepDownload:
 
44
    def __init__(self,gladeFileName):
 
45
        self.gladeFile = gladeFileName
 
46
        self.totaltime = 0
 
47
        self.totalsize = 0
 
48
        self.speed = 0
 
49
        self.totalread = 0
 
50
        self.treeline = -1
 
51
        self.DownloadResult = gtk.RESPONSE_NONE
 
52
    
 
53
    def load_conf(self, file):
 
54
        XMLFile = xmlfile.XMLFile()
 
55
        self.node_text = XMLFile.parse(file)
 
56
        return XMLFile.load_conf(file)
 
57
     
 
58
    def CalcSpeed(self, bytesRead):
 
59
        if time.time() != self.initialtime:
 
60
            TimeFraction = ((self.totalsize - self.totalread )* (time.time() - self.initialtime)) / self.totalread
 
61
            speed = self.totalread / (time.time() - self.initialtime)
 
62
            
 
63
            self.speed = speed
 
64
            self.totaltime = TimeFraction 
 
65
            self.labelestimatedtime.set_text(str(utils.TimeFormat(self.totaltime)) )
 
66
            self.labeldownloadrate.set_text(utils.fileSizeFormat(speed))
 
67
            
 
68
            fraction = (float(self.totalread) / self.totalsize)
 
69
            self.progressbar.set_fraction(fraction)  
 
70
            
 
71
            self.progressbar.set_text("%d%%" %(100 * float(self.totalread) / self.totalsize) )
 
72
            
 
73
            
 
74
    def scrollToPath(self, index):
 
75
        try :
 
76
            self.treeprogress.scroll_to_cell(index)
 
77
        except:
 
78
            pass
 
79
 
 
80
    def pCancelEvent(self, widgets = None):
 
81
        self.DownloadResult = gtk.RESPONSE_CANCEL
 
82
        self.Packages.cancelDownload()
 
83
    
 
84
    def show_progress(self):
 
85
        self.progressgui = gtk.glade.XML(self.gladeFile, "frmFetchRepo")
 
86
        
 
87
        self.frmFetchRepo = self.progressgui.get_widget("frmFetchRepo")
 
88
        self.frmFetchRepo.set_position(gtk.WIN_POS_CENTER)
 
89
        self.frmFetchRepo.set_modal(True)
 
90
        self.frmFetchRepo.show()
 
91
        self.frmFetchRepo.connect("delete-event", self.close_fetchwindow)
 
92
        
 
93
        self.labeltotalsize = self.progressgui.get_widget("lblTotalSize")
 
94
        self.labeltotalsize.set_text(utils.fileSizeFormat(self.totalsize))
 
95
        
 
96
        self.labelnumfiles = self.progressgui.get_widget("lblNumberFiles")
 
97
        self.labelnumfiles.set_text(str(len(self.PackFiles)))
 
98
        
 
99
        self.labelestimatedtime = self.progressgui.get_widget("lblEstimatedTime")
 
100
        self.labelestimatedtime.set_text(str("%0.2f" % (self.totaltime))) 
 
101
        
 
102
        self.labeldownloadrate = self.progressgui.get_widget("lblDownloadRate")
 
103
        self.labeldownloadrate.set_text(utils.fileSizeFormat(self.speed))
 
104
        
 
105
        self.labeldist = self.progressgui.get_widget("lblDistribuition")
 
106
        self.labeldist.set_text( self.conf[3])
 
107
        
 
108
        self.labelversion = self.progressgui.get_widget("lblVersion")
 
109
        self.labelversion.set_text(self.conf[4])
 
110
 
 
111
        self.labelsection = self.progressgui.get_widget("lblSection")
 
112
        self.labelsection.set_text(self.conf[5])
 
113
 
 
114
        self.labelarch = self.progressgui.get_widget("lblArchitecture")
 
115
        self.labelarch.set_text(self.conf[6])
 
116
 
 
117
        self.labelhost = self.progressgui.get_widget("lblMirror")
 
118
        self.labelhost.set_text(self.conf[2])
 
119
        
 
120
        self.button_cancel = self.progressgui.get_widget("btnCancel")
 
121
        self.button_cancel.connect("clicked", self.close_fetchwindow)
 
122
        
 
123
        self.progressbar = self.progressgui.get_widget("pbDownloadTotal")
 
124
        
 
125
        self.treeprogress = self.progressgui.get_widget("tvDownload")
 
126
        bar = gtk.CellRendererProgress()
 
127
        
 
128
        self.mgrade = gtk.ListStore(str, str, int)
 
129
        
 
130
        self.treeprogress.set_model(self.mgrade)
 
131
        column1 = gtk.TreeViewColumn(msg.MESSAGE_0060, gtk.CellRendererText(), text = 0)
 
132
        column2 = gtk.TreeViewColumn(msg.MESSAGE_0061, gtk.CellRendererText(), text = 1)
 
133
        column3 = gtk.TreeViewColumn(msg.MESSAGE_0062, bar, value = 2)
 
134
        column1.set_resizable(True)
 
135
        column2.set_resizable(True)
 
136
        column3.set_resizable(True)
 
137
        column1.set_sort_column_id(0)
 
138
        column2.set_sort_column_id(1)
 
139
        column3.set_sort_column_id(2)
 
140
        self.treeprogress.append_column(column1)
 
141
        self.treeprogress.append_column(column2)
 
142
        self.treeprogress.append_column(column3)
 
143
 
 
144
    def close_fetchwindow(self, widget, data = None):
 
145
        self.DownloadResult = gtk.RESPONSE_CANCEL
 
146
        self.frmFetchRepo.destroy()       
 
147
         
 
148
    def SetOptions(self):
 
149
        self.DownloadGUI = DownloadGUI.DownloadGUI(self.gladeFile)
 
150
        result = self.DownloadGUI.run()
 
151
        column3.set_resizable(True)
 
152
        column1.set_sort_column_id(0)
 
153
        column2.set_sort_column_id(1)
 
154
        column3.set_sort_column_id(2)
 
155
        self.treeprogress.append_column(column1)
 
156
        self.treeprogress.append_column(column2)
 
157
        self.treeprogress.append_column(column3)
 
158
 
 
159
    def close_fetchwindow(self, widget, data = None):
 
160
        self.DownloadResult = gtk.RESPONSE_CANCEL
 
161
        self.frmFetchRepo.destroy()       
 
162
    
 
163
    def on_downloadStatus(self,widget, filedown, size, bytesRead, percent):
 
164
        if "Packages.gz" not in filedown:
 
165
            if self.DownloadResult == gtk.RESPONSE_CANCEL:
 
166
                widget.cancelDownload()
 
167
            self.totalread+= bytesRead
 
168
            self.mgrade[self.treeline][2] = percent
 
169
            self.CalcSpeed(bytesRead)
 
170
        else:
 
171
            self.totalread+= bytesRead
 
172
            fraction = (float(self.totalread) / self.totalsize)
 
173
            #fraction =  (float(percent)/100)
 
174
            self.wProgress.updateFraction(fraction)
 
175
            time.sleep(0.2)
 
176
        utils.updateUI()
 
177
        return
 
178
    
 
179
    def Download_TarGZ(self, PackFiles):
 
180
        self.initialtime = time.time()
 
181
        self.totalread = 0
 
182
 
 
183
        self.wProgress.connectEvent(self.pCancelEvent)
 
184
        
 
185
        downloaded = []
 
186
        downresult = False
 
187
        for files in PackFiles:
 
188
            try:
 
189
                self.Packages = download.Download(files[2], files[1])
 
190
                self.wProgress.set_task(msg.MESSAGE_0063 % self.conf[5])
 
191
                #self.wProgress.TaskLenght(Packages.getDownloadSize())
 
192
                self.Packages.connect('download_status',self.on_downloadStatus)
 
193
                downresult = self.Packages.download(True)
 
194
                if self.DownloadResult == gtk.RESPONSE_CANCEL:
 
195
                   self. Packages.cancelDownload()
 
196
                   downloaded = []
 
197
                   break
 
198
            except:
 
199
                pass
 
200
            if downresult:
 
201
                downloaded.append(files)    
 
202
                downresult = False
 
203
            
 
204
        return downloaded
 
205
    
 
206
    def donwload_files(self):
 
207
        self.initialtime = time.time()
 
208
        self.totalread = 0
 
209
        self.DownloadResult == gtk.RESPONSE_NONE
 
210
        for nfiles in self.PackFiles:
 
211
            if self.DownloadResult == gtk.RESPONSE_CANCEL:
 
212
                self.Packages.cancelDownload()
 
213
                break
 
214
            #self.labelsection.set_text(nfiles[download.SECTION])
 
215
            self.mgrade.append([nfiles[download.PACKAGE] , utils.fileSizeFormat(nfiles[download.SIZE]) , 0])
 
216
            self.treeline+= 1
 
217
            self.scrollToPath(self.treeline)
 
218
            
 
219
            mainPath = os.path.join(self.conf[xmlfile.PATH],self.conf[xmlfile.DISTRIBUTION],  nfiles[download.REMOTEFILEPATH])
 
220
            
 
221
            utils.mkdir(mainPath)
 
222
            
 
223
            filePath = os.path.join(mainPath,nfiles[download.DEBFILENAME] )
 
224
    
 
225
            if not utils.fileExist(filePath):
 
226
                remotePath = self.conf[xmlfile.METHOD] + '://' + self.conf[xmlfile.HOST] + '/' + self.conf[xmlfile.DISTRIBUTION]  + '/' + nfiles[download.FILENAME]
 
227
                self.Packages = download.Download(remotePath, mainPath)
 
228
                self.Packages.connect('download_status',self.on_downloadStatus)
 
229
                self.Packages.download(True)
 
230
            else:
 
231
                fraction = float((100 * float(nfiles[download.SIZE])) / self.totalsize)
 
232
                self.on_downloadStatus(self.frmFetchRepo, nfiles[download.SIZE],float(nfiles[download.SIZE]), float(nfiles[download.SIZE]), float(fraction))
 
233
                self.mgrade[self.treeline][2] =100
 
234
       
 
235
        if self.DownloadResult == gtk.RESPONSE_CANCEL:
 
236
            dialog = gtk.MessageDialog(None, gtk.DIALOG_MODAL, gtk.MESSAGE_WARNING, gtk.BUTTONS_OK, msg.MESSAGE_0067)
 
237
            dialog.run()
 
238
            dialog.destroy()
 
239
            
 
240
    def SetOptions(self):
 
241
        self.totalsize = 0
 
242
        self.DownloadGUI = DownloadGUI.DownloadGUI(self.gladeFile)
 
243
        result = self.DownloadGUI.run()
 
244
        
 
245
        if result  != gtk.RESPONSE_OK:
 
246
            dialog = gtk.MessageDialog(None, gtk.DIALOG_MODAL, gtk.MESSAGE_WARNING, gtk.BUTTONS_OK, msg.MESSAGE_0067)
 
247
            dialog.run()
 
248
            dialog.destroy()
 
249
            return
 
250
        
 
251
        
 
252
        self.conf = self.load_conf(config.XML_FILE)
 
253
        ## just removing for teste purpouse
 
254
 
 
255
        self.wProgress = ProgressWindow.AptOnCDProgressDialog(self.DownloadGUI.getWindow())
 
256
        self.wProgress.Title(msg.MESSAGE_0059 )
 
257
        self.wProgress.Task(msg.MESSAGE_0057)
 
258
        self.wProgress.Description(msg.MESSAGE_0058) 
 
259
        self.wProgress.TaskTitle(msg.MESSAGE_0056 % self.conf[xmlfile.HOST] )
 
260
        self.wProgress.set_text('%d%%' % 0)
 
261
        
 
262
        utils.updateUI()
 
263
       
 
264
        #create the path fot destination of all Packages.tar.gz
 
265
        #folowing the rule down
 
266
        #DISTRIBUTION/dists/VERSION/SECTION/binary-ARCH/
 
267
        mainPath = os.path.join(self.conf[xmlfile.PATH],self.conf[xmlfile.DISTRIBUTION], 'dists', self.conf[xmlfile.VERSION])
 
268
        utils.mkdir(mainPath)
 
269
        tarfiles = []
 
270
        tarsize =0
 
271
        
 
272
        for sectionName in self.conf[xmlfile.SECTION].split(';'): 
 
273
           Pathx = os.path.join(mainPath, sectionName , 'binary-' + self.conf[xmlfile.ARCHITECTURE] )
 
274
           utils.mkdir(Pathx)
 
275
 
 
276
           downloadGz = ( \
 
277
                          self.conf[xmlfile.METHOD]  + '://' + self.conf[xmlfile.HOST] + '/' + self.conf[xmlfile.DISTRIBUTION]  + \
 
278
                          '/dists/' + self.conf[xmlfile.VERSION]  + '/' + sectionName  + '/binary-' + \
 
279
                          self.conf[xmlfile.ARCHITECTURE]  + '/Packages.gz')
 
280
           size = float(download.getRemoteFileSize(downloadGz))
 
281
           self.totalsize+=size
 
282
           
 
283
           tarfiles.append( ['Packages.gz',Pathx , downloadGz, size])
 
284
              
 
285
        validTars = self.Download_TarGZ(tarfiles)
 
286
        
 
287
        if len(validTars) == 0 or self.DownloadResult == gtk.RESPONSE_CANCEL:
 
288
            self.wProgress.destroy()
 
289
            dialog = gtk.MessageDialog(None, gtk.DIALOG_MODAL, gtk.MESSAGE_WARNING, gtk.BUTTONS_OK, msg.MESSAGE_0067)
 
290
            dialog.run()
 
291
            dialog.destroy()
 
292
            return
 
293
        
 
294
        self.PackFiles = []
 
295
        self.totalsize = 0
 
296
        self.totalread = 0
 
297
        for tar in validTars:
 
298
            s = parsegz.openGZ(tar[1] + '/Packages.gz'  )
 
299
            self.PackFiles.extend(parsegz.ParseFile(s).Parse())
 
300
            
 
301
        for nfiles in self.PackFiles:
 
302
            if nfiles[3] != '':
 
303
                self.totalsize+= float(nfiles[3])
 
304
        self.wProgress.destroy()
 
305
 
 
306
        if self.DownloadResult != gtk.RESPONSE_CANCEL:
 
307
            self.show_progress()
 
308
            self.donwload_files()
 
309
        self.frmFetchRepo.hide()
 
310
     
 
311
        self.DownloadResult = gtk.RESPONSE_OK
 
312
 
 
313
        if self.DownloadResult != gtk.RESPONSE_CANCEL:
 
314
            if self.conf[xmlfile.MEDIA] == 'DVD':
 
315
                cdSize =config.DVD
 
316
            else:
 
317
                cdSize = config.CD
 
318
            a = MediaSplitter.Splitter(self.conf[xmlfile.PATH], config.WORK_DIR, self.conf[xmlfile.DISTRIBUTION], self.conf[xmlfile.VERSION], self.conf[xmlfile.SECTION].split(';'), self.conf[xmlfile.ARCHITECTURE], cdSize)
 
319
            iSorted = a.SplitMedia()
 
320
            
 
321
            #we must show a progress window while copy is in progress
 
322
            #TODO - create strings in msg.py and input error handler
 
323
            taskLen = 0
 
324
            for s in iSorted:
 
325
                taskLen += len(s[1])
 
326
                
 
327
            self.wProgress.Title(msg.MESSAGE_0064 )
 
328
            self.wProgress.Task(msg.MESSAGE_0066)
 
329
            self.wProgress.TaskLenght( int(taskLen)) # Task's len
 
330
            self.wProgress.Description(msg.MESSAGE_0065) 
 
331
            self.wProgress.TaskTitle(msg.MESSAGE_0064 )
 
332
            self.wProgress.set_text('%d%%' % 0)
 
333
 
 
334
            i =1
 
335
            for s in iSorted:
 
336
                #print s[0]
 
337
                for n in s[1]:
 
338
                    #filesize = (float(n.Size)/1024.0**2 + 1)
 
339
                    utils.mkdir(n.TarGZ_Path % n.PackageSection)
 
340
                    utils.copyFile(n.SourceLocation, n.Destination)
 
341
                    self.wProgress.update(i)
 
342
                    i +=1
 
343
                    
 
344
            curdir = os.getcwd()
 
345
            cds ={}
 
346
            for s in iSorted:
 
347
                mediadir = s[0]+'/'
 
348
                os.chdir(mediadir)
 
349
                medianame = s[0].split('/')[-1].encode('utf8')
 
350
                cds[medianame] = []
 
351
                for sec in self.conf[xmlfile.SECTION].split(';'): 
 
352
                   pkgzdir = "dists/%s/%s/binary-%s/" % ( self.conf[xmlfile.VERSION],sec,self.conf[xmlfile.ARCHITECTURE])   
 
353
                   if utils.pathExists(os.path.join(mediadir,pkgzdir)):
 
354
                       os.system('apt-ftparchive packages  pool/ > ' + pkgzdir + 'Packages')
 
355
                       #make a .gz file
 
356
                       PackagesFile = os.path.join(mediadir,pkgzdir +'Packages')
 
357
                       fileObj = gzip.GzipFile(PackagesFile + '.gz', 'wb');
 
358
                       fileObj.write(file(PackagesFile, 'rb').read())
 
359
                       fileObj.close()
 
360
                       utils.compress(PackagesFile, PackagesFile)
 
361
                       
 
362
                       os.system('apt-ftparchive release  dists/' + self.conf[xmlfile.VERSION] + ' > dists/' + self.conf[xmlfile.VERSION] + '/Release')
 
363
                       
 
364
                       cds[medianame].append(sec)
 
365
 
 
366
                os.system('ln -s . ' + self.conf[xmlfile.DISTRIBUTION])
 
367
                os.chdir(curdir)
 
368
            
 
369
            self.wProgress.Task(msg.MESSAGE_0020) 
 
370
            #print cds  
 
371
            for x in cds.keys():
 
372
                section = ''
 
373
                for n in cds[x]:
 
374
                    section+=n+'-'
 
375
                section = section[:-1]
 
376
                isoname = self.conf[xmlfile.DISTRIBUTION] + '-' + self.conf[xmlfile.VERSION] + '-' 
 
377
                isoname += section + '-' + self.conf[xmlfile.ARCHITECTURE] +'-' + x
 
378
                os.system('mkisofs -quiet -iso-level 4 -pad -l -r -J -joliet-long -v -V "aptoncd-'+ x  +'" -hide-rr-moved -o ' + os.path.join(config.WORK_DIR ,isoname +'.iso') + ' ' + mediadir + ' 2> /dev/null')
 
379
            
 
380
            utils.removePath(os.path.join(config.WORK_DIR,'repository'))    
 
381
            self.wProgress.destroy()
 
382
        #the comment below must be removed when code is ok
 
383
        #self.close_fetchwindow(self)