~grubng-dev/grubng/clients-python

« back to all changes in this revision

Viewing changes to grub.py

  • Committer: Giorgos Logiotatidis
  • Date: 2009-10-29 17:00:07 UTC
  • Revision ID: seadog@sealabs.net-20091029170007-5h1a3sat0l0i1zty
Logging functionality using python.logging
Added DATAPRFX option to store arc files

Show diffs side-by-side

added added

removed removed

Lines of Context:
32
32
from cStringIO import StringIO
33
33
import zlib
34
34
from shutil import copy
 
35
import logging
 
36
import logging.handlers
35
37
 
36
38
NUM_CLIENTS = 5
37
39
NUM_UPLOADERS = 1
39
41
PASSWORD = None
40
42
VERBOSE = True
41
43
DEBUG = False
 
44
LOG = False
42
45
DISPATCHER = 'http://soap.grub.org/cgi-bin/dispatch.cgi'
43
46
VERSION = '0.4.3'
44
 
 
 
47
WORKUNITS = 'workunits.dat'
 
48
DATAPRFX = None
45
49
 
46
50
# set socket timeout
47
51
socket.setdefaulttimeout(30)
64
68
    
65
69
    def run(self):
66
70
        while self.LOOP:           
67
 
            if os.path.exists("workunits.dat") == False:
 
71
            if os.path.exists( DATAPRFX + WORKUNITS ) == False:
68
72
                # nothing ready yet. Get some sleep and try again later
69
73
                time.sleep(self._sleeptime)
70
74
                continue
72
76
            threadlock.acquire()
73
77
            
74
78
            try:
75
 
                fp = open("workunits.dat", "rb")           
 
79
                fp = open(DATAPRFX + WORKUNITS, "rb")           
76
80
                workUnit = fp.readlines()
77
81
                fp.close()
78
 
                fp = open("workunits.dat", "wb")
 
82
                fp = open(DATAPRFX + WORKUNITS, "wb")
79
83
                fp.writelines(workUnit[1:])
80
84
                fp.close()
81
85
            except IOError, e:
82
 
                printer("Workunit.dat open failed in %s: %s" % (self.getName(), e), debug=1)
 
86
                printer( str(DATAPRFX + WORKUNITS) + "open failed in %s: %s" % (self.getName(), e), debug=1)
83
87
            
84
88
            threadlock.release()
85
89
               
101
105
                    
102
106
                # compress the workunit
103
107
                try:
104
 
                    self.compressWorkUnit(filename)
 
108
                    self.compressWorkUnit(DATAPRFX + filename)
105
109
                except IOError, e:
106
110
                    # hmm arc file maybe deleted. just move on
107
111
                    printer("CompressWorkUnit error %s:%s" % (self.getName(), e), debug=1)
115
119
                        break
116
120
                    
117
121
                    try:
118
 
                        result = self.uploadWorkUnit(host, url, filename)
 
122
                        result = self.uploadWorkUnit(host, url, DATAPRFX + filename)
119
123
                    except (httplib.HTTPException, socket.error):
120
124
                        result = -500
121
125
 
128
132
                    elif result == -401:
129
133
                        # skip cleaning by changing the filename
130
134
                        if DEBUG:
131
 
                            copy(filename, "%s.401" % filename)
132
 
                            copy("%s.wu" % filename, "%s.wu.401" % filename)
 
135
                            copy(DATAPRFX + filename, "%s.401" % str(DATAPRFX + filename))
 
136
                            copy("%s.wu" % filename, "%s.wu.401" % str(DATAPRFX + filename ))
133
137
                        break
134
138
                       
135
 
                self.cleanup(filename)  
 
139
                self.cleanup(DATAPRFX + filename)  
136
140
 
137
141
    def compressWorkUnit(self, filename):
138
142
        # gzip the data
260
264
                    line = line.split()[1].split('/')
261
265
                    self.filename = line[len(line)-1][:-3]
262
266
                    
263
 
                    if os.path.exists("%s.gz" % self.filename):
 
267
                    if os.path.exists("%s.gz" % str(DATAPRFX + self.filename)):
264
268
                        # oops file already exists!
265
269
                        # The server feed us previous stuff
266
270
                        # we should fetch another wu
280
284
            
281
285
            # if debug save workunit
282
286
            if DEBUG:
283
 
                f = open("%s.wu" % self.filename, 'wb')
 
287
                f = open("%s.wu" % str(DATAPRFX + self.filename), 'wb')
284
288
                f.writelines(self.wu)
285
289
                f.close()
286
290
 
287
 
            self.arc = open("%s" % self.filename, 'wb')
 
291
            self.arc = open("%s" % str(DATAPRFX + self.filename), 'wb')
288
292
            self.arc.write("filedesc://dummy.arc.gz 0.0.0.0 %s text/plain 69\n" % time.strftime("%Y%m%d%H%M%S", time.localtime()))
289
293
            self.arc.write('1 0 grub.org\n')
290
294
            self.arc.write('URL IP-address Archive-date Content-type Archive-length\n\n')
462
466
            
463
467
        threadlock.acquire()
464
468
        try:            
465
 
            fp = open("workunits.dat", "ab")
 
469
            fp = open(DATAPRFX + WORKUNITS, "ab")
466
470
            fp.write("%s %s %s\n" % (host, url, self.filename))
467
471
            fp.close()
468
472
        except IOError, e:
488
492
 
489
493
def printer(msg, error=0, debug=0):
490
494
    p = False
491
 
    if (debug==1):
492
 
        if DEBUG:
493
 
            p = True
 
495
    if (debug==1 and DEBUG):
 
496
        logging.debug(msg)
494
497
    elif (error==1):
495
 
        p = True
 
498
        logging.error(msg)
496
499
    elif (VERBOSE == True):
497
 
        p = True
 
500
        logging.info(msg)
498
501
            
499
 
    if p:
500
 
        print "[%s] %s" % (time.strftime("%a %b %d %H:%M:%S %Y"), msg)
501
 
 
502
 
    sys.stdout.flush()
 
502
def setupLogging():
 
503
    global LOG
 
504
    logging.getLogger('').setLevel(level=logging.DEBUG)
 
505
 
 
506
    formatter = logging.Formatter('%(asctime)s %(message)s', '[%a %b %d %H:%M:%S %Y]')
 
507
 
 
508
    # log to stout
 
509
    console = logging.StreamHandler()
 
510
    console.setFormatter(formatter)
 
511
    console.setLevel(logging.DEBUG)
 
512
    logging.getLogger('').addHandler(console)
 
513
 
 
514
    if LOG:
 
515
        # log to lfile
 
516
        handler = logging.handlers.TimedRotatingFileHandler("grub.log", 'midnight', backupCount=5)
 
517
        handler.setFormatter(formatter)
 
518
        handler.setLevel(logging.DEBUG)
 
519
        logging.getLogger('').addHandler(handler)
 
520
    
 
521
                        
 
522
 
503
523
 
504
524
def printUsage():
505
525
    global VERSION
540
560
            printer("Error reading username and password from %s" % passwdfile)
541
561
            sys.exit(1)
542
562
 
 
563
def addslash(path):
 
564
        if path[len(path)-1:] == '/':
 
565
            return path
 
566
        else:
 
567
            return path + '/'
 
568
 
543
569
def printInfo():
544
570
    print "Grub Python Client version: %s" % VERSION
545
571
    print "Username: %s" % USERNAME
549
575
    print "Python version: %s" % sys.version
550
576
 
551
577
def main():
552
 
    global USERNAME, PASSWORD, VERBOSE, NUM_CLIENTS, DEBUG
 
578
    global USERNAME, PASSWORD, VERBOSE, NUM_CLIENTS, DEBUG, LOG, DATAPRFX
553
579
 
554
580
    try:
555
 
        opts, args = getopt.getopt(sys.argv[1:], 'ihvqt:u:p:', ["username=", "password=", "threads=", "version", "info", "debug", "help"])
 
581
        opts, args = getopt.getopt(sys.argv[1:], 'ihvqt:u:p:', ["username=", "password=", "threads=", "version", "info", "debug", "log", "help"])
556
582
    except getopt.GetoptError, err:
557
583
        print str(err)
558
584
        sys.exit(2)
580
606
            printInfo()
581
607
            sys.exit(0)
582
608
        elif o in ("--debug"):
583
 
            DEBUG = True
 
609
            DEBUG = True      
 
610
        elif o in ("--log"):
 
611
            LOG = True      
584
612
        elif o in ("--version"):
585
613
            print "Grub Python Client, version %s" % VERSION
586
614
            sys.exit(0)
587
615
        else:
588
616
            assert False, "unhandled option"
589
617
 
 
618
 
 
619
    if not DATAPRFX:
 
620
        DATAPRFX = "data/"
 
621
    else:
 
622
        DATAPRFX = addslash(os.path.normpath(DATAPRFX))        
 
623
 
 
624
    try:
 
625
        if not os.path.exists("data"):
 
626
            os.mkdir("data")            
 
627
    except OSError:
 
628
        printer("Creating data directory failed!", error=1)
 
629
        sys.exit(13)                   
 
630
 
 
631
 
590
632
    parseConfig()
 
633
    setupLogging()
591
634
 
592
635
    for i in range(NUM_CLIENTS): 
593
636
        w = Crawler(i)