~semi-hallikas/sspamm/3.0-devel

« back to all changes in this revision

Viewing changes to sspamm.py

  • Committer: Sami-Pekka Hallikas
  • Date: 2010-10-11 08:44:18 UTC
  • Revision ID: semi@hallikas.com-20101011084418-o41gdwfhmbc3p1om
This version has quite major updates. It has all required functions (oneliner, basic fileoperations, is_listed, etc.).
Configuration stuff is added.
Also first Thread has been added (configuration reload)

This is first version that one can relly use for testing. All 'tests' are disabled for now, they would be added to next commit.

Show diffs side-by-side

added added

removed removed

Lines of Context:
13
13
import os
14
14
import time
15
15
import locale
16
 
import signal
 
16
import ConfigParser
17
17
import re
 
18
import thread
 
19
from email import message_from_file
 
20
from email.Header import decode_header
 
21
from string import maketrans, letters, digits, punctuation, whitespace
 
22
from signal import signal, SIGINT, SIGHUP, SIGBUS, SIGTERM
18
23
 
19
24
# Debug
20
 
import syslog
 
25
from traceback import print_exc
21
26
from syslog import \
22
27
        LOG_ERR, LOG_WARNING, LOG_NOTICE, LOG_INFO, \
23
28
        LOG_DEBUG, LOG_MAIL, LOG_PID, LOG_CONS
34
39
 
35
40
 
36
41
 
 
42
##############################################################################
37
43
##
38
 
## Global variables
 
44
## Global variables / Configuration
39
45
##
40
46
conffile = "sspamm.conf"
41
47
 
42
 
conf = {}
 
48
confdefaults = {
 
49
        "main": {
 
50
                "name":                 "sspamm",
 
51
                "port":                 "local:/tmp/sspamm.sock",
 
52
                "sspammdir":            None,
 
53
                "pid":                  "sspamm.pid",
 
54
                "logfile":              "sspamm.log",
 
55
                "verbose":              0,
 
56
                "timeme":               True,
 
57
                "tmpdir":               "/dev/shm",
 
58
                "savedir":              None,
 
59
                "watchmode":            True,
 
60
#               "detailed":             False,
 
61
        },
 
62
#       "stats": {
 
63
#               "driver":               None,
 
64
#               "user":                 "sspamm",
 
65
#               "pass":                 "sspamm",
 
66
#               "host":                 "localhost",
 
67
#               "database":             "sspamm2",
 
68
#               "table":                "stats",
 
69
#       },
 
70
        "filter": {
 
71
                "tests":                ["connect", "helo", "ipfromto", "charset", "headers", "wordscan", "rbl", "dyndns", "bayesian"],
 
72
                "domains":              [],
 
73
                "rules":                [],
 
74
        },
 
75
        "connect": {
 
76
                "action":               "Flag",
 
77
                "ignore_ip":            ["127.0.0.1"],
 
78
#               "ignore_dns":           [],
 
79
#               "allow_ip":             [],
 
80
#               "allow_dns":            [],
 
81
#               "block_ip":             [],
 
82
#               "block_dns":            []
 
83
        },
 
84
        "helo": {
 
85
                "action":               "Flag",
 
86
#               "recursive":            False,
 
87
#               "rules":                [],
 
88
        },
 
89
        "samefromto": {
 
90
                "action":               "Flag",
 
91
        },
 
92
        "accept": {
 
93
                "rules":                [],
 
94
        },
 
95
        "block": {
 
96
                "rules":                [],
 
97
        },
 
98
#       "ipfromto": {
 
99
#               "action":               "Flag",
 
100
#               "maxrcpt":              2,
 
101
#               "rules":                [],
 
102
#       },
 
103
#       "rbl": {
 
104
#               "action":               "Flag",
 
105
#               "recursive":            False,
 
106
#               "ipservers":            ["sbl-xbl.spamhaus.org", "dnsbl.sorbs.net", "bl.spamcop.net"],
 
107
#               "skip_ip":              [],
 
108
#               "skip_dns":             [],
 
109
#       },
 
110
#       "dyndns": {
 
111
#               "action":               "Flag",
 
112
#               "recursive":            False,
 
113
#               "ignore_authmx":        [],
 
114
#               "known_authmx":         [],
 
115
#               "skip_dyndns":          [],
 
116
#               "rules":                [
 
117
#                                       "^[0-9]{1,3}[\-\.][0-9]{1,3}[\-\.][0-9]{1-3}\..*",
 
118
#               ],
 
119
#       },
 
120
#       "bayesian": {
 
121
#               "action":               "Flag",
 
122
#               "unsure":               "Flag",
 
123
#               "spampros":             90,
 
124
#               "hampros":              20,
 
125
#               "maxbodysize":          200,
 
126
#               "commondb":             "sspamm.db",
 
127
#               "mimeparts":            False,
 
128
#               "usedomaindb":          False,
 
129
#               "dbtrain":              False,
 
130
#               "savembox":             False,
 
131
#               "saveunsure":           True,
 
132
#               "maxtrainsize":         512,
 
133
#               "msgsneeded":           10,
 
134
#               "maxtrain":             10000,
 
135
#               "ratio":                "50:50",
 
136
#       },
 
137
#       "headers": {
 
138
#               "action":               "Flag",
 
139
#               "rules":                [],
 
140
#       },
 
141
#       "charset": {
 
142
#               "action":               "Flag",
 
143
#               "rules":                [],
 
144
#       },
 
145
#       "wordscan": {
 
146
#               "action":               "Flag",
 
147
#               "maxbodysize":          200,
 
148
#               "links":                [],
 
149
#               "blockhtml":            [],
 
150
#               "subject":              [],
 
151
#               "blockwords":           [],
 
152
#       },
 
153
}
 
154
conf = confdefaults.copy()
 
155
 
43
156
conf["runtime"] = {
44
 
        "starttime": 0,
45
 
        "endtime": 0,
46
 
#       "conftime": 0,
47
 
#       "logtime": 0,
48
 
        "offline": False,
49
 
#       "syslog": False,
50
 
}
51
 
 
52
 
conf["main"] = {
53
 
        "timeme": True,
54
 
}
55
 
 
 
157
        "starttime":    0,
 
158
        "endtime":      0,
 
159
        "conftime":     0,
 
160
        "logtime":      0,
 
161
        "conffile":     None,
 
162
        "offline":      False,
 
163
}
 
164
 
 
165
##############################################################################
 
166
##
 
167
## Basic fileoperations (Usualy we don't need to care if it success or not)
 
168
##
 
169
def rm(file, id=None):
 
170
        debug("rm(\"%s\")" % (file), LOG_DEBUG, id=id)
 
171
        try:
 
172
                if os.path.exists(file): os.remove(file)
 
173
        except:
 
174
                pass
 
175
        return
 
176
 
 
177
def rmdir(path, id=None):
 
178
        debug("rmdir(\"%s\")" % (path), LOG_DEBUG, id=id)
 
179
        try:
 
180
                os.rmdir(path)
 
181
        except OSError, (errno, strerror):
 
182
                if errno != 39: debug("%s" % sys.exc_value, LOG_ERR)
 
183
        except:
 
184
                debug("%s: %s" % (sys.exc_type, sys.exc_value), LOG_ERR)
 
185
        return
 
186
 
 
187
def mkdir(path, id=None):
 
188
        debug("mkdir(\"%s\")" % (path), LOG_DEBUG, id=id)
 
189
        try:
 
190
                os.mkdir(path)
 
191
        except OSError, (errno, strerror):
 
192
                if errno != 17: debug("%s" % sys.exc_value, LOG_ERR)
 
193
        except:
 
194
                debug("%s: %s" % (sys.exc_type, sys.exc_value), LOG_ERR)
 
195
        return
 
196
 
 
197
def mv(what, where, id=None):
 
198
        debug("mv(\"%s\" \"%s\")" % (what, where), LOG_DEBUG, id=id)
 
199
        try:
 
200
                os.rename(what, where)
 
201
        except:
 
202
                try:
 
203
                        fpin = open(what,"r")
 
204
                        fpout = open(where,"w+b")
 
205
                        while 1:
 
206
                                buf = fpin.read(1024*16)
 
207
                                if len(buf) == 0: break
 
208
                                fpout.write(buf)
 
209
                        fpin.close()
 
210
                        fpout.close()
 
211
                        rm(what)
 
212
                except:
 
213
                        debug("move failed: %s -> %s" % (what, where), LOG_DEBUG)
 
214
        return
 
215
 
 
216
##############################################################################
 
217
##
 
218
## Debug
 
219
##
56
220
def debug(args, level=LOG_DEBUG, id=None, trace=None):
57
221
        datetime = ""
58
222
        msg = ""
65
229
        msg = datetime + msg
66
230
        print msg
67
231
        if trace:
68
 
                traceback.print_exc(limit=None, file=sys.stderr)
 
232
                print_exc(limit=None, file=sys.stderr)
69
233
        sys.stdout.flush()
70
234
        return
71
235
 
 
236
def save_vars(var, fname, id=None):
 
237
        if conf["runtime"]["offline"]: return
 
238
        debug("save_vars(\"%s\")" % (fname), LOG_DEBUG, id=id)
 
239
 
 
240
### DEVELOPER DEBUG
 
241
        if var.has_key("raw"): del var["raw"]
 
242
        if var.has_key("mime"): del var["mime"]
 
243
 
 
244
        fp = open(fname, "w+b")
 
245
        if fp: fp.write(show_vars(var))
 
246
        fp.close()
 
247
        return
 
248
 
 
249
def load_vars(fname, id=None):
 
250
        debug("load_vars(\"%s\")" % (fname), LOG_DEBUG, id=id)
 
251
        fp = open(fname, "r")
 
252
        is_raw = False
 
253
        do_skip = False
 
254
        buf = ""
 
255
        while 1:
 
256
                line = fp.readline()
 
257
                if len(line) < 1: break
 
258
                if line[1:10] == "\"mime\": {":
 
259
                        do_skip = True
 
260
                if line[1:13] == "\"raw\": 'From":
 
261
                        raw = line[9:]
 
262
                        is_raw = True
 
263
                elif is_raw:
 
264
                        if line == "',\n":
 
265
                                is_raw = False
 
266
                        else:
 
267
                                raw += line
 
268
                elif do_skip:
 
269
                        if line[1:3] == "},":
 
270
                                do_skip = False
 
271
                        pass
 
272
                else:
 
273
                        buf += line
 
274
        fp.close()
 
275
        vars = eval(buf)
 
276
        vars["raw"] = raw
 
277
        return vars
 
278
 
72
279
def show_vars(var, lvl=0):
73
280
        if lvl==0: lvl=1
74
281
        st=""
126
333
                st += str(var)
127
334
        return st
128
335
 
 
336
# Timeme:
 
337
# Used for trace used time of processing, so one can find how long does
 
338
# each part of processing take.
 
339
#
129
340
def timeme(timer=0, noshow=None, id=None, title="Timer"):
130
341
        if timer == 0: return time.time()
131
342
        timer = float(time.time())-float(timer)
132
343
        if timer < 0: timer = 0
133
 
        if not noshow: debug("\t%s: %.4f" % (title, timer), LOG_NOTICE, id=id)
 
344
#       if not noshow: debug("\t%s: %.4f" % (title, timer), LOG_DEBUG, id=id)
134
345
        return float(timer)
135
346
 
136
347
##
 
348
## Oneliner
 
349
##
 
350
def oneliner(value, id=None):
 
351
#       debug("*oneliner(\"%s%s\")" % (value[0:160]), LOG_DEBUG, id=id)
 
352
        return re.sub(" + ", " ", re.sub("[\r\n\t]", " ", value))
 
353
 
 
354
##
 
355
## Strip unprintable
 
356
##
 
357
def stripUnprintable(input_string, id=None):
 
358
#       debug("stripUnprintable()", LOG_DEBUG, id=id)
 
359
        try: filterUnprintable = stripUnprintable.filter
 
360
        except AttributeError: # only the first time it is called
 
361
                allchars = maketrans('','')
 
362
                delchars = allchars.translate(allchars, letters+digits+punctuation+whitespace)
 
363
                filterUnprintable = stripUnprintable.filter = lambda input: input.translate(allchars, delchars)
 
364
        return filterUnprintable(input_string)
 
365
 
 
366
##
 
367
## Unique keys
 
368
##
 
369
def uniq(seq, idfun=None): 
 
370
        # order preserving
 
371
        if idfun is None:
 
372
                def idfun(x): return x
 
373
        seen = {}
 
374
        result = []
 
375
        for item in seq:
 
376
                if item == None: continue
 
377
                marker = idfun(item)
 
378
                # in old Python versions:
 
379
                # if seen.has_key(marker)
 
380
                # but in new ones:
 
381
                if marker in seen: continue
 
382
                seen[marker] = 1
 
383
                result.append(item)
 
384
        return result
 
385
 
 
386
##
 
387
## Mime Part
 
388
##
 
389
def mimepart(part, lvl=0, param=None, id=None):
 
390
        if lvl == 0:
 
391
                debug("*mimepart()", LOG_DEBUG, id=id)
 
392
                if conf["main"]["timeme"] is True: timer = timeme()
 
393
        if not part.is_multipart() and part.get_content_type() != None:
 
394
                text = None
 
395
                if part.get_content_maintype() == "text":
 
396
                        if part.get_content_subtype() == "plain" or part.get_content_subtype() == "html":
 
397
                                text = part.get_payload(decode=True)
 
398
                        else:
 
399
                                if part.get_content_subtype() == "rfc822-headers":
 
400
                                        return
 
401
                                debug("Unknown content type text/%s" % part.get_content_subtype(), LOG_NOTICE, id=id)
 
402
                                return
 
403
                                #text = part.get_payload(decode=True)
 
404
                if lvl>0:
 
405
                        if text:
 
406
                                return (part.get_content_maintype(), part.get_content_subtype(), "", param, text.strip())
 
407
                        else:
 
408
                                return (part.get_content_maintype(), part.get_content_subtype(), "", param, "")
 
409
                else:
 
410
                        if text:
 
411
                                return {0: (part.get_content_maintype(), part.get_content_subtype(), "", param, text.strip())}
 
412
                        else:
 
413
                                return {0: (part.get_content_maintype(), part.get_content_subtype(), "", param, "")}
 
414
                # Nothing to return
 
415
                return
 
416
        tree = {}
 
417
        sublvl=0
 
418
        tab="  "
 
419
        lvl+=1
 
420
 
 
421
        debug("\t%d.%d%s%s" % (lvl,sublvl,tab*lvl,part.get_content_type()), LOG_DEBUG, id=id)
 
422
        #tree[lvl*10+sublvl] = (part.get_content_type())
 
423
        if not part.get_payload():
 
424
                return tree
 
425
        for p in part.get_payload():
 
426
                ret = None
 
427
                if type(p) is str:
 
428
                        if lvl > 1:
 
429
                                return (part.get_content_maintype(), part.get_content_subtype(), "", param, part.get_payload().strip())
 
430
                        return {10: (part.get_content_maintype(), part.get_content_subtype(), "", param, part.get_payload().strip())}
 
431
                if p.is_multipart():
 
432
                        if p.get_param("x-spam-type") == "original":
 
433
                                param="spam"
 
434
                                tree = {}
 
435
                        ret=mimepart(p, lvl, param, id=id)
 
436
                        if ret:
 
437
                                for r in ret:
 
438
                                        tree[r] = ret[r]
 
439
                                continue
 
440
                else:
 
441
                        sublvl += 1
 
442
                        if p.get_param("x-spam-type") == "original":
 
443
                                debug("\t%d.%d%s%s ( = original spam)" % (lvl,sublvl,tab*lvl,p.get_content_type()), LOG_DEBUG, id=id)
 
444
                                msg=email.message_from_string(p.get_payload(decode=True))
 
445
                                ret=mimepart(msg, lvl, param="spam", id=id)
 
446
                        else:
 
447
                                ret = mimepart(p, lvl, param, id=id)
 
448
                if ret:
 
449
                        debug("\t%d.%d%s%s" % (lvl,sublvl,tab*lvl,p.get_content_type()), LOG_DEBUG, id=id)
 
450
                        
 
451
                        if p.get_param("x-spam-type"):
 
452
                                tree = {}
 
453
                        if type(ret) in [tuple]:
 
454
                                tree[lvl*10+sublvl] = ret
 
455
                        else:
 
456
                                return ret
 
457
        if lvl == 0 and conf["main"]["timeme"] is True: mail["timer"]["mimepart"] = str("%.4f") % timeme(timer)
 
458
        return tree
 
459
 
 
460
 
 
461
##
 
462
## dumbregtest
 
463
##
 
464
# Test regexp rules for stupid mistakes like || which would match EVERYTHING.
 
465
# If this test matches, rule should not be used!
 
466
def dumbregtest(regrule):
 
467
        try:
 
468
                test = re.search(regrule, "The quick brown fox jumps over the lazy dog.\n\t1234567890@${[]}!#&/()=*+-_,;:", re.IGNORECASE+re.MULTILINE)
 
469
                if test:
 
470
                        debug("regexp error: Matched too easily: %s" % (regrule), LOG_ERR)
 
471
                        return False
 
472
        except:
 
473
                return False
 
474
        return regrule
 
475
 
 
476
##
 
477
## Is Listed
 
478
##
 
479
# We could use already defined functions, like domainmatch(ptrs, domainsuffix)
 
480
# or cidrmatch(i, ipaddrs, cidr_length=32) ... good ideas, but does not do RegExp ;)
 
481
def is_listed(where,what,flags=re.IGNORECASE+re.MULTILINE,id=None,noshow=None):
 
482
        noshow=False
 
483
        if what == None or where == None or len(what) < 1 or len(where) < 1: return
 
484
        if type(where) is str: where = [where]
 
485
        if type(what) is str: what = [what]
 
486
 
 
487
        if type(what) is dict:
 
488
                for a in what:
 
489
                        tmp = is_listed(where,a,flags=flags,id=id,noshow=noshow)
 
490
                        if tmp: return what[a]
 
491
        elif type(what[0]) is tuple:
 
492
                for a in what:
 
493
                        tmp = is_listed(where,a[0],flags=flags,id=id,noshow=noshow)
 
494
                        if tmp: return a[1]
 
495
        else:
 
496
                try:
 
497
#                       debug("*is_listed(where=\"%s\")" % (where), LOG_DEBUG, id=id)
 
498
                        for haystack in where:
 
499
                                for trap in what:
 
500
                                        try:
 
501
                                                if id: globaltmp[id] += 1
 
502
                                        except:
 
503
                                                pass
 
504
#                                       debug("*is_listed(what=\"%s\")" % (trap), LOG_DEBUG, id=id)
 
505
                                        tmp = re.search(trap, haystack, flags)
 
506
                                        if tmp:
 
507
#                                               debug("\t*is_listed matched:", LOG_DEBUG, id=id)
 
508
                                                return (trap, tmp.end()-tmp.start(), haystack[tmp.start():tmp.end()])
 
509
                except:
 
510
                        debug("%s: %s" % (sys.exc_type, sys.exc_value), LOG_DEBUG)
 
511
                        debug("FAILED: is_listed(%s, %s)" % (haystack, trap), LOG_DEBUG, id=id)
 
512
        return
 
513
 
 
514
##
 
515
## is filtered
 
516
##
 
517
def is_filtered(mail):
 
518
        if conf["main"]["timeme"] is True: timer = timeme()
 
519
        debug("is_filtered(%s)" % (mail["to"][0]), LOG_DEBUG, id=mail["id"])
 
520
        found = False
 
521
        try:
 
522
                mail["todomain"] = mail["to"][0].split("@")[1]
 
523
                found = is_listed(mail["todomain"], conf["filter"]["domains"], id=mail["id"])
 
524
                if found: debug("\tFound: %s" % (found), LOG_DEBUG, id=mail["id"])
 
525
        except:
 
526
                debug("FAILED: is_filtered %s: %s" % (sys.exc_type, sys.exc_value), LOG_ERR, id=mail["id"], trace=True)
 
527
                mail["failed"] = "is_filtered(\"%s\"[0].split(\"@\")[1])" % (mail["to"])
 
528
                save_vars(self.mail, "%s/%08d.var" % ("/tmp", mail["id"]), id=mail["id"]);
 
529
        if conf["main"]["timeme"] is True: mail["timer"]["is_filtered"] = str("%.4f") % timeme(timer, id=mail["id"])
 
530
        return (found, mail)
 
531
 
 
532
 
 
533
##
 
534
## Makepid
 
535
##
 
536
def makepid(fname):
 
537
        debug("makepid(\"%s\")" % (fname), LOG_DEBUG)
 
538
 
 
539
        if os.path.exists(fname):
 
540
                fp = open(fname, "r")
 
541
                pid=int(fp.readline().strip())
 
542
                fp.close
 
543
 
 
544
                if os.path.exists("/proc/%d" % pid):
 
545
                        fp = open("/proc/%d/stat" % pid, "r")
 
546
                        pidstat = fp.readline().strip().split(" ")
 
547
                        fp.close
 
548
                        if pidstat[2] == "S": pidstat[2] = "Sleeping"
 
549
                        elif pidstat[2] == "R": pidstat[2] = "Running"
 
550
                        elif pidstat[2] == "T": pidstat[2] = "Stopped"
 
551
                        debug("PID file %s found for process %s (%s)." % (conf["main"]["pid"], pidstat[1][1:-1], pidstat[2]), LOG_ERR)
 
552
                        return False
 
553
                else:
 
554
                        rm(fname)
 
555
        try:
 
556
                fp = open(fname, "w+b")
 
557
                fp.write("%s\n" % os.getpid())
 
558
                fp.close()
 
559
        except:
 
560
                return ("Error", sys.exc_value)
 
561
                debug("Couldn't create %s: %s" % (conf["main"]["pid"], tmp[1]), LOG_ERR)
 
562
                return False
 
563
        return True
 
564
 
 
565
##
 
566
## parse_addrs - Get only address part of address string. Do lowercase and
 
567
## strip blanks. Also return all parsed addresses as array.
 
568
##
 
569
def parse_addrs(addr, id=None):
 
570
#       debug("*parse_addrs(\"%s\")" % (addr), LOG_DEBUG, id=id)
 
571
        addr=re.sub(" ", "", re.sub(' ?[("].*?[)"]', "", addr.lower().strip()))
 
572
        if addr.startswith("<") or addr.endswith(">"):
 
573
                return [addr[addr.find("<")+1:addr.rfind(">")]]
 
574
        elif addr.find(","):
 
575
                return addr.split(",")
 
576
        else:
 
577
                return [addr]
 
578
 
 
579
##############################################################################
 
580
##
 
581
## Configuration
 
582
##
 
583
class MyParser(ConfigParser.ConfigParser):
 
584
        def getvalue(self,section,option,default=None,warn=False):
 
585
                value = default
 
586
                try:
 
587
                        value = self.getboolean(section, option)
 
588
                except ConfigParser.NoOptionError, (err):
 
589
                        if warn:
 
590
                                print err
 
591
                        pass
 
592
                except:
 
593
                        try:
 
594
                                value = self.getint(section, option)
 
595
                        except:
 
596
                                err = "Warning: No value for %s: %s" % (section, option)
 
597
                                if warn:
 
598
                                        print err
 
599
                                pass
 
600
                        pass
 
601
                return value
 
602
 
 
603
        def getlines(self,section,option,default=None,warn=False):
 
604
                try:
 
605
                        value = self.get(section, option)
 
606
                        t = value.split('\n')
 
607
                        try:
 
608
                                if t[0] == "": t.remove('')
 
609
                        except IndexError:
 
610
                                t = []
 
611
                        return t
 
612
                except ConfigParser.NoOptionError, (err):
 
613
                        if warn:
 
614
                                print err
 
615
                        pass
 
616
                except:
 
617
                        err = "Warning: No lines for %s: %s" % (section, option)
 
618
                        if warn:
 
619
                                print err
 
620
                        return None
 
621
                return default
 
622
 
 
623
        def getlist(self,section,option,default=None,warn=False):
 
624
                try:
 
625
                        value = self.get(section, option)
 
626
                        (value, count) = re.compile(",|;|\n").subn(' ', value)
 
627
                        t = value.split()
 
628
                        try:
 
629
                                if t[0] == "": t.remove('')
 
630
                        except IndexError:
 
631
                                t = []
 
632
                        return t
 
633
                except ConfigParser.NoOptionError, (err):
 
634
                        if warn:
 
635
                                print err
 
636
                        pass
 
637
                except:
 
638
                        err = "Warning: No list for %s: %s" % (section, option)
 
639
                        if warn:
 
640
                                print err
 
641
                        return None
 
642
                return default
 
643
 
 
644
 
 
645
def config_read(cfgfile = conffile):
 
646
        tmpconf = conf.copy()
 
647
        cp = MyParser()
 
648
        if not os.access(cfgfile, os.R_OK):
 
649
                print "FATAL: Can't access %s." % (cfgfile)
 
650
                return
 
651
        cp.read(cfgfile)
 
652
 
 
653
        if not (cp.has_section("main")):
 
654
                print "FATAL: Main section is missing!"
 
655
 
 
656
        conf["main"]["verbose"] = cp.getvalue("main", "verbose")
 
657
 
 
658
        for s in confdefaults.keys():
 
659
                if cp.has_section(s):
 
660
                        for k in confdefaults[s].keys():
 
661
                                try:
 
662
                                        if "%s/%s" % (s,k) in ["filter/domains"]:
 
663
                                                t = []
 
664
 
 
665
                                                for i in cp.getlines(s, k, []):
 
666
                                                        i=re.sub("\t| ", "", i).split(":")
 
667
                                                        if '' in i: i.remove('')
 
668
                                                        if len(i) == 1: i.append("all")
 
669
                                                        t.append((i[0].split(","), i[1].split(",")))
 
670
                                                tmpconf[s][k] = t
 
671
 
 
672
                                        elif "%s/%s" % (s,k) in ["filter/rules"]:
 
673
                                                t = []
 
674
 
 
675
                                                for i in cp.getlines(s, k, []):
 
676
                                                        i=re.sub("\t| ", "", i).split(":")
 
677
                                                        if '' in i: i.remove('')
 
678
                                                        if len(i) > 1:
 
679
                                                                t.append((i[0].split(","), ":".join(i[1:]).split(",")))
 
680
                                                tmpconf[s][k] = t
 
681
 
 
682
                                        # Parameters that can be bolean value
 
683
                                        elif k in ["enabled", "verbose", "detailed", "watchmode", "timeme", "recursive", "usedomaindb", "dbtrain", "savembox", "saveunsure", "mimeparts", "learning"]:
 
684
                                                tmpconf[s][k] = cp.getvalue(s, k)
 
685
                                        # RegExp lines
 
686
                                        elif k in ["links", "blockhtml", "blockwords", "subject"]:
 
687
                                                testconf = []
 
688
                                                tmpconf[s][k] = cp.getlines(s, k, tmpconf[s][k])
 
689
                                                for t in tmpconf[s][k]:
 
690
                                                        if dumbregtest(t): testconf.append(t)
 
691
                                                tmpconf[s][k] = testconf
 
692
                                        elif "%s/%s" % (s,k) in [ "headers/rules", "helo/rules" ]:
 
693
                                                testconf = []
 
694
                                                tmpconf[s][k] = cp.getlines(s, k, tmpconf[s][k])
 
695
                                                for t in tmpconf[s][k]:
 
696
                                                        if dumbregtest(t): testconf.append(t)
 
697
                                                tmpconf[s][k] = testconf
 
698
                                        elif type(tmpconf[s][k]) is int:
 
699
                                                try:
 
700
                                                        tmpconf[s][k] = cp.getint(s, k)
 
701
                                                except ValueError:
 
702
                                                        debug("%s/%s should be numerical! Using default value %s" % (s, k, tmpconf[s][k]), LOG_DEBUG)
 
703
                                        elif type(tmpconf[s][k]) is list:
 
704
                                                tmpconf[s][k] = cp.getlist(s, k, tmpconf[s][k])
 
705
                                        else:
 
706
                                                tmpconf[s][k] = cp.get(s, k)
 
707
                                except ConfigParser.NoOptionError, (err):
 
708
                                        if "%s/%s" % (s,k) in [ "main/pid" ]:
 
709
                                                tmpconf[s][k] = confdefaults[s][k]
 
710
                                        elif "%s/%s" % (s,k) in [ "main/sspammdir", "main/timeme", "stats/driver", "main/logfile", "main/savedir", "bayesian/commondb", "wordscan/blockhtml" ]:
 
711
                                                tmpconf[s][k] = None
 
712
                                        else:
 
713
                                                debug("%s" % err, LOG_DEBUG)
 
714
                                if tmpconf[s][k] == "None":
 
715
                                        tmpconf[s][k] = None
 
716
                                else:
 
717
                                        # Write 'our special regexp' (_ip and _dns) to real regexp
 
718
                                        # Notice, even SPACE is rule separator!
 
719
                                        if "%s/%s" % (s,k) in [ "accept/rules", "block/rules", "ipfromto/rules" ]:
 
720
                                                testconf = []
 
721
                                                for t in tmpconf[s][k]:
 
722
                                                        if dumbregtest(t): testconf.append(t)
 
723
                                                tmpconf[s][k] = testconf
 
724
                                        elif "%s/%s" % (s,k) in [ "connect/allow_ip", "connect/allow_dns", "connect/block_ip", "connect/block_dns", "rbl/skip_ip", "rbl/skip_dns", "dyndns/skip_ip", "dyndns/skip_dns" ]:
 
725
                                                testconf = []
 
726
                                                for t in tmpconf[s][k]:
 
727
                                                        if "%s/%s" % (s,k) in [] or ("%s/%s" % (s,k))[-3:] == "_ip":
 
728
                                                                t = re.sub("^", "^", re.sub("\?", ".", re.sub("\.", "\.", t)))
 
729
                                                        elif "%s/%s" % (s,k) in ["dyndns/rules"] or ("%s/%s" % (s,k))[-4:] == "_dns":
 
730
                                                                t = re.sub("$", "$", re.sub("\*", ".*", re.sub("\?", ".", re.sub("\.", "\.", t))))
 
731
                                                        if dumbregtest(t): testconf.append(t)
 
732
                                                tmpconf[s][k] = testconf
 
733
                else:
 
734
                        if s != "runtime":
 
735
                                debug("Section %s not found" % s, LOG_DEBUG)
 
736
                                del tmpconf[s]
 
737
 
 
738
# Now config is fully loaded, do some special parsing.
 
739
        tmp = []
 
740
        for val in tmpconf["filter"]["domains"]:
 
741
                retests = val[1]
 
742
                if 'all' in val[1]:
 
743
                        retests += tmpconf["filter"]["tests"]
 
744
                        retests.remove('all')
 
745
                else:
 
746
                        retests = val[1]
 
747
                if '' in retests: retests.remove('')
 
748
 
 
749
                tests = []
 
750
                rules = 0
 
751
                action = {}
 
752
                for test in retests:
 
753
                        if test in tests: continue
 
754
                        tests.append(test)
 
755
                        if test[0] >= 'a' and test[0] <= 'z':
 
756
                                rules += 1
 
757
 
 
758
                if rules == 0:
 
759
                        tests += tmpconf["filter"]["tests"]
 
760
                if 'all' in tests: tests.remove('all')
 
761
                t = list(tests)
 
762
                for test in tests:
 
763
                        if test[0] == "-" or test[0] == "!":
 
764
                                if test[1:] in t: t.remove(test[1:])
 
765
                                t.remove(test)
 
766
                        if test[0] == "+":
 
767
                                if test[1:] in t: t.remove(test[1:])
 
768
                                t.remove(test)
 
769
                                t.append(test[1:])
 
770
                        
 
771
                tmp.append((val[0], t))
 
772
        tmpconf["filter"]["domains"] = tmp
 
773
 
 
774
        tmp = {}
 
775
        for val in tmpconf["filter"]["rules"]:
 
776
                for domain in val[0]:
 
777
                        if not tmp.has_key(domain): tmp[domain] = {}
 
778
                        for rule in val[1]:
 
779
                                r = rule.split('=')
 
780
                                if len(r) > 2:
 
781
                                        test=r[0].lower()
 
782
                                        param="=".join(r[1:])
 
783
                                elif len(r) == 2:
 
784
                                        test=r[0].lower()
 
785
                                        param=r[1]
 
786
                                else:
 
787
                                        test="rules"
 
788
                                        param=r[0]
 
789
                                if test == "rules":
 
790
                                        if not tmp[domain].has_key(test): tmp[domain][test] = []
 
791
                                        tmp[domain][test].append(param)
 
792
                                else:
 
793
                                        tmp[domain][test] = param
 
794
        tmpconf["filter"]["rules"] = tmp
 
795
 
 
796
        return tmpconf
 
797
 
 
798
def config_load(file):
 
799
        global conf
 
800
 
 
801
        try:
 
802
                conf = config_read(file);
 
803
        except:
 
804
                debug("CONFIG LOAD ERROR. %s: %s" % (sys.exc_type, sys.exc_value), LOG_ERR)
 
805
                print_exc(limit=None, file=sys.stderr)
 
806
 
 
807
def config_save():
 
808
        global conf, conffile
 
809
 
 
810
        try:
 
811
                fp = open("%s.bak" % (conffile), "w+b")
 
812
                tmpconf = conf.copy()
 
813
                del tmpconf["runtime"]
 
814
                fp.write(show_vars(tmpconf))
 
815
                fp.close()
 
816
        except:
 
817
                pass
 
818
 
 
819
##############################################################################
 
820
##
137
821
## SpamMilter Class 
138
822
##
139
823
class SpamMilter(Milter.Milter):
157
841
                        "todomain": "",
158
842
                        "size": 0,
159
843
                        "subject": "",
160
 
#                       "charset": "",
161
 
#                       "header": { },
162
 
#                       "tests": [],
163
 
#                       "type": "",
 
844
                        "charset": "",
 
845
                        "header": { },
 
846
                        "tests": [],
 
847
                        "type": "",
164
848
                }
165
849
 
166
850
        def __init__(self):
167
851
                self.id = Milter.uniqueID()
168
 
                self.log("SpamMilter.__init__()")
 
852
                debug("SpamMilter.__init__()", LOG_DEBUG, id=self.id)
169
853
                self.mail = self.maildef()
170
854
                self.mail["id"] = self.id
 
855
                self.tmpname = "%08d.tmp" % (self.id)
 
856
#               self.dns = spf.query("", "", "")
 
857
                try:
 
858
                ## Temp file in DISK
 
859
                        if not os.path.exists(conf["main"]["tmpdir"]): mkdir(conf["main"]["tmpdir"])
 
860
                        self.tmp = open("%s/%s" % (conf["main"]["tmpdir"], self.tmpname),"w+b")
 
861
                except IOError, (errno, strerror):
 
862
                        debug("Temp file failure (%s: %s)" % (errno, strerror), LOG_DEBUG, id=self.id)
 
863
                except:
 
864
                        debug("Temp file (%s) failure" % "%s/%s" % (conf["main"]["tmpdir"], self.tmpname), LOG_DEBUG, id=self.id)
 
865
                self.mail["tmpfile"]=self.tmpname
171
866
 
 
867
## This is runned for every SMTP Connection, and ONLY when Sendmail
 
868
## connects.
172
869
        def _cleanup(self):
173
 
                self.log("SpamMilter._cleanup()")
174
 
                if conf["main"]["timeme"] is True and self.mail["timer"].has_key("timepass"): self.mail["timer"]["timepass"] = str("%.4f") % timeme(self.mail["timer"]["timepass"], id=self.id, title="TTimer")
175
 
                print show_vars(self.mail)
 
870
                debug("SpamMilter._cleanup()", LOG_DEBUG, id=self.id)
 
871
                if self.tmp:
 
872
                        self.tmp.close()
 
873
                        rm(self.tmp.name, id=self.id)
 
874
                if self.mail:
 
875
                        if conf["main"]["timeme"] and self.mail["timer"].has_key("timepass"): self.mail["timer"]["timepass"] = str("%.4f") % timeme(self.mail["timer"]["timepass"], id=self.id, title="TTimer")
 
876
                        if not conf["runtime"]["offline"]:
 
877
                                if conf["main"]["savedir"]:
 
878
                                        save_vars(self.mail, "%s/%08d.var" % (conf["main"]["savedir"], self.id), id=self.id)
 
879
#                       else:
 
880
#                               if self.mail.has_key("raw"): del self.mail["raw"]
 
881
#                               if self.mail.has_key("mime"): del self.mail["mime"]
 
882
#                               if self.mail.has_key("header"): del self.mail["header"]
 
883
#                       print show_vars(self.mail)
176
884
                sys.stdout.flush()
177
885
                if not conf["runtime"]["offline"]: del self.mail
178
886
                return
179
887
 
180
888
        def log(self,*msg):
181
 
                debug(msg, LOG_INFO, id=self.id)
 
889
                debug(msg, LOG_DEBUG, id=self.id)
 
890
 
 
891
        def abort(self):
 
892
                debug("SpamMilter.abort()", LOG_DEBUG, id=self.id)
 
893
                self.mail = None
 
894
                self._cleanup()
 
895
                return CONTINUE
 
896
 
 
897
        def close(self):
 
898
                debug("SpamMilter.close()", LOG_DEBUG, id=self.id)
 
899
                if self.mail: self.mail["smtpcmds"].append("close")
 
900
                self._cleanup()
 
901
                return CONTINUE
182
902
 
183
903
        def connect(self,hostname,family,hostaddr):
184
 
                if conf["main"]["timeme"] is True: timer = timeme()
185
 
                self.log("SpamMilter.connect(%s, %s)" % (hostname,hostaddr))
 
904
                if conf["main"]["timeme"]: timer = timeme()
 
905
                debug("SpamMilter.connect(%s, %s)" % (hostname,hostaddr), LOG_DEBUG, id=self.id)
186
906
                if self.mail: self.mail["smtpcmds"].append("connect")
187
907
                self.mail["received"][1]["ip"] = hostaddr[0]
188
908
                self.mail["received"][1]["dns"] = hostname
189
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_connect"] = str("%.4f") % timeme(timer, id=self.mail["id"], noshow=True)
 
909
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_connect"] = str("%.4f") % timeme(timer, id=self.id, noshow=True)
190
910
                return CONTINUE
191
911
 
192
912
        def hello(self,hostname):
193
 
                if conf["main"]["timeme"] is True: timer = timeme()
194
 
                self.log("SpamMilter.hello(%s)" % (hostname))
 
913
                if conf["main"]["timeme"]: timer = timeme()
 
914
                debug("SpamMilter.hello(%s)" % (hostname), LOG_DEBUG, id=self.id)
195
915
                if self.mail: self.mail["smtpcmds"].append("hello")
196
916
                self.mail["received"][1]["helo"] = hostname
197
917
                self.reuse = self.mail["received"][1]
198
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_hello"] = str("%.4f") % timeme(timer, noshow=True)
 
918
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_hello"] = str("%.4f") % timeme(timer, id=self.id, noshow=True)
199
919
                return CONTINUE
200
920
 
201
921
        def envfrom(self,mailfrom,*vars):
202
 
                if conf["main"]["timeme"] is True: timer = timeme()
203
 
                self.log("SpamMilter.envfrom(\"%s\", %s)" % (mailfrom,vars))
204
922
                if not self.mail or "eom" in self.mail["smtpcmds"]:
205
923
                        self._cleanup()
206
 
                        debug("Connection reused" % (self.id), LOG_INFO, id=self.id)
 
924
                        debug("Connection reused", LOG_DEBUG, id=self.id)
207
925
                        self.__init__()
208
926
                        self.mail["received"][1] = self.reuse
209
927
                        self.mail["smtpcmds"].append("reused")
 
928
 
 
929
                if conf["main"]["timeme"]: timer = timeme()
 
930
                debug("SpamMilter.envfrom(\"%s\", %s)" % (mailfrom,vars), LOG_DEBUG, id=self.id)
 
931
                if conf["main"]["timeme"]: self.mail["timer"]["timepass"] = timeme()
210
932
                if self.mail: self.mail["smtpcmds"].append("envfrom")
 
933
 
 
934
                if mailfrom == "<>":
 
935
                        if self.mail["received"][1].has_key("dns"):
 
936
                                mailfrom = "<MAILER-DAEMON@%s>" % (self.mail["received"][1]["dns"])
 
937
                        else:
 
938
                                mailfrom = "<MAILER-DAEMON@[%s]>" % (self.mail["received"][1]["ip"])
 
939
                self.mail["from"] = parse_addrs(mailfrom, id=self.id)
 
940
 
211
941
                if not conf["runtime"]["offline"]:
212
942
                        self.mail["my"]["ip"] = self.getsymval('{if_addr}')
213
943
                        self.mail["my"]["dns"] = self.getsymval('{if_name}')
214
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_envfrom"] = str("%.4f") % timeme(timer, noshow=True)
 
944
                        if self.getsymval('{auth_type}'):
 
945
                                self.mail["smtp_auth"] = self.getsymval('{auth_authen}')
 
946
 
 
947
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_envfrom"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
 
948
 
215
949
                return CONTINUE
216
950
 
217
951
        def envrcpt(self,rcpt,*vars):
218
 
                if conf["main"]["timeme"] is True: timer = timeme()
219
 
                self.log("SpamMilter.envrcpt(\"%s\")" % (rcpt))
 
952
                if conf["main"]["timeme"]: timer = timeme()
 
953
                debug("SpamMilter.envrcpt(\"%s\")" % (rcpt), LOG_DEBUG, id=self.id)
220
954
                if self.mail: self.mail["smtpcmds"].append("envrcpt")
221
 
                if conf["main"]["timeme"] is True:
 
955
 
 
956
                if rcpt.startswith('<MAILER-DAEMON@'): debug("?To MAILER-DAEMON ?", LOG_DEBUG, id=self.id)
 
957
 
 
958
                if len(self.mail["to"]) > 0 and rcpt not in self.mail["to"]:
 
959
                        self.mail["to"].append(parse_addrs(rcpt, id=self.id)[0])
 
960
                else:
 
961
                        self.mail["to"] = parse_addrs(rcpt, id=self.id)
 
962
 
 
963
                if conf["main"]["timeme"]:
222
964
                        if self.mail["timer"].has_key("smtp_envrcpt"):
223
 
                                self.mail["timer"]["smtp_envrcpt"] = str("%.4f") % (float(self.mail["timer"]["smtp_envrcpt"]) + timeme(timer, noshow=True))
 
965
                                self.mail["timer"]["smtp_envrcpt"] = str("%.4f") % (float(self.mail["timer"]["smtp_envrcpt"]) + timeme(timer, id=self.id, noshow=True))
224
966
                        else:
225
 
                                self.mail["timer"]["smtp_envrcpt"] = str("%.4f") % timeme(timer, noshow=True)
 
967
                                self.mail["timer"]["smtp_envrcpt"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
226
968
                return CONTINUE
227
969
 
228
970
        def header(self,field,value):
229
 
                if conf["main"]["timeme"] is True: timer = timeme()
230
 
                self.log("SpamMilter.header(%s, %s)" % (field,value))
 
971
                if conf["main"]["timeme"]: timer = timeme()
231
972
                if self.mail and "header" not in self.mail["smtpcmds"]: self.mail["smtpcmds"].append("header")
232
 
                if conf["main"]["timeme"] is True:
 
973
 
 
974
                debug("SpamMilter.header(%s, %s)" % (field,value), LOG_DEBUG, id=self.id)
 
975
 
 
976
                if self.tmp and len(self.mail["header"]) == 0:
 
977
                        self.tmp.write("From %s %s\n" % (self.mail["from"][0], time.ctime()))
 
978
                        self.mail["size"] = 0
 
979
                        self.mail["subject"] = ""
 
980
                if self.tmp: self.tmp.write("%s: %s\n" % (field, value))
 
981
 
 
982
# Note, this is NOT endless loop, it is just used like switch-case
 
983
# statement. We break out when perferred line has been processed.
 
984
# Save header line as is, before prosessing
 
985
 
 
986
                while 1:
 
987
# Headers to drop ... just proof-of-concept
 
988
                        lfield = field.lower()
 
989
                        if lfield in [ "x-spambayes-classification", "x-spam-level" ]:
 
990
                                break
 
991
 
 
992
                        onelinerh = oneliner(value, id=self.id).strip()
 
993
                        if self.mail["header"].has_key(field) and lfield not in [ "subject" ]:
 
994
                                if type(self.mail["header"][field]) is not list:
 
995
                                        tmp = self.mail["header"][field]
 
996
                                        del self.mail["header"][field]
 
997
                                        self.mail["header"][field] = [ tmp ]
 
998
                                self.mail["header"][field].append(onelinerh)
 
999
                        else:
 
1000
                                self.mail["header"][field] = onelinerh
 
1001
 
 
1002
                        if lfield == "subject":
 
1003
                                self.mail["subject"] = self.mail["header"][field][:]
 
1004
                                break
 
1005
 
 
1006
                        if lfield == "received":
 
1007
                                a = re.compile("(?:from ((?P<ip3>[\d\.]+)|(?P<helo>\S+)) (?:\(helo (?P<helo2>[\w\d\.]+?)\) )?((?:\(?(?:\w+@)?(?:\S+(?: )?)?)?(?:(?:\[)(?P<ip>[\d.]+?)(?:\](?:[: ].+?)?))\)? )?|(\((?P<ip2>[\d\.]+?)\) )?)(?:\(using .*?\))?by (?P<by>.+?)(?: \(.+?\))? (with|id) ").match(onelinerh.lower())
 
1008
                                if a == None:
 
1009
                                        break
 
1010
                                reclen = len(self.mail["received"])+1
 
1011
                                self.mail["received"][reclen]=a.groupdict()
 
1012
                                if self.mail["received"][reclen]["ip"] == None and self.mail["received"][reclen]["ip3"] != None: self.mail["received"][reclen]["ip"] = self.mail["received"][reclen]["ip3"]
 
1013
                                del self.mail["received"][reclen]["ip3"]
 
1014
                                if self.mail["received"][reclen]["ip"] == None and self.mail["received"][reclen]["ip2"] != None: self.mail["received"][reclen]["ip"] = self.mail["received"][reclen]["ip2"]
 
1015
                                del self.mail["received"][reclen]["ip2"]
 
1016
                                if self.mail["received"][reclen]["ip"] == None:
 
1017
                                        del self.mail["received"][reclen]
 
1018
                                        break
 
1019
                                if self.mail["received"][reclen]["helo"] == None and self.mail["received"][reclen]["helo2"] != None: self.mail["received"][reclen]["helo"] = self.mail["received"][reclen]["helo2"]
 
1020
                                del self.mail["received"][reclen]["helo2"]
 
1021
 
 
1022
#                               try:
 
1023
#                                       dns = self.dns.dns("%s.in-addr.arpa" % spf.reverse_dots(self.mail["received"][reclen]["ip"]), "PTR")
 
1024
#                                       if len(dns) > 0 and type(dns) is list:
 
1025
#                                               self.mail["received"][reclen]["dns"] = dns[0]
 
1026
#                               except DNS.DNSError, (errmsg):
 
1027
#                                       if errmsg in ["Timeout"]:
 
1028
#                                               debug("TIMEOUT SEEN")
 
1029
#                                       debug("DNS Error: %s (in header, received PTR)" % errmsg, LOG_DEBUG, id=self.id)
 
1030
#                                       pass
 
1031
                                break
 
1032
                        break
 
1033
 
 
1034
                if conf["main"]["timeme"]:
233
1035
                        if self.mail["timer"].has_key("smtp_header"):
234
 
                                self.mail["timer"]["smtp_header"] = str("%.4f") % (float(self.mail["timer"]["smtp_header"]) + timeme(timer, noshow=True))
 
1036
                                self.mail["timer"]["smtp_header"] = str("%.4f") % (float(self.mail["timer"]["smtp_header"]) + timeme(timer, id=self.id, noshow=True))
235
1037
                        else:
236
 
                                self.mail["timer"]["smtp_header"] = str("%.4f") % timeme(timer, noshow=True)
 
1038
                                self.mail["timer"]["smtp_header"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
237
1039
                return CONTINUE
238
1040
 
239
1041
        def eoh(self):
240
 
                if conf["main"]["timeme"] is True: timer = timeme()
241
 
                self.log("SpamMilter.eoh()")
 
1042
                if conf["main"]["timeme"]: timer = timeme()
 
1043
                debug("SpamMilter.eoh()", LOG_DEBUG, id=self.id)
242
1044
                if self.mail: self.mail["smtpcmds"].append("eoh")
243
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_eoh"] = str("%.4f") % timeme(timer, id=self.mail["id"], noshow=True)
 
1045
 
 
1046
                toremove = []
 
1047
                for r in self.mail["received"]:
 
1048
                        # Now we have received lines, make readable and remove stupid entries
 
1049
                        if self.mail["received"][r].has_key("by"):
 
1050
                                del self.mail["received"][r]["by"]
 
1051
                        for f in self.mail["received"][r].keys():
 
1052
                                if self.mail["received"][r][f] == None:
 
1053
                                        del self.mail["received"][r][f]
 
1054
 
 
1055
                        if not self.mail["received"][r].has_key("ip"):
 
1056
                                toremove.append(r)
 
1057
                                continue
 
1058
                        if r>1 and is_listed(self.mail["received"][r]["ip"], ["^127\.", "^192\.168\.", "^10\."], noshow=True):
 
1059
                                toremove.append(r)
 
1060
                                continue
 
1061
 
 
1062
                        try:
 
1063
                                if self.mail["received"][r].has_key("dns"):
 
1064
                                        if self.mail["received"][r]["dns"][0] == "[" or is_listed(self.mail["received"][r]["dns"], ["^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"], noshow=True):
 
1065
                                                del self.mail["received"][r]["dns"]
 
1066
                                if self.mail["received"][r].has_key("helo"):
 
1067
                                        if self.mail["received"][r]["helo"][0] == "[":
 
1068
                                                self.mail["received"][r]["helo"] = self.mail["received"][r]["helo"][1:-1]
 
1069
### I had this commented out, dunno why :)
 
1070
                                        if self.mail["received"][r].has_key("dns") and self.mail["received"][r]["dns"] == self.mail["received"][r]["helo"]:
 
1071
                                                del self.mail["received"][r]["helo"]
 
1072
                                        elif is_listed(self.mail["received"][r]["helo"], ["^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"], noshow=True):
 
1073
                                                if self.mail["received"][r]["helo"] == self.mail["received"][r]["ip"]:
 
1074
                                                        del self.mail["received"][r]["helo"]
 
1075
                                                elif is_listed(self.mail["received"][r]["helo"], ["^127\.", "^192\.168\.", "^10\."], noshow=True):
 
1076
                                                        del self.mail["received"][r]["helo"]
 
1077
###
 
1078
                        except:
 
1079
                                debug("%s: %s (eoh -> received fix)" % (sys.exc_type, sys.exc_value), LOG_ERR, id=self.id, trace=False)
 
1080
 
 
1081
                for r in toremove:
 
1082
                        del self.mail["received"][r]
 
1083
 
 
1084
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_eoh"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
244
1085
                return CONTINUE
245
1086
 
246
1087
        def body(self,chunk):
247
 
                if conf["main"]["timeme"] is True: timer = timeme()
248
 
                self.log("SpamMilter.body() (chunk size: %d)" % len(chunk))
 
1088
                if conf["main"]["timeme"]: timer = timeme()
 
1089
                debug("SpamMilter.body() (chunk size: %d)" % len(chunk), LOG_DEBUG, id=self.id)
 
1090
 
249
1091
                if self.mail: self.mail["smtpcmds"].append("body")
250
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_body"] = str("%.4f") % timeme(timer, noshow=True)
 
1092
                if self.tmp and self.mail["size"] == 0:
 
1093
                        self.tmp.write("\n")
 
1094
                self.mail["size"] += len(chunk)
 
1095
                if self.tmp: self.tmp.write(chunk)
 
1096
 
 
1097
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_body"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
251
1098
                return CONTINUE
252
1099
 
253
1100
        def eom(self):
254
 
                if conf["main"]["timeme"] is True: timer = timeme()
255
 
                self.log("SpamMilter.eom()")
 
1101
                if conf["main"]["timeme"]: timer = timeme()
 
1102
                debug("SpamMilter.eom()", LOG_DEBUG, id=self.id)
256
1103
                if self.mail: self.mail["smtpcmds"].append("eom")
257
 
                if conf["main"]["timeme"] is True: self.mail["timer"]["smtp_eom"] = str("%.4f") % timeme(timer, noshow=True)
258
 
 
259
 
                return CONTINUE
260
 
 
261
 
        def abort(self):
262
 
                self.log("SpamMilter.abort()")
263
 
                self.mail = None
264
 
                self._cleanup()
265
 
                return CONTINUE
266
 
 
267
 
        def close(self):
268
 
                self.log("SpamMilter.close()")
269
 
                if self.mail: self.mail["smtpcmds"].append("close")
270
 
                self._cleanup()
 
1104
 
 
1105
###
 
1106
### Sample how filter can 'answer' mail messages
 
1107
###
 
1108
                if self.mail["to"][0][0:11] == "spamfilter@":
 
1109
                        debug("Test request from %s" % self.mail["from"][0], LOG_ERR, id=self.id)
 
1110
                        if conf["runtime"]["offline"]: return
 
1111
                        self.delrcpt(self.mail["to"][0])
 
1112
                        self.addrcpt(self.mail["from"][0])
 
1113
                        self.chgheader("MIME-Version", 1, "")
 
1114
                        self.chgheader("Reply-To", 0, "")
 
1115
                        self.chgheader("From", 0, "Spam Filter <abuse@%s>" % (self.mail["my"]["dns"]))
 
1116
                        self.chgheader("Content-Type", 1, "text/plain")
 
1117
                        self.chgheader("Subject", 1, "Test message from %s" % (self.mail["from"][0]))
 
1118
                        self.replacebody("""
 
1119
 
 
1120
Your test message was received
 
1121
 
 
1122
""")
 
1123
                        return Milter.ACCEPT
 
1124
 
 
1125
                # Authenticated sender, accept without logging
 
1126
                if not conf["runtime"]["offline"] and self.mail.has_key("smtp_auth"):
 
1127
                        debug("\tskip, authenticated", LOG_DEBUG, id=self.id)
 
1128
                        return Milter.ACCEPT
 
1129
                # Domain (not) found from filtered list
 
1130
#
 
1131
#
 
1132
# TODO: root@smtp-dev1.sspamm.com should match with "filter sspamm.com"
 
1133
#
 
1134
#
 
1135
 
 
1136
                (tests, self.mail) = is_filtered(self.mail)
 
1137
 
 
1138
                if not tests:
 
1139
                        debug("\tskip, not filtered domain", LOG_DEBUG, id=self.id)
 
1140
                        return Milter.CONTINUE
 
1141
                else:
 
1142
                        self.mail["rules"] = is_listed(self.mail["todomain"], conf["filter"]["rules"], id=self.id)
 
1143
 
 
1144
                if self.mail["size"] == 0 and self.mail["subject"] == "":
 
1145
                        self.mail["type"] = "empty"
 
1146
                        return Milter.DISCARD
 
1147
 
 
1148
                subchar = []
 
1149
                try:
 
1150
                        if self.tmp:
 
1151
                                self.tmp.seek(0)
 
1152
                                msg = message_from_file(self.tmp)
 
1153
                                (subj, subchar) = decode_header(msg["subject"])[0]
 
1154
                                self.mail["subject"] = oneliner(stripUnprintable(subj), id=self.id)
 
1155
                                self.mail["rawsubject"] = oneliner(stripUnprintable(msg["subject"]), id=self.id)
 
1156
                                self.mail["raw"] = """%s""" % msg
 
1157
                                self.mail["mime"] = mimepart(msg, id=self.id)
 
1158
                                charset = msg.get_charsets()
 
1159
                                charset.append(subchar)
 
1160
                                self.mail["charset"]=uniq(charset)
 
1161
                except:
 
1162
# WORKME? If 'crashes' on email.message_from_file, save that tmp file into
 
1163
# /tmp so we can take look of it later. Now we just need to pass mail
 
1164
# without checking.
 
1165
                        debug("eom(): %s: %s" % (sys.exc_type, sys.exc_value), LOG_ERR, id=self.id, trace=False)
 
1166
                        if self.tmp:
 
1167
                                self.tmp.close()
 
1168
                                mv(self.tmp.name, "/tmp/%s" % (self.tmpname))
 
1169
                                save_vars(self.mail, "/tmp/%s.var" % (self.tmpname), id=mail["id"]);
 
1170
                                debug("saved as /tmp/%s" % (self.tmpname), LOG_ERR, id=self.id, trace=False)
 
1171
                        return Milter.CONTINUE
 
1172
 
 
1173
##
 
1174
## Now message is received and processed. Fun part begins now, testing.
 
1175
##
 
1176
                for test in tests:
 
1177
                        if test in ['disable']: continue
 
1178
                        try:
 
1179
                                debug("*call test_%s()" % (test), LOG_DEBUG, id=self.mail["id"])
 
1180
#                               (ret, self.mail) = eval("test_"+test)(self.mail)
 
1181
#                               if conf["runtime"]["offline"]:
 
1182
#                                       if ret:
 
1183
#                                               matches[test] = "%s: %s%s" % (self.mail["action"], self.mail["reason"], self.mail["detail"])
 
1184
#                                       else:
 
1185
#                                               matches[test] = "pass"
 
1186
#                               if ret:
 
1187
#                                       if not conf["runtime"]["offline"]:
 
1188
#                                               debug("TEST MATCHES: %s" % (test), LOG_NOTICE, id=self.mail["id"])
 
1189
#                                       break
 
1190
#                               else:
 
1191
#                                       if test == "bayesian": self.mail["bayesiantype"] = self.mail["type"]
 
1192
#                               continue
 
1193
                        except:
 
1194
                                debug("Test: %s" % (test), LOG_DEBUG)
 
1195
                                debug("%s: %s (do tests)" % (sys.exc_type, sys.exc_value), LOG_DEBUG, id=self.id, trace=False)
 
1196
                                save_vars(self.mail, "/tmp/%s.var" % (self.tmpname), id=mail["id"]);
 
1197
                                traceback.print_exc(limit=None, file=sys.stderr)
 
1198
 
 
1199
                        continue
 
1200
 
 
1201
 
 
1202
 
 
1203
 
 
1204
 
 
1205
 
 
1206
 
 
1207
 
 
1208
 
 
1209
                if conf["main"]["timeme"]: self.mail["timer"]["smtp_eom"] = str("%.4f") % (timeme(timer, id=self.id, noshow=True))
 
1210
 
271
1211
                return CONTINUE
272
1212
 
273
1213
## List of Milter commands:
282
1222
#       def progress(self):
283
1223
 
284
1224
 
 
1225
##############################################################################
 
1226
##
 
1227
## CHILD THREADS
 
1228
##
 
1229
def Tconfig(childname=None):
 
1230
        global conffile, conf
 
1231
        
 
1232
        debug("Tconfig")
 
1233
        if conf["runtime"]["conffile"] == None:
 
1234
                debug("Scan for config file location")
 
1235
 
 
1236
                files = [ ]
 
1237
                if conffile[0] != "/":
 
1238
                        files.append("%s/%s" % (startdir, conffile))
 
1239
                        files.append("/etc/%s" % conffile)
 
1240
                        files.append("/etc/sspamm/%s" % conffile)
 
1241
                        files.append(None)
 
1242
 
 
1243
                cf = conffile
 
1244
                for conffile in files:
 
1245
                        if conffile != None and os.access(conffile, os.R_OK):
 
1246
                                break
 
1247
 
 
1248
                if conffile == None:
 
1249
                        debug("FATAL: Can't find or read %s." % (cf), LOG_ERR)
 
1250
                        return
 
1251
 
 
1252
                debug("Config file %s found." % conffile)
 
1253
 
 
1254
        debug("Tconfig loop started")
 
1255
        while conf["runtime"]["endtime"] == 0:
 
1256
                if not os.access(conffile, os.R_OK):
 
1257
                        debug("FATAL: Can't access %s." % (conffile), LOG_ERR)
 
1258
                        time.sleep(60)
 
1259
                        continue
 
1260
 
 
1261
                debug("Configuration file %s tested for update" % (conffile), LOG_DEBUG)
 
1262
                if conf["runtime"]["conftime"] < os.stat(conffile)[8]:
 
1263
                        if conf["runtime"]["conftime"] > 0:
 
1264
                                config_save()
 
1265
                        config_load(conffile)
 
1266
                        debug("Configuration %s reloaded" % (conffile), LOG_DEBUG)
 
1267
                        conf["runtime"]["conftime"] = os.stat(conffile)[8]
 
1268
                        conf["runtime"]["conffile"] = conffile
 
1269
                        if conf["main"]["sspammdir"]:
 
1270
                                conf["runtime"]["confpath"] = conf["main"]["sspammdir"]
 
1271
                        else:
 
1272
                                conf["runtime"]["confpath"] = conf["runtime"]["conffile"][0:conf["runtime"]["conffile"].rfind("/")+1]
 
1273
 
 
1274
                        if conf["main"]["savedir"]:
 
1275
                                mkdir(conf["main"]["savedir"])
 
1276
                else:
 
1277
                        time.sleep(5)
 
1278
                if childname != "Configuration Loader":
 
1279
                        break
 
1280
                time.sleep(1)
 
1281
        return
285
1282
        
286
1283
 
287
1284
##
288
1285
## Main Function
289
1286
##
290
 
def cleanquit(signal=None, frame=None):
 
1287
def cleanquit():
291
1288
        global conf
292
1289
 
293
1290
        debug("cleanquit()")
294
 
        conf["runtime"]["endtime"] = time.time()
295
 
        debug("Spam Filter runtime was %.3f" % (conf["runtime"]["endtime"]-conf["runtime"]["starttime"]), LOG_ERR)
 
1291
        rm(conf["main"]["pid"])
 
1292
        conf["runtime"]["endtime"] = "%.0f" % time.time()
 
1293
        debug("Spam Filter runtime was %.0f seconds" % (int(conf["runtime"]["endtime"])-int(conf["runtime"]["starttime"])), LOG_ERR)
 
1294
#       print show_vars(conf["main"])
 
1295
        print show_vars(conf["runtime"])
296
1296
        sys.stdout.flush()
297
1297
        sys.exit(0)
298
1298
 
299
1299
def main():
300
1300
        global conf
301
1301
 
302
 
        # Signals does not work with MilterModule, so they are here only as sample
303
 
        signal.signal(signal.SIGHUP , signal.SIG_IGN) # 1
304
 
        signal.signal(signal.SIGQUIT, signal.SIG_IGN) # 3
305
 
        signal.signal(signal.SIGTRAP, signal.SIG_IGN) # 5
306
 
        signal.signal(signal.SIGABRT, signal.SIG_IGN) # 6
307
 
        signal.signal(signal.SIGBUS , cleanquit     ) # 7
308
 
        signal.signal(signal.SIGUSR1, signal.SIG_IGN) # 10
309
 
        signal.signal(signal.SIGUSR2, signal.SIG_IGN) # 12
310
 
        signal.signal(signal.SIGTERM, cleanquit     ) # 15
311
 
        signal.signal(signal.SIGINT , cleanquit     ) # ^C
312
 
        #signal.signal(signal.SIGALRM, cleanquit    )
313
 
        #signal.alarm(90)
 
1302
#       if conf["main"]["tmpdir"]:
 
1303
#               mkdir(conf["main"]["tmpdir"])
 
1304
#               os.chdir(conf["main"]["tmpdir"])
 
1305
 
 
1306
        if 0:
 
1307
                thread.start_new_thread(Tconfig,("Configuration Loader",))
 
1308
        else:
 
1309
                Tconfig()
 
1310
                signal(SIGHUP , Tconfig) # 1
 
1311
                signal(SIGINT , cleanquit) # ^C
 
1312
                signal(SIGBUS , cleanquit) # 7
 
1313
                signal(SIGTERM, cleanquit) # 15
 
1314
 
 
1315
 
 
1316
        wtime = 0
 
1317
        maxwtime = 5
 
1318
        while conf["runtime"]["conftime"] == 0 and wtime < maxwtime:
 
1319
                sys.stdout.flush()
 
1320
                time.sleep(1)
 
1321
                wtime += 1
 
1322
        if conf["runtime"]["conftime"] == 0:
 
1323
                debug("Couldn't load configuration file in %d seconds." % (maxwtime), LOG_ERR)
 
1324
                sys.stdout.flush()
 
1325
                return
314
1326
 
315
1327
        Milter.factory = SpamMilter
316
1328
        Milter.set_flags(ADDRCPT + DELRCPT + ADDHDRS + CHGHDRS + CHGBODY)
317
1329
 
 
1330
        debug("Spam Filter started", LOG_ERR)
318
1331
        try:
319
 
                Milter.runmilter("sspamm","inet:7999",300)
 
1332
                Milter.runmilter(conf["main"]["name"],conf["main"]["port"],300)
320
1333
        except SystemExit:
321
1334
                pass
322
1335
        except:
328
1341
##
329
1342
if __name__ == "__main__":
330
1343
        tmp = None
331
 
        conf["runtime"]["starttime"] = time.time()
 
1344
        conf["runtime"]["starttime"] = "%.0f" % time.time()
332
1345
        startdir = os.getcwd()
333
1346
 
334
1347
        os.nice(5)
335
1348
        locale.setlocale(locale.LC_CTYPE, 'fi_FI.UTF-8')
 
1349
 
336
1350
        if not sys.argv[1:]:
337
 
                main()
 
1351
                if(makepid(conf["main"]["pid"])): main()
338
1352
        else:
 
1353
                """ Usage """
339
1354
                sys.exit(1)
340
1355
sys.exit(0)