nixspamsum
author Pascal Volk <user@localhost.localdomain.org>
Sun, 07 Jun 2009 11:40:02 +0000
changeset 0 2d97e75f16cf
child 1 7d5cee19c20a
permissions -rwxr-xr-x
initial commit: »don't fear the nervous delete finger«
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     1
#!/usr/bin/env python
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     2
# -*- coding: utf-8 -*-
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     3
# Copyright 2009 Pascal Volk
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     4
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     5
__author__ = 'Pascal Volk'
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     6
__version__ = '0.1'
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     7
__date__ = '2009-06-07'
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     8
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
     9
import os
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    10
import re
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    11
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    12
class NiXSpamPlot:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    13
    """Do sth ..."""
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    14
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    15
    """Regular expression pattern for mail logs from Postfix"""
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    16
    RE_PF = '''^[\w\s:-]{17,80}\spostfix\/smtpd\[[\d]{3,5}\]: NOQUEUE: reject:.*blocked using ix.dnsbl.manitu.net; Spam sent to the mailhost ((?:[a-z0-9-]{1,63}\.){1,}[a-z]{2,6}) was detected by NiX Spam.*$'''
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    17
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    18
    def __init__(self):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    19
        self._doms = {}
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    20
        self._mxs  = {}
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    21
        self._repo = None
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    22
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    23
    def setLogFormat(self, format='postfix'):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    24
        if format == 'postfix':
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    25
            self._repo = re.compile(NiXSpamPlot.RE_PF)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    26
        else:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    27
            raise Exception('MTA/Logformat not supported yet.')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    28
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    29
    def parseLog(self, filehandle):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    30
        for l in filehandle:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    31
            mo = self._repo.match(l)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    32
            if mo:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    33
                mx = mo.group(1)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    34
                try:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    35
                    self._mxs[mx] += 1
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    36
                except KeyError:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    37
                    self._mxs[mx]  = 1
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    38
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    39
    def countByDom(self):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    40
        for mx in self._mxs.keys():
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    41
            dom = '.'.join(mx.split('.')[-2:])
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    42
            try:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    43
                self._doms[dom] += self._mxs[mx]
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    44
            except KeyError:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    45
                self._doms[dom]  = self._mxs[mx]
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    46
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    47
    def getDomains(self):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    48
        return self._doms
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    49
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    50
    def getMXs(self):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    51
        return self._mxs
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    52
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    53
    def getTotal(self):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    54
        return sum(self._mxs.values())
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    55
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    56
def getOptionParser():
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    57
    from optparse import OptionParser
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    58
    usage  = 'usage: %prog [options] maillog [maillog [...]]'
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    59
    parser = OptionParser(usage=usage, description='do something ...')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    60
    parser.add_option('-d', action='store_true', dest='countByDom',
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    61
            default=False, help='summarize all MX by domain')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    62
    parser.add_option('-m', action='store_false', dest='countByDom',
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    63
            help='count per MX host [default]')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    64
    parser.add_option('-o', dest='oFormat', default='table',metavar='FORMAT',
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    65
            help='the output format: table or csv [default: %default]')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    66
    parser.add_option('-p', action='store_true', dest='percent', default=False,
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    67
            help='show also percentages in table output [default: %default]')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    68
    parser.add_option('-s', dest='order', default='name', metavar='SORTBY',
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    69
            help='arrange output by: name or count [default: %default]')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    70
    parser.add_option('-t', dest='format', default='postfix',metavar='MTA',
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    71
            help='MTA that generated the maillog [default: %default]')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    72
    return parser
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    73
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    74
def openLogFile(fname):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    75
    try:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    76
        fh = open(fname)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    77
        return fh
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    78
    except IOError, e:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    79
        os.sys.stderr.write('Warning: %s\nskipped file: %s\n' % (e.strerror,
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    80
            fname))
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    81
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    82
def showResult(nixspamplot, options):
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    83
    if options.countByDom:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    84
        nixspamplot.countByDom()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    85
        domains = nixspamplot.getDomains()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    86
    else:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    87
        domains = nixspamplot.getMXs()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    88
    if options.percent:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    89
        total   = nixspamplot.getTotal()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    90
    k = 0 if options.order == 'name' else 1
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    91
    doms = sorted(domains.items(), lambda d,c: cmp(d[k],c[k]), reverse=k)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    92
    for d in doms:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    93
        print "%s -> %d" % d
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    94
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    95
def main():
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    96
    parser = getOptionParser()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    97
    opts, args = parser.parse_args()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    98
    if len(args) < 1:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
    99
        parser.error('No logfiles specified')
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   100
    nsp = NiXSpamPlot()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   101
    nsp.setLogFormat(opts.format)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   102
    for fn in args:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   103
        fh = openLogFile(fn)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   104
        if fh is not None:
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   105
            nsp.parseLog(fh)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   106
            fh.close()
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   107
    showResult(nsp, opts)
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   108
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   109
if __name__ == '__main__':
2d97e75f16cf initial commit: »don't fear the nervous delete finger«
Pascal Volk <user@localhost.localdomain.org>
parents:
diff changeset
   110
    main()