[hautefeuille] Traitement d’un fichier de log Apache et génération de svg

python grabip.py log.txt

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

"""
Usage: grabip.py [OPTIONS] filein

"""

import re
import collections

import GeoIP
import clize

import pygal
from pygal.style import NeonStyle

class Inspector(object):
    '''
    Extract IP from log file, count ip, localise with GeoIP country
    '''
    def __init__(self, in_file, **kwds):
        super(Inspector, self).__init__(**kwds)
        self.in_file = open(in_file, 'r')
        self.exclude = ["0.0.0.0", "127.0.0.1"]
        self.ip = []
        self.cnt = collections.Counter()
        self.cnt_u = collections.Counter()
        self.result = []
        self.hits = {}
        self.uniques = {}
        self.total = 0
        self.unique = 0
        self.killdoublon = set()
        self.geoip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)

        self.filter()
        self.count()
        self.geo()
        self.hitbycountry()
        self.totalhits()
        self.totalunique()
        self.uniquebycountry()
        self.in_file.close()

    def filter(self):
        '''
        Strip line and regex for ip

        self.ip return ['78.46.16.11', '78.46.16.11']
        '''
        for line in self.in_file:
            ipp = re.search("([\d]+\.[\d]+\.[\d]+\.[\d]+)",
                            line.rstrip('\n\r'))
            if ipp and ipp not in self.exclude:
                self.ip.append(ipp.group())

        print("=> filter : %s" % self.ip)

    def count(self):
        '''
        Add a count for all ip found

        self.cnt.items() return [('78.46.16.11', 2)]
        '''
        for ip in self.ip:
            self.cnt[ip] += 1

        print("=> count : %s" % self.cnt.items())

    def geo(self):
        '''
        Add goip information to dataset

        self.result return [('78.46.16.11', 2, 'DE')]
        '''
        for ip, i in self.cnt.items():
            country = self.geoip.country_code_by_addr(ip)
            if country:
                self.result.append((ip, i, country))

        print("=> geo : %s" % self.result)

    def hitbycountry(self):
        '''
        Count hit by country

        self.hits return {'DE': 2}
        '''
        for ip, hit, country in self.result:
            if country in self.hits:
                tmp = int(self.hits.get(country)) + int(hit)
                self.hits[country] = tmp
            else:
                self.hits[country] = hit

        print("=> hits : %s" % self.hits)

    def uniquebycountry(self):
        '''
        Count unique by country
        '''
        for ip in self.killdoublon:
            country = self.geoip.country_code_by_addr(ip)
            if country:
                self.cnt_u[country] += 1
        self.uniques = dict(self.cnt_u)

        print("=> uniques : %s" % self.uniques)

    def totalhits(self):
        '''
        Return total hits

        self.total return 2
        '''
        for value in self.hits.itervalues():
            self.total += value

        print("=> total hits : %s" % self.total)

    def totalunique(self):
        '''
        Return total unique visitor

        self.unique return 1
        '''
        for elem in self.ip:
            self.killdoublon.add(elem)
        self.unique = len(self.killdoublon)

        print("=> total unique : %s" % self.unique)

class Reporting(Inspector):
    '''
    Create a report file text
    '''
    def __init__(self, in_file, **kwds):
        super(Reporting, self).__init__(in_file, **kwds)
        self.out_file = open("rapport.txt", 'w')
        self.simplereport()
        self.out_file.close()

    def simplereport(self):
        '''
        Print result and write file report
        '''
        for ip, i, country in self.result:
            self.out_file.write(ip + ' ' + str(i) + ' ' + country + '\n')

        print("=> Report done")

class Drawing(Reporting):
    '''
    Create visual representation

    '''
    def __init__(self, in_file, **kwds):
        super(Drawing, self).__init__(in_file, **kwds)
        chart = pygal.HorizontalBar(
            width=800,
            height=600,
            x_title="Visites",
            y_title="Pays",
            legend_at_bottom=True,
            tooltip_border_radius=10,
            human_readable=True,
            no_data_text='No result found',
            pretty_print=True,
            style=NeonStyle)
        chart.title = u"Visiteurs uniques à travers le monde sur\
        le site hautefeuille.eu en 2013"
        for k, v in self.uniques.iteritems():
            if v >= 50:
                chart.add(k, [{'value': v, 'label': k}])
        chart.render_to_file('chart.svg')

@clize.clize()
def main(filein):
    '''
    Main
    '''
    grab = Drawing(filein)

if __name__ == '__main__':
    clize.run(main)