PNG  IHDRxsBIT|d pHYs+tEXtSoftwarewww.inkscape.org<,tEXtComment File Manager

File Manager

Path: /opt/alt/python27/lib/python2.7/site-packages/postomaat/plugins/

Viewing File: suspect_collect.py

"""
Plugin will extract data from configured backends
using query/task/mapping configurations provided
in a form of file/database/etc

This plugin will set additional tags for suspect object
those tags can then be accessed and used by other plugins
"""

__version__ = "0.0.1"

import os
import re
from collections import defaultdict
from postomaat.extensions.sql import SQL_EXTENSION_ENABLED, get_session
from postomaat.shared import DUNNO, ScannerPlugin

if SQL_EXTENSION_ENABLED:
    from sqlalchemy.sql import select, column, table, text


class Query(object):
    def __init__(self):
        self.columns = list()
        self.table = str()
        self.filters = list()
        self.column_fmap = dict()
        self.filter_fmap = dict()
        self.statement = None

    def __str__(self):
        return "<Query columns={columns:s} table={table:s} filters={filters:s}>".format(
            columns=self.columns,
            table=self.table,
            filters=self.filters
        )


class SuspectCollect(ScannerPlugin):
    def __init__(self, config, section=None):
        ScannerPlugin.__init__(self, config, section)
        self.backendconfig = self.config.get(self.section, 'backendconfig')
        self.queryfile = self.config.get(self.section, 'queryfile')
        self.requiredvars = {
            "backendconfig": {
                'default': 'mysql://root@localhost/sender_meta_db?charset=utf8',
                'description': 'SQLAlchemy Connection string'
            },
            "queryfile": {
                'default': "/etc/postomaat/conf.d/suspectcollect.queries",
                'description': 'file with queriy configs to run for data collection'
            }
        }
        self.logger = self._logger()
        self.queries = None

    def load_queries(self, queryconfig):
        patt = re.compile(r'^select\s+columns=(?P<columns>[^\s]+)\s+from=(?P<from>[^\s]+)\s+filters=(?P<filters>[^\s]+)$') #pylint: disable=C0301
        queries = []
        lineno = 0
        for line in queryconfig.split('\n'):
            lineno += 1
            line = line.strip()

            if line.startswith('#') or line.strip() == '':
                continue

            match = patt.match(line)
            if match is None:
                self.logger.error('cannot parse limiter config line %d', lineno)
                continue

            gdict = match.groupdict()
            query = Query()
            query.columns = gdict['columns']
            query.table = gdict['from']
            query.filters = gdict['filters']
            queries.append(query)
        return queries

    def get_fieldmap(self, fieldstring):
        """
        Create databse column name to Postomaat suspect tag map based on the query string
        by default, database column name is the same as Postomaat suspect tag name,
        but, to override the mapping, the config can be in the form:

        db_column_name:suspect_tag

        eg.
        select columns=order_id:orderid,last_login:lastlogin from=mailbox filterby=address:sender

        You will send (1):
            - order_id:orderid,last_login:lastlogin
            - as a result select will:
                - select 'order_id' column from database as 'orderid' suspect tag
                - select 'last_login' column from database as 'lastlogin' suspect tag

        You will send (2):
            - address:sender
            - as a result select will:
                - filter by 'address' column in database using suspect tag 'sender' as value

        :param fields: list of strings generate fieldmap from (type: list)
        :return: dictionary of fieldmaps (type: dict)
        """

        fields = fieldstring.split(',')

        fieldmap = {}
        for field in fields:
            if ':' in field:
                (db_column, suspect_tag) = field.split(':', 1)
                fieldmap[db_column] = suspect_tag
            else:
                fieldmap[field] = field
        return fieldmap

    def get_suspect_attribute(self, suspect, attribute):
        if attribute == 'from_address':
            attribute_value = suspect.from_address
        elif attribute == 'from_domain':
            attribute_value = suspect.from_domain
        elif attribute == 'to_address':
            attribute_value = suspect.to_address
        elif attribute == 'to_domain':
            attribute_value = suspect.to_domain
        else:
            attribute_value = suspect.get_value(attribute)

        if attribute_value is None:
            self.logger.warning("Suspect does not have attribute %s. Typo or missing plugin?",
                                attribute)
        return attribute_value

    def build_query(self, columns=None, from_table=None, filters=None):
        """Build SQL query object

        :param columns: list of columns (type: list)
        :param table: table name to select from (type: string)
        :param filter: list of filters to build 'where' clause (AND) (type: list)
        :return: returns query object (type: sqlalchemy.engine.result.RowProxy)
        """
        stmt = (
            select()
            .with_only_columns([column(c) for c in columns])
            .select_from(table(from_table))
            .limit('1')
        )
        for filter_string in filters:
            stmt = stmt.where(text(filter_string))
        return stmt

    def add_tags(self, suspect, query, db_result):
        for db_column, value in db_result.items():
            tag = query.column_fmap[db_column]
            suspect.tags[tag] = value

    def lint(self):
        if not SQL_EXTENSION_ENABLED:
            print("sqlalchemy is not installed")
            return False

        if not self.checkConfig():
            return False

        return True

    def examine(self, suspect):
        session = get_session(self.backendconfig)

        if self.queries is None:
            filename = self.queryfile
            if not os.path.exists(filename):
                self.logger.error('Limiter config file %s not found', filename)
                return None
            with open(filename) as filehandle:
                queryconfig = filehandle.read()
            self.queries = self.load_queries(queryconfig)

            for query in self.queries:
                query.column_fmap = self.get_fieldmap(query.columns)
                query.filter_fmap = self.get_fieldmap(query.filters)
                query.statement = self.build_query(
                    columns=query.column_fmap.keys(),
                    filters=[
                        str(column) + ' = :' + str(tag) for column, tag in query.filter_fmap.items()
                    ],
                    from_table=query.table
                )
            self.logger.info('Found %d query configurations', len(self.queries))

        for query in self.queries:
            # We GET attributes ("real" data of suspect), but SET tags (additional data)
            filter_data = defaultdict()
            for _, attribute in query.filter_fmap.items():
                filter_data[attribute] = self.get_suspect_attribute(suspect, attribute)

            db_result = None

            try:
                db_result = session.execute(
                    query.statement,
                    filter_data
                ).fetchone()
            except Exception as err:
                self.logger.critical(
                    "Got exception while running query %s with parameters %s. Exception was: %s",
                    query.statement,
                    filter_data,
                    err
                )

            if db_result is None:
                self.logger.warning(
                    "Query %s did not return result with parameters %s. Not trying to add tags for this suspect.",
                    query.statement,
                    filter_data
                )
            else:
                self.add_tags(suspect, query, db_result)

        return DUNNO

    def __str__(self):
        return "SuspectCollect"

b IDATxytVսϓ22 A@IR :hCiZ[v*E:WũZA ^dQeQ @ !jZ'>gsV仿$|?g)&x-EIENT ;@xT.i%-X}SvS5.r/UHz^_$-W"w)Ɗ/@Z &IoX P$K}JzX:;` &, ŋui,e6mX ԵrKb1ԗ)DADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADA݀!I*]R;I2$eZ#ORZSrr6mteffu*((Pu'v{DIߔ4^pIm'77WEEE;vƎ4-$]'RI{\I&G :IHJ DWBB=\WR޽m o$K(V9ABB.}jѢv`^?IOȅ} ڶmG}T#FJ`56$-ھ}FI&v;0(h;Б38CӧOWf!;A i:F_m9s&|q%=#wZprrrla A &P\\СC[A#! {olF} `E2}MK/vV)i{4BffV\|ۭX`b@kɶ@%i$K z5zhmX[IXZ` 'b%$r5M4º/l ԃߖxhʔ)[@=} K6IM}^5k㏷݆z ΗÿO:gdGBmyT/@+Vɶ纽z񕏵l.y޴it뭷zV0[Y^>Wsqs}\/@$(T7f.InݺiR$푔n.~?H))\ZRW'Mo~v Ov6oԃxz! S,&xm/yɞԟ?'uaSѽb,8GלKboi&3t7Y,)JJ c[nzӳdE&KsZLӄ I?@&%ӟ۶mSMMњ0iؐSZ,|J+N ~,0A0!5%Q-YQQa3}$_vVrf9f?S8`zDADADADADADADADADAdqP,تmMmg1V?rSI꒟]u|l RCyEf٢9 jURbztѰ!m5~tGj2DhG*{H9)꒟ר3:(+3\?/;TUݭʴ~S6lڧUJ*i$d(#=Yݺd{,p|3B))q:vN0Y.jkק6;SɶVzHJJЀ-utѹսk>QUU\޲~]fFnK?&ߡ5b=z9)^|u_k-[y%ZNU6 7Mi:]ۦtk[n X(e6Bb."8cۭ|~teuuw|ήI-5"~Uk;ZicEmN/:]M> cQ^uiƞ??Ңpc#TUU3UakNwA`:Y_V-8.KKfRitv޲* 9S6ֿj,ՃNOMߤ]z^fOh|<>@Å5 _/Iu?{SY4hK/2]4%it5q]GGe2%iR| W&f*^]??vq[LgE_3f}Fxu~}qd-ږFxu~I N>\;͗O֊:̗WJ@BhW=y|GgwܷH_NY?)Tdi'?խwhlmQi !SUUsw4kӺe4rfxu-[nHtMFj}H_u~w>)oV}(T'ebʒv3_[+vn@Ȭ\S}ot}w=kHFnxg S 0eޢm~l}uqZfFoZuuEg `zt~? b;t%>WTkķh[2eG8LIWx,^\thrl^Ϊ{=dž<}qV@ ⠨Wy^LF_>0UkDuʫuCs$)Iv:IK;6ֲ4{^6եm+l3>݆uM 9u?>Zc }g~qhKwڭeFMM~pМuqǿz6Tb@8@Y|jx](^]gf}M"tG -w.@vOqh~/HII`S[l.6nØXL9vUcOoB\xoǤ'T&IǍQw_wpv[kmO{w~>#=P1Pɞa-we:iǏlHo׈꒟f9SzH?+shk%Fs:qVhqY`jvO'ρ?PyX3lх]˾uV{ݞ]1,MzYNW~̈́ joYn}ȚF߾׮mS]F z+EDxm/d{F{-W-4wY듏:??_gPf ^3ecg ҵs8R2מz@TANGj)}CNi/R~}c:5{!ZHӋӾ6}T]G]7W6^n 9*,YqOZj:P?Q DFL|?-^.Ɵ7}fFh׶xe2Pscz1&5\cn[=Vn[ĶE鎀uˌd3GII k;lNmشOuuRVfBE]ۣeӶu :X-[(er4~LHi6:Ѻ@ԅrST0trk%$Č0ez" *z"T/X9|8.C5Feg}CQ%͞ˣJvL/?j^h&9xF`њZ(&yF&Iݻfg#W;3^{Wo^4'vV[[K';+mӍִ]AC@W?1^{එyh +^]fm~iԵ]AB@WTk̏t uR?l.OIHiYyԶ]Aˀ7c:q}ힽaf6Z~қm(+sK4{^6}T*UUu]n.:kx{:2 _m=sAߤU@?Z-Vކеz왍Nэ{|5 pڶn b p-@sPg]0G7fy-M{GCF'%{4`=$-Ge\ eU:m+Zt'WjO!OAF@ik&t݆ϥ_ e}=]"Wz_.͜E3leWFih|t-wZۍ-uw=6YN{6|} |*={Ѽn.S.z1zjۻTH]흾 DuDvmvK.`V]yY~sI@t?/ϓ. m&["+P?MzovVЫG3-GRR[(!!\_,^%?v@ҵő m`Y)tem8GMx.))A]Y i`ViW`?^~!S#^+ѽGZj?Vģ0.))A꨷lzL*]OXrY`DBBLOj{-MH'ii-ϰ ok7^ )쭡b]UXSְmռY|5*cֽk0B7镹%ڽP#8nȎq}mJr23_>lE5$iwui+ H~F`IjƵ@q \ @#qG0".0" l`„.0! ,AQHN6qzkKJ#o;`Xv2>,tێJJ7Z/*A .@fفjMzkg @TvZH3Zxu6Ra'%O?/dQ5xYkU]Rֽkق@DaS^RSּ5|BeHNN͘p HvcYcC5:y #`οb;z2.!kr}gUWkyZn=f Pvsn3p~;4p˚=ē~NmI] ¾ 0lH[_L hsh_ғߤc_њec)g7VIZ5yrgk̞W#IjӪv>՞y睝M8[|]\շ8M6%|@PZڨI-m>=k='aiRo-x?>Q.}`Ȏ:Wsmu u > .@,&;+!!˱tﭧDQwRW\vF\~Q7>spYw$%A~;~}6¾ g&if_=j,v+UL1(tWake:@Ș>j$Gq2t7S?vL|]u/ .(0E6Mk6hiۺzښOrifޱxm/Gx> Lal%%~{lBsR4*}{0Z/tNIɚpV^#Lf:u@k#RSu =S^ZyuR/.@n&΃z~B=0eg뺆#,Þ[B/?H uUf7y Wy}Bwegל`Wh(||`l`.;Ws?V@"c:iɍL֯PGv6zctM̠':wuW;d=;EveD}9J@B(0iհ bvP1{\P&G7D޴Iy_$-Qjm~Yrr&]CDv%bh|Yzni_ˆR;kg}nJOIIwyuL}{ЌNj}:+3Y?:WJ/N+Rzd=hb;dj͒suݔ@NKMԄ jqzC5@y°hL m;*5ezᕏ=ep XL n?מ:r`۵tŤZ|1v`V뽧_csج'ߤ%oTuumk%%%h)uy]Nk[n 'b2 l.=͜E%gf$[c;s:V-͞WߤWh-j7]4=F-X]>ZLSi[Y*We;Zan(ӇW|e(HNNP5[= r4tP &0<pc#`vTNV GFqvTi*Tyam$ߏWyE*VJKMTfFw>'$-ؽ.Ho.8c"@DADADADADADADADADA~j*֘,N;Pi3599h=goضLgiJ5փy~}&Zd9p֚ e:|hL``b/d9p? fgg+%%hMgXosج, ΩOl0Zh=xdjLmhݻoO[g_l,8a]٭+ӧ0$I]c]:粹:Teꢢ"5a^Kgh,&= =՟^߶“ߢE ܹS J}I%:8 IDAT~,9/ʃPW'Mo}zNƍ쨓zPbNZ~^z=4mswg;5 Y~SVMRXUյڱRf?s:w ;6H:ºi5-maM&O3;1IKeamZh͛7+##v+c ~u~ca]GnF'ټL~PPPbn voC4R,ӟgg %hq}@#M4IÇ Oy^xMZx ) yOw@HkN˖-Sǎmb]X@n+i͖!++K3gd\$mt$^YfJ\8PRF)77Wא!Cl$i:@@_oG I{$# 8磌ŋ91A (Im7֭>}ߴJq7ޗt^ -[ԩSj*}%]&' -ɓ'ꫯVzzvB#;a 7@GxI{j޼ƌ.LÇWBB7`O"I$/@R @eee@۷>}0,ɒ2$53Xs|cS~rpTYYY} kHc %&k.], @ADADADADADADADADA@lT<%''*Lo^={رc5h %$+CnܸQ3fҥK}vUVVs9G R,_{xˇ3o߾;TTTd}馛]uuuG~iԩ@4bnvmvfϞ /Peeeq}}za I~,誫{UWW뮻}_~YƍSMMMYχ֝waw\ďcxꩧtEƍկ_?۷5@u?1kNׯWzz/wy>}zj3 k(ٺuq_Zvf̘:~ ABQ&r|!%KҥKgԞ={<_X-z !CyFUUz~ ABQIIIjݺW$UXXDٳZ~ ABQƍecW$<(~<RSSvZujjjԧOZQu@4 8m&&&jԩg$ď1h ͟?_{768@g =@`)))5o6m3)ѣƌJ;wҿUTT /KZR{~a=@0o<*狔iFɶ[ˎ;T]]OX@?K.ۈxN pppppppppppppppppPfl߾] ,{ァk۶mڿo5BTӦMӴiӴ|r DB2e|An!Dy'tkΝ[A $***t5' "!駟oaDnΝ:t֭[gDШQ06qD;@ x M6v(PiizmZ4ew"@̴ixf [~-Fٱc&IZ2|n!?$@{[HTɏ#@hȎI# _m(F /6Z3z'\r,r!;w2Z3j=~GY7"I$iI.p_"?pN`y DD?: _  Gÿab7J !Bx@0 Bo cG@`1C[@0G @`0C_u V1 aCX>W ` | `!<S `"<. `#c`?cAC4 ?c p#~@0?:08&_MQ1J h#?/`7;I  q 7a wQ A 1 Hp !#<8/#@1Ul7=S=K.4Z?E_$i@!1!E4?`P_  @Bă10#: "aU,xbFY1 [n|n #'vEH:`xb #vD4Y hi.i&EΖv#O H4IŶ}:Ikh @tZRF#(tXҙzZ ?I3l7q@õ|ۍ1,GpuY Ꮿ@hJv#xxk$ v#9 5 }_$c S#=+"K{F*m7`#%H:NRSp6I?sIՖ{Ap$I$I:QRv2$Z @UJ*$]<FO4IENDB`