PNG
IHDR x sBIT|d pHYs + tEXtSoftware www.inkscape.org< ,tEXtComment
File Manager
File Manager
Viewing File: decision_maker.py
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains DecisionMaker class
"""
__package__ = 'ssa.modules'
import json
import logging
import os
from os.path import isfile
import numpy as np
from ssa.db import setup_database
from .common import Common
from .storage import (
iter_domains_data,
iter_urls_data,
get_url_durations
)
from ..configuration import load_tunables
from ..configuration.schemes import ssa_tunables_schema
from ..internal.constants import report_path
from ..internal.utils import previous_day_date, sentry_init
class DecisionMaker(Common):
"""
SSA Decision maker implementation.
"""
def __init__(self, engine=None):
super().__init__()
self.logger = logging.getLogger('decision_maker')
self.logger.info('DecisionMaker enabled: %s', __package__)
self.engine = engine if engine else setup_database()
def __call__(self):
self.logger.info('DecisionMaker started')
self.logger.debug('DecisionMaker loaded config: %s', self.config)
self.external_tunables = self.load_external_conf()
self.logger.debug('DecisionMaker loaded tunables: %s',
self.external_tunables)
report = self.data_processing()
self.add_json_report(report)
self.logger.info('DecisionMaker report: %s', report)
return report
@staticmethod
def _report_file(name) -> str:
"""
Full path to given filename in DM reports directory
"""
return os.path.join(report_path, name)
@property
def current_report_file(self) -> str:
"""
Full path to current DM report: report.json in DM reports directory
"""
return self._report_file('report.json')
@property
def _empty_report(self) -> dict:
"""
Returns empty report
"""
return dict(date=previous_day_date(), domains=[])
@property
def solo_filtered_options(self) -> set:
return {'correlation'}
@staticmethod
def load_external_conf():
"""Load external configuration values"""
return load_tunables('ssa.json', ssa_tunables_schema)
def data_processing(self) -> dict:
"""
Going through the list of domains, for each domain we go through
the list of urls. During data processing, we will form
the resulting dictionary.
"""
report = self._empty_report
for domain_data in iter_domains_data(self.engine):
# goes through the list of domains
urls_data = list()
domain_slow_reqs = 0
domain_url_durations = dict(get_url_durations(
self.engine, domain_data.domain_name))
for domain_data_key, domain_data_value in iter_urls_data(self.engine,
domain_data.domain_name,
list(domain_url_durations.keys())):
if self.is_ignored(domain_data_key):
self.logger.debug('%s ignored', domain_data_key)
continue
# goes through the list of urls, "domain_total_reqs" is also here
if domain_data_key not in self.non_url_fields:
# domain_data_key below - it is current url
if not self.is_throttling_suitable(
domain_data_value.get('url_throttled_reqs',
list([0] * 24)),
domain_data_value['url_total_reqs']):
# skip by allowed throttling percentage
continue
correlation_value = self.get_correlation(
domain_data_value['url_total_reqs'], domain_data.domain_total_reqs)
durations = domain_url_durations.get(domain_data_key)
if durations is None:
self.logger.error('Unable to get durations for %s', str(domain_data_key))
continue
if (self.request_number_exceeded(
domain_data_value['url_slow_reqs']) and
self.correlation_conditions(correlation_value)):
average_duration_calculation = np.mean(durations)
sum_url_slow_reqs = sum(
domain_data_value['url_slow_reqs'])
domain_slow_reqs += sum_url_slow_reqs
urls_data.append(dict(
name=domain_data_key, reqs_num=sum_url_slow_reqs,
average_duration=int(average_duration_calculation),
correlation=float(f'{correlation_value:.2f}')))
if urls_data:
sorted_urls = self.report_sorting(
list_to_sort=urls_data, leave_top=self.urls_number,
key_for_sorting='reqs_num')
report['domains'].append(dict(
name=domain_data.domain_name, slow_urls=len(sorted_urls),
slow_reqs=domain_slow_reqs,
total_reqs=sum(domain_data.domain_total_reqs), urls=sorted_urls))
if report['domains']:
report['domains'] = self.report_sorting(
list_to_sort=report['domains'], leave_top=self.domains_number,
key_for_sorting='slow_reqs')
return report
def list_handling_considering_time(self, url_slow_reqs: list) -> list:
"""
Based on the 'url_slow_reqs' list, a new list will be formed,
where the elements of the original list will be iteratively
summed by the number of elements equal to 'time'
"""
time = self.time or 24
return [sum(url_slow_reqs[i:time + i]) for i in
range(0, len(url_slow_reqs), time)]
def compare_elements_with_request_number(self,
url_slow_reqs_by_time: list) -> bool:
"""
This functions will check if any of elements is greater than "request_number"
"""
for i in url_slow_reqs_by_time:
if i >= self.request_number:
return True
return False
def get_correlation(self, url_total_reqs: list, domain_total_reqs: list):
"""
Calculates the correlation coefficient using the "url_total_reqs" and
the "domain_total_reqs" lists
"""
if not self.correlation:
return 0
return np.amin(np.corrcoef(url_total_reqs, domain_total_reqs))
@staticmethod
def report_sorting(list_to_sort: list, leave_top: int,
key_for_sorting: str) -> list:
"""
Will sort the domain list by "slow_reqs", the goal is to leave only
"domains_number" of uppers, also per each domain will sort urls by
"reqs_num", the goal is to leave only "urls_number" of uppers.
leave_top == 0 allows to keep the full list
"""
list_to_sort.sort(key=lambda dict_: dict_[key_for_sorting],
reverse=True)
if leave_top:
return list_to_sort[:leave_top]
else:
return list_to_sort
def rename_old_report(self):
"""
Rename old report
"""
old_report = self.current_report_file
if isfile(old_report):
with open(old_report) as json_data:
try:
d = json.load(json_data)
except json.JSONDecodeError:
date_from_report = 'unknown'
else:
date_from_report = d.get('date', 'dd.mm.yyyy').replace('.', '_')
new_report_name = f'report__{date_from_report}.json'
new_report = self._report_file(new_report_name)
os.rename(old_report, new_report)
def add_json_report(self, report: dict):
"""
Makes json report
"""
self.rename_old_report()
with open(self.current_report_file, 'w', encoding='utf-8') as f:
json.dump(report, f, ensure_ascii=False, indent=4)
def get_json_report(self) -> dict:
"""
Return contents of current report or empty report in case of error
"""
_filtering_hook = None
try:
with open(self.current_report_file) as report:
report_dict = json.load(report, object_hook=_filtering_hook)
except (OSError, json.JSONDecodeError):
report_dict = self._empty_report
return report_dict
def correlation_conditions(self, correlation_value: int) -> bool:
"""
If correlation flag is enabled - we'll compare correlation_coefficient
from configuration with calculated correlation coefficient.
If the calculated value exceeds the configuration value - we return
True otherwise False. At the same time if correlation flag is disabled -
we'll also return "True" since in this case the correlation coefficient
is not checked and its value is specified as zero in final report.
"""
if not self.correlation:
return True
return correlation_value > self.correlation_coefficient
def request_number_exceeded(self, url_slow_reqs):
"""
At least one element from the received list (url_slow_reqs_by_time)
must be greater than request_number
"""
url_slow_reqs_by_time = self.list_handling_considering_time(
url_slow_reqs)
return self.compare_elements_with_request_number(url_slow_reqs_by_time)
def is_throttling_suitable(self, url_throttled_reqs: list,
url_total_reqs: list) -> bool:
"""
Check that percent of throttled requests per URL passes given threshold
"""
throttled_percent = (sum(url_throttled_reqs) / sum(
url_total_reqs)) * 100
self.logger.debug('Calculated throttled percent %s', throttled_percent)
return throttled_percent <= self.external_tunables.get(
'allowed_throttling_percentage', 0)
if __name__ == "__main__":
sentry_init()
logging.basicConfig(filename='decision_maker_standalone.log',
level=logging.INFO)
dm = DecisionMaker()
dm()
b IDATxytVսϓ22 A@IR :hCiZ[v*E:WũZA ^dQeQ @ !jZ'>gsV仿$|?g)&x-E IENT ; @x T.i%-X} SvS5.r/ UHz^_$-W" w)Ɗ/ @Z &IoX P$K} JzX:;` &, ŋui,e6mX
Եr Kb1ԗ) D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A ݀!I*]R;I2$eZ#ORZSrr 6mteffu*((Pu'v{ DIߔ4^pIm '77WEEE;vƎ 4- $]'RI{\I&G :IHJ
DWBB=\WRm
o$K(V 9 ABB.}jѢv `^?IOȅ} ڶmG }T#FJ`5 6$- ھ}FI&v;0(h; Б 38CӧOWf!; A
i:F _m9s&|q%=# wZprrrla
A & P\\СC[A#! { olF}
`E2} MK/vV )i{ 4BffV\|ۭX `b@kɶ@%i$K z5zhmX [IXZ ` 'b%$r5 M4º /l
ԃ ߖ xhʔ)[@=}
K6IM}^ 5k㏷݆z
ΗÿO:gdGBmyT/@+Vɶ纽zl.yit뭷zV 0[Y^>Wsqs}\/@$(T7f.Inݺi R$푔n.~?H))\Z RW'Mo~v
Ov6oԃ xz! S,&xm/yɞԟ?'uaSѽb, 8GלKboi&3 t7Y,)JJ c[nzӳdE&KsZLӄ I ?@ & %ӟ۶mSMMњ0 iؐSZ, |J+N
~,0A 0!5%Q-YQQa 3 }$_vVr f9f?S 8` z D A D A D A D A D A D A D A D A D A dqP,تmMmg 1V?rSI꒟]u|l
RCyEf٢9 jURbztѰ!m5~tGj2DhG*{H9)꒟ר3:(+3\?/;TUݭʴ~S6lڧUJ*i$d(#=Yݺd{,p|3B)) q:vN0Y.jkק6;SɶVzHJJЀ-utѹսk>QUU \~]fFnK?&ߡ5b=z9)^|u_k-[ y%ZNU6 7Mi:]ۦ tk[n
X(e6 Bb."8cۭ|~teuuw|ήI-5"~Uk;ZicEmN/:]M>
cQ^uiƞ??Ң pc# TUU3UakNwA`:Y_V-8.KKfRi tv*
9S6ֿj,ՃNOMߤ]z^fOh|<>@Å5 _/Iu?{SY4hK/2]4%it5q]GGe2%iR|
W&f*^]??vq[LgE_3f}Fxu~}qd-ږFxu~I N >\;͗O֊:̗WJ@ Bh W=y|GgwܷH_NY?)Tdi'?խwhlmQi !SUUsw4kӺe4rfxu-[nHtMFj}H_u~w>)oV}(T'ebʒv3_[+vn @Ȭ\S}ot}w=k HFnxg S 0eޢm~l}uqZfF oZuuEg
`zt~? b;t%>WTkķh[ 2eG8LIWx,^\thrl^Ϊ{=dž<}qV@ ⠨Wy^LF_>0UkD uʫuCs$)I v:IK; 6ֲ4{^6 եm+l 3>݆uM 9u ?>Zc }g~qhKwڭ e FMM~pМuqǿz6Tb@8 @Y|jx](^]gf}M"tG
-w.@ vOqh~/HII` S[l.6nØXL9vU cOoB\xoǤ'T&I ǍQw_wpv[kmO{w~>#=P1Pɞa-we:iǏlHo꒟f9SzH?+shk%Fs:q VhqY `jvO'ρ?PyX3lх]˾uV{ݞ]1,MzYNW~̈́ joYn}ȚF߾mS]F z+EDxm/d{F{-W-4wY듏:??_gPf
^3ecg ҵs8R2מz@TA NGj)}CNi/R~}c:5{! ZHӋӾ6}T] G]7W6^n
9*,YqOZj :P?Q D FL|?-^.Ɵ7}fFhxe2Ps cz1&5\cn[=Vn[ĶE鎀uˌd3GII k;lNmشOuuRVfBE]ۣe Ӷu
:X-[(er4~LHi6:Ѻ@ԅr ST0 trk%$Č0 ez" *z " T/X9|8.C5Feg} C Q%͞ˣJvL/? j^ h & 9xF`њZ( & yF&Iݻfg #W;3^{Wo^4'v V[[K';+mӍִ] AC @ W?1^{එyh +^]fm~iԵ] AB @WTk̏t uR?l.OIHiYyԶ] Aˀ 7c:q}ힽaf6Z~қm(+sK4{^6}T *UUu]n.:kx{:2 _m=sAߤU@? Z-Vކе z왍Nэ{|5 pڶn b
p-@sPg]0G7fy-M{GCF'%{4` =$-Ge\ eU:m+Zt'WjO!OAF @ik&t݆ϥ_
e}=]" Wz_.͜E3leW Fih|t-wZۍ-uw=6 YN {6|} | *={Ѽn.S .z1z jۻTH]흾 D uDvmvK .` V]yY~sI@t?/ϓ. m&[ " +P?MzovVЫG3- G RR[(!!\_,^%?v@ ҵő
m`Y)tem8GM x.))A]Y i`ViW`?^~! S#^+ѽGZj?Vģ 0.))AlzL*]OXrY `DBBLOj{-MH'ii-ϰ ok7^
)쭡b]UXSְmռY|5 *cֽk0B7镹%ڽP# 8nȎq}mJr23_>lE5$iwui+ H~F`IjƵ@q
\ @#qG 0" . 0" l ` . 0! , AQHN6qzkKJ#o;`Xv2>,tێJJ7Z/*A .@fفjMzkg @TvZH3Zxu6Ra'%O?/d Q5x YkU]Rֽkق@ DaS^RSּ5|BeHNN͘p
HvcYcC5:y
#`οb; z2 .!kr}gUWkyZn=f Pvsn3p~;4p˚=ē~NmI] ¾ 0lH[_L hs h_ ғߤc_њe c) g 7VIZ5 yrgk̞W#IjӪv>՞y睝M8[ |]\շ 8M 6%|@P ZڨI-m>=k= 'aiRo-x?>Q. }`Ȏ:Wsmu u >
.@,&;+!!˱tﭧD QwRW\vF\~Q7>sp Yw$%A~;~}6 ¾ g &if_=j,v+U L1(tW ake:@Ș>j$Gq2t7S?vL|]u/ .(0 E 6Mk6hiۺzښOrifޱxm/Gx> Lal%%~{lBsR4* }{0Z/ tNIɚpV ^# Lf:u@k#RSu
=S^ZyuR/.@n&z~B=0eg뺆#,Þ[ B/?H uUf7y
Wy}Bwegל`Wh(||`l`.;Ws?V@" c:iɍL֯PGv6zctM̠':wuW;d=;EveD}9J@ B(0iհ bvP1{\P&G7DIy_$-Q jm~Yrr&]CDv%bh|Yzni _ R ;kg}nJOIIwy uL}{ЌNj}:+3Y?:WJ/N+Rzd=hb;dj͒suݔ@NKMԄ jqzC5 @y°hL m;*5ezᕏ=ep
XL n?מ:r`۵tŤZ|1v`V뽧_csج'ߤ%oTuumk%%%h)uy]Nk[n
'b2 l.=͜E%gf$[c;s:V-͞WߤWh-j7]4=F-X]>ZLSi[Y*We; Zan(Ӈ W|e(HNNP5[=
r4tP
&0< pc# `vTNV
GFqvTi * Tyam$ߏWyE* VJKMTfFw >'$-ؽ.Ho.8c"@ D A D A D A D A D A D A D A D A D A~j*֘, N;Pi3599h=g oض L giJ5փy~}&Zd9 p֚
e:|hL``b/d9 p? fgg+%%hMg Xosج, ΩOl0Zh=x djL mhݻo O [g_l, 8a]٭+ӧ0 $I] c]:粹:Teꢢ"5a^Kgh,&= =՟ ^߶ߢE ܹS J}I%:8
IDAT~,9/ʃ PW'Mo }zNƍ쨓zPb NZ~^z=4mswg;5 Y~SVMR XUյڱRf?s:w
; 6H:º i 5 -maM &O3; 1IKeam Zh͛7+##v+c ~u~ca] GnF'ټL~ PPPbn
v oC 4R,ӟgg %hq} @#M4IÇ Oy^xM Zx
) yO w @HkN˖-Sǎ mb]X@n+i͖ !++K3gd\$mt$^ YfJ \8PRF )77Wא!Cl $i: @@_oG I{$# 8磌ŋ9 1A (Im7 ֭>}ߴJq7ޗt^ -[ԩSj*}%]&'
- ɓ'ꫯVzzvB#; a
7@GxI{jƌ.LÇWBB7 `O"I$ /@R @ eee@۷ >}0 ,ɒ2$53Xs|cS~r pTYYY} k Hc %&k.], @ A D A D A D A D A D A D A D A D A @lT<%''*Lo^={رc5h %$+CnܸQ3fҥK}vUVVs9G
R,_{ xˇ3 o߾;TTTd}馛]uuuG~iԩ@4 bnvmvfϞ /Peeeq}}z a
I~,誫{UWW뮻}_~YƍSMMMYχ ֝waw \ ď cxꩧtEƍկ_?۷5@u ?1kNׯWzz/wy>}zj3 k (ٺu q_Zvf̘:~ AB Q&r| ! %KҥKg Ԟ={<_X-z ! CyFUUz~ AB QIIIjݺW $ UXXDٳZ~ AB QƍecW $<(~<RSSvZujjjԧO ZQu@4 8m&&&jԩg $ ď 1h ͟?_{768@ g
=@ `)))5o6m 3 ) ѣƌJ;wҿUTT /KZR{~a=@ 0o<*狔iFɶ[ˎ ;T]] OX@ ?K.ۈxN p p p p p p p p p p p p p p p p pP fl߾] ,{ｧk۶mڿo5 BTӦMӴiӴ|r D B2e |An!D y'tkΝ[A $***t5'
" !駟oaD nΝ:t֭[gD ШQ0 6qD;@ x M6v( PiizmZ 4ew" @ ̴ixf [~-F ٱc&IZ 2|n!?$ @{[HTɏ #@h Ȏ I# _m (F /6Z3 z'\r, r!;w2Z3j=~G Y 7"I$i I. p_" ? p N` y D D ?: _ G ÿa b7 J ! Bx @0 Bo
cG @`1C [ @0G
@`0C _ u V1 aC X > W ` | `! < S `" <. `# c `? cA C 4
? c p# ~ @0 ?: 0 8& _ M Q1 J h# ? /` 7; I q 7 a w Q A 1 H p
! #< 8/# @1Ul7 =S=K.4Z?E _$i@ ! 1!E 4? ` P_ @B ă 1 0# : " aU,xb F Y1
[n |n
#' v EH:`x b
# v D 4 Y hi.i& EΖv# O H 4IŶ}:Ik h @tZRF # (tXҙzZ ? I3l7 q@õ|ۍ 1,GpuY Ꮿ@ hJv# x xk$
v# 9 5 }_$ c S#=+"K {F*m7`# %H:NRS p6I?sIՖ{Ap$I$I:QRv2$Z @UJ*$]<FO4 IENDB`