Beanz/searxng
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #85 from searxng/fix-metrics-offline-engine

Browse Source 
Fix metrics offline engine
This commit is contained in:
Markus Heiser 2021-05-22 13:34:44 +00:00 committed by GitHub
commit 2398e9a1fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 86 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
searx/metrics/__init__.py
View File

@ -156,57 +156,76 @@ def get_reliabilities(engline_name_list, checker_results):
return reliabilities
def round_or_none(number, digits):
return round(number, digits) if number else number
def get_engines_stats(engine_name_list):
assert counter_storage is not None
assert histogram_storage is not None
list_time = []
max_time_total = max_result_count = None
max_time_total = max_result_count = None # noqa
for engine_name in engine_name_list:
sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
if sent_count == 0:
continue
result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
if successful_count and result_count_sum:
score = counter('engine', engine_name, 'score') # noqa
score_per_result = score / float(result_count_sum)
else:
score = score_per_result = 0.0
max_time_total = max(time_total or 0, max_time_total or 0)
max_result_count = max(result_count or 0, max_result_count or 0)
list_time.append({
stats = {
'name': engine_name,
'total': round_or_none(time_total, 1),
'total_p80': round_or_none(time_total_p80, 1),
'total_p95': round_or_none(time_total_p95, 1),
'http': round_or_none(time_http, 1),
'http_p80': round_or_none(time_http_p80, 1),
'http_p95': round_or_none(time_http_p95, 1),
'processing': round(time_total - time_http, 1) if time_total else None,
'processing_p80': round(time_total_p80 - time_http_p80, 1) if time_total else None,
'processing_p95': round(time_total_p95 - time_http_p95, 1) if time_total else None,
'score': score,
'score_per_result': score_per_result,
'total': None,
'total_p80': None,
'total_p95': None,
'http': None,
'http_p80': None,
'http_p95': None,
'processing': None,
'processing_p80': None,
'processing_p95': None,
'score': 0,
'score_per_result': 0,
'result_count': result_count,
})
}
if successful_count and result_count_sum:
score = counter('engine', engine_name, 'score')
stats['score'] = score
stats['score_per_result'] = score / float(result_count_sum)
time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
time_http_p80 = time_http_p95 = 0
if time_http is not None:
time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
stats['http'] = round(time_http, 1)
stats['http_p80'] = round(time_http_p80, 1)
stats['http_p95'] = round(time_http_p95, 1)
if time_total is not None:
time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
stats['total'] = round(time_total, 1)
stats['total_p80'] = round(time_total_p80, 1)
stats['total_p95'] = round(time_total_p95, 1)
stats['processing'] = round(time_total - (time_http or 0), 1)
stats['processing_p80'] = round(time_total_p80 - time_http_p80, 1)
stats['processing_p95'] = round(time_total_p95 - time_http_p95, 1)
list_time.append(stats)
return {
'time': list_time,
'max_time': math.ceil(max_time_total or 0),

12
searx/network/__init__.py
View File

@ -44,7 +44,8 @@ def reset_time_for_thread():
def get_time_for_thread():
return THREADLOCAL.total_time
"""returns thread's total time or None"""
return THREADLOCAL.__dict__.get('total_time')
def set_timeout_for_thread(timeout, start_time=None):
@ -57,10 +58,11 @@ def set_context_network_name(network_name):
def get_context_network():
try:
return THREADLOCAL.network
except AttributeError:
return get_network()
"""If set return thread's network.
If unset, return value from :py:obj:`get_network`.
"""
return THREADLOCAL.__dict__.get('network') or get_network()
def request(method, url, **kwargs):

7
searx/results.py
View File

@ -371,11 +371,12 @@ class ResultContainer:
self.unresponsive_engines.add((engine_name, error_type, error_message, suspended))
def add_timing(self, engine_name, engine_time, page_load_time):
self.timings.append({
timing = {
'engine': engines[engine_name].shortcut,
'total': engine_time,
'load': page_load_time
})
'load': page_load_time,
}
self.timings.append(timing)
def get_timings(self):
return self.timings

6
searx/templates/oscar/preferences.html
View File

@ -25,11 +25,13 @@
<td class="{{ label }}" style="padding: 2px">{{- "" -}}
{%- if stats[engine_name].time != None -%}
<span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
{%- if max_rate95 is not none and max_rate95 > 0 -%}
<span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
<span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
<span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
<span class="stacked-bar-chart-rate100"></span>{{- "" -}}
<span class="stacked-bar-chart-rate100"></span>
{%- endif -%}
</span>{{- "" -}}
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
<p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}

12
searx/templates/oscar/stats.html
View File

@ -52,11 +52,11 @@
{%- endif -%}
</td>
<td class="response-time">
{%- if engine_stat.total -%}
{%- if engine_stat.total is not none -%}
<div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
<div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">{{- "" -}}
<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
{% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
{% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
</div>
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
<table class="table table-striped">
@ -69,19 +69,19 @@
<tr>
<th scope="col">{{ _('Median') }}</th>
<td>{{ engine_stat.total }}</td>
<td>{{ engine_stat.http }}</td>
<td>{{ engine_stat.http or '' }}</td>
<td>{{ engine_stat.processing }}</td>
</tr>
<tr>
<th scope="col">{{ _('P80') }}</th>
<td>{{ engine_stat.total_p80 }}</td>
<td>{{ engine_stat.http_p80 }}</td>
<td>{{ engine_stat.http_p80 or '' }}</td>
<td>{{ engine_stat.processing_p80 }}</td>
</tr>
<tr>
<th scope="col">{{ _('P95') }}</th>
<td>{{ engine_stat.total_p95 }}</td>
<td>{{ engine_stat.http_p95 }}</td>
<td>{{ engine_stat.http_p95 or '' }}</td>
<td>{{ engine_stat.processing_p95 }}</td>
</tr>
</table>

6
searx/templates/simple/preferences.html
View File

@ -39,11 +39,13 @@
<td class="{{ label }}" style="padding: 2px; width: 13rem;">{{- "" -}}
{%- if stats[engine_name].time != None -%}
<span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
{%- if max_rate95 is not none and max_rate95 > 0 -%}
<span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
<span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
<span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
<span class="stacked-bar-chart-rate100"></span>{{- "" -}}
<span class="stacked-bar-chart-rate100"></span>
{%- endif -%}
</span>{{- "" -}}
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
<p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}

13
searx/templates/simple/stats.html
View File

@ -52,12 +52,11 @@
{%- endif -%}
</td>
<td class="response-time">
{%- if engine_stat.total -%}
{%- if engine_stat.total is not none -%}
<div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
<div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">
<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
{% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
{% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
</div>
<div class="engine-tooltip" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
<table>
@ -70,19 +69,19 @@
<tr>
<th scope="col">{{ _('Median') }}</th>
<td>{{ engine_stat.total }}</td>
<td>{{ engine_stat.http }}</td>
<td>{{ engine_stat.http or ''}}</td>
<td>{{ engine_stat.processing }}</td>
</tr>
<tr>
<th scope="col">{{ _('P80') }}</th>
<td>{{ engine_stat.total_p80 }}</td>
<td>{{ engine_stat.http_p80 }}</td>
<td>{{ engine_stat.http_p80 or '' }}</td>
<td>{{ engine_stat.processing_p80 }}</td>
</tr>
<tr>
<th scope="col">{{ _('P95') }}</th>
<td>{{ engine_stat.total_p95 }}</td>
<td>{{ engine_stat.http_p95 }}</td>
<td>{{ engine_stat.http_p95 or '' }}</td>
<td>{{ engine_stat.processing_p95 }}</td>
</tr>
</table>

8
searx/webapp.py
View File

@ -568,7 +568,7 @@ def post_request(response):
timings_total = ['total_' + str(i) + '_' + v['engine'] +
';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]
timings_load = ['load_' + str(i) + '_' + v['engine'] +
';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]
';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings) if v.get('load')]
timings_all = timings_all + timings_total + timings_load
response.headers.add('Server-Timing', ', '.join(timings_all))
return response
@ -923,9 +923,9 @@ def preferences():
result_count = int(result_count_sum / float(successful_count)) if successful_count else 0
stats[e.name] = {
'time': median if median else None,
'rate80': rate80 if rate80 else None,
'rate95': rate95 if rate95 else None,
'time': median,
'rate80': rate80,
'rate95': rate95,
'warn_timeout': e.timeout > settings['outgoing']['request_timeout'],
'supports_selected_language': _is_selected_language_supported(e, request.preferences),
'result_count': result_count,