diff --git a/searx/engines/command.py b/searx/engines/command.py index 176388e3a..29950ae4e 100644 --- a/searx/engines/command.py +++ b/searx/engines/command.py @@ -81,6 +81,7 @@ from subprocess import Popen, PIPE from threading import Thread from searx import logger +from searx.result_types import EngineResults engine_type = 'offline' @@ -93,7 +94,6 @@ query_enum = [] environment_variables = {} working_dir = realpath('.') result_separator = '\n' -result_template = 'key-value.html' timeout = 4.0 _command_logger = logger.getChild('command') @@ -126,17 +126,17 @@ def init(engine_settings): environment_variables = engine_settings['environment_variables'] -def search(query, params): +def search(query, params) -> EngineResults: + res = EngineResults() cmd = _get_command_to_run(query) if not cmd: - return [] + return res - results = [] - reader_thread = Thread(target=_get_results_from_process, args=(results, cmd, params['pageno'])) + reader_thread = Thread(target=_get_results_from_process, args=(res, cmd, params['pageno'])) reader_thread.start() reader_thread.join(timeout=timeout) - return results + return res def _get_command_to_run(query): @@ -153,7 +153,7 @@ def _get_command_to_run(query): return cmd -def _get_results_from_process(results, cmd, pageno): +def _get_results_from_process(res: EngineResults, cmd, pageno): leftover = '' count = 0 start, end = __get_results_limits(pageno) @@ -173,12 +173,11 @@ def _get_results_from_process(results, cmd, pageno): continue if start <= count and count <= end: # pylint: disable=chained-comparison - result['template'] = result_template - results.append(result) + res.add(res.types.KeyValue(kvmap=result)) count += 1 if end < count: - return results + return res line = process.stdout.readline() diff --git a/searx/engines/demo_offline.py b/searx/engines/demo_offline.py index ffcdb46a9..2cef4f0d0 100644 --- a/searx/engines/demo_offline.py +++ b/searx/engines/demo_offline.py @@ -13,6 +13,7 @@ close to the implementation, its just a simple example. To get in use of this """ import json + from searx.result_types import EngineResults engine_type = 'offline' @@ -29,7 +30,7 @@ about = { } # if there is a need for globals, use a leading underline -_my_offline_engine = None +_my_offline_engine: str = "" def init(engine_settings=None): @@ -50,24 +51,28 @@ def init(engine_settings=None): def search(query, request_params) -> EngineResults: - """Query (offline) engine and return results. Assemble the list of results from - your local engine. In this demo engine we ignore the 'query' term, usual - you would pass the 'query' term to your local engine to filter out the + """Query (offline) engine and return results. Assemble the list of results + from your local engine. In this demo engine we ignore the 'query' term, + usual you would pass the 'query' term to your local engine to filter out the results. - """ res = EngineResults() - result_list = json.loads(_my_offline_engine) - - for row in result_list: - entry = { + count = 0 + for row in json.loads(_my_offline_engine): + count += 1 + kvmap = { 'query': query, 'language': request_params['searxng_locale'], 'value': row.get("value"), - # choose a result template or comment out to use the *default* - 'template': 'key-value.html', } - res.append(entry) - + res.add( + res.types.KeyValue( + caption=f"Demo Offline Engine Result #{count}", + key_title="Name", + value_title="Value", + kvmap=kvmap, + ) + ) + res.add(res.types.LegacyResult(number_of_results=count)) return res diff --git a/searx/engines/elasticsearch.py b/searx/engines/elasticsearch.py index c721114a7..6331a992e 100644 --- a/searx/engines/elasticsearch.py +++ b/searx/engines/elasticsearch.py @@ -43,6 +43,8 @@ authentication configured to read from ``my-index`` index. from json import loads, dumps from searx.exceptions import SearxEngineAPIException +from searx.result_types import EngineResults +from searx.extended_types import SXNG_Response base_url = 'http://localhost:9200' @@ -145,23 +147,20 @@ def _custom_query(query): return custom_query -def response(resp): - results = [] +def response(resp: SXNG_Response) -> EngineResults: + res = EngineResults() resp_json = loads(resp.text) if 'error' in resp_json: - raise SearxEngineAPIException(resp_json['error']) - - for result in resp_json['hits']['hits']: - r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()} - r['template'] = 'key-value.html' + raise SearxEngineAPIException(resp_json["error"]) + for result in resp_json["hits"]["hits"]: + kvmap = {key: str(value) if not key.startswith("_") else value for key, value in result["_source"].items()} if show_metadata: - r['metadata'] = {'index': result['_index'], 'id': result['_id'], 'score': result['_score']} + kvmap["metadata"] = {"index": result["_index"], "id": result["_id"], "score": result["_score"]} + res.add(res.types.KeyValue(kvmap=kvmap)) - results.append(r) - - return results + return res _available_query_types = { diff --git a/searx/engines/mariadb_server.py b/searx/engines/mariadb_server.py index 7343f4342..26f537373 100644 --- a/searx/engines/mariadb_server.py +++ b/searx/engines/mariadb_server.py @@ -35,6 +35,8 @@ except ImportError: # the engine pass +from searx.result_types import EngineResults + if TYPE_CHECKING: import logging @@ -63,7 +65,6 @@ query_str = "" limit = 10 paging = True -result_template = 'key-value.html' _connection = None @@ -79,17 +80,16 @@ def init(engine_settings): _connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port) -def search(query, params): +def search(query, params) -> EngineResults: query_params = {'query': query} query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) logger.debug("SQL Query: %s", query_to_run) + res = EngineResults() with _connection.cursor() as cur: cur.execute(query_to_run, query_params) - results = [] col_names = [i[0] for i in cur.description] - for res in cur: - result = dict(zip(col_names, map(str, res))) - result['template'] = result_template - results.append(result) - return results + for row in cur: + kvmap = dict(zip(col_names, map(str, row))) + res.add(res.types.KeyValue(kvmap=kvmap)) + return res diff --git a/searx/engines/meilisearch.py b/searx/engines/meilisearch.py index e22674251..1f6c4d8b2 100644 --- a/searx/engines/meilisearch.py +++ b/searx/engines/meilisearch.py @@ -33,15 +33,15 @@ Here is a simple example to query a Meilisearch instance: # pylint: disable=global-statement -from json import loads, dumps - +from json import dumps +from searx.result_types import EngineResults +from searx.extended_types import SXNG_Response base_url = 'http://localhost:7700' index = '' auth_key = '' facet_filters = [] _search_url = '' -result_template = 'key-value.html' categories = ['general'] paging = True @@ -75,13 +75,12 @@ def request(query, params): return params -def response(resp): - results = [] +def response(resp: SXNG_Response) -> EngineResults: + res = EngineResults() - resp_json = loads(resp.text) - for result in resp_json['hits']: - r = {key: str(value) for key, value in result.items()} - r['template'] = result_template - results.append(r) + resp_json = resp.json() + for row in resp_json['hits']: + kvmap = {key: str(value) for key, value in row.items()} + res.add(res.types.KeyValue(kvmap=kvmap)) - return results + return res diff --git a/searx/engines/mongodb.py b/searx/engines/mongodb.py index 57eaa8537..b6aef3dbd 100644 --- a/searx/engines/mongodb.py +++ b/searx/engines/mongodb.py @@ -37,6 +37,7 @@ Implementations =============== """ +from __future__ import annotations import re @@ -47,6 +48,8 @@ except ImportError: # to use the engine pass +from searx.result_types import EngineResults + engine_type = 'offline' @@ -63,7 +66,6 @@ key = None paging = True results_per_page = 20 exact_match_only = False -result_template = 'key-value.html' _client = None @@ -74,7 +76,7 @@ def init(_): def connect(): global _client # pylint: disable=global-statement - kwargs = {'port': port} + kwargs: dict[str, str | int] = {'port': port} if username: kwargs['username'] = username if password: @@ -82,8 +84,8 @@ def connect(): _client = MongoClient(host, **kwargs)[database][collection] -def search(query, params): - results = [] +def search(query, params) -> EngineResults: + res = EngineResults() if exact_match_only: q = {'$eq': query} else: @@ -92,11 +94,10 @@ def search(query, params): query = _client.find({key: q}).skip((params['pageno'] - 1) * results_per_page).limit(results_per_page) - results.append({'number_of_results': query.count()}) - for r in query: - del r['_id'] - r = {str(k): str(v) for k, v in r.items()} - r['template'] = result_template - results.append(r) + res.add(res.types.LegacyResult(number_of_results=query.count())) + for row in query: + del row['_id'] + kvmap = {str(k): str(v) for k, v in row.items()} + res.add(res.types.KeyValue(kvmap=kvmap)) - return results + return res diff --git a/searx/engines/mysql_server.py b/searx/engines/mysql_server.py index 30d59332e..016fd036f 100644 --- a/searx/engines/mysql_server.py +++ b/searx/engines/mysql_server.py @@ -25,6 +25,8 @@ Implementations """ +from searx.result_types import EngineResults + try: import mysql.connector # type: ignore except ImportError: @@ -55,7 +57,6 @@ query_str = "" limit = 10 paging = True -result_template = 'key-value.html' _connection = None @@ -78,21 +79,15 @@ def init(engine_settings): ) -def search(query, params): +def search(query, params) -> EngineResults: + res = EngineResults() query_params = {'query': query} query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) with _connection.cursor() as cur: cur.execute(query_to_run, query_params) + for row in cur: + kvmap = dict(zip(cur.column_names, map(str, row))) + res.add(res.types.KeyValue(kvmap=kvmap)) - return _fetch_results(cur) - - -def _fetch_results(cur): - results = [] - for res in cur: - result = dict(zip(cur.column_names, map(str, res))) - result['template'] = result_template - results.append(result) - - return results + return res diff --git a/searx/engines/postgresql.py b/searx/engines/postgresql.py index ddd0ef929..78af8783c 100644 --- a/searx/engines/postgresql.py +++ b/searx/engines/postgresql.py @@ -28,6 +28,8 @@ except ImportError: # manually to use the engine. pass +from searx.result_types import EngineResults + engine_type = 'offline' host = "127.0.0.1" @@ -50,7 +52,6 @@ query_str = "" limit = 10 paging = True -result_template = 'key-value.html' _connection = None @@ -72,7 +73,7 @@ def init(engine_settings): ) -def search(query, params): +def search(query, params) -> EngineResults: query_params = {'query': query} query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit) @@ -82,20 +83,16 @@ def search(query, params): return _fetch_results(cur) -def _fetch_results(cur): - results = [] - titles = [] - +def _fetch_results(cur) -> EngineResults: + res = EngineResults() try: titles = [column_desc.name for column_desc in cur.description] - - for res in cur: - result = dict(zip(titles, map(str, res))) - result['template'] = result_template - results.append(result) + for row in cur: + kvmap = dict(zip(titles, map(str, row))) + res.add(res.types.KeyValue(kvmap=kvmap)) # no results to fetch except psycopg2.ProgrammingError: pass - return results + return res diff --git a/searx/engines/redis_server.py b/searx/engines/redis_server.py index 3268378c6..eebb5809b 100644 --- a/searx/engines/redis_server.py +++ b/searx/engines/redis_server.py @@ -36,6 +36,8 @@ Implementations import redis # pylint: disable=import-error +from searx.result_types import EngineResults + engine_type = 'offline' # redis connection variables @@ -46,7 +48,6 @@ db = 0 # engine specific variables paging = False -result_template = 'key-value.html' exact_match_only = True _redis_client = None @@ -63,30 +64,25 @@ def init(_engine_settings): ) -def search(query, _params): +def search(query, _params) -> EngineResults: + res = EngineResults() + if not exact_match_only: - return search_keys(query) + for kvmap in search_keys(query): + res.add(res.types.KeyValue(kvmap=kvmap)) + return res - ret = _redis_client.hgetall(query) - if ret: - ret['template'] = result_template - return [ret] - - if ' ' in query: - qset, rest = query.split(' ', 1) - ret = [] - for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)): - ret.append( - { - res[0]: res[1], - 'template': result_template, - } - ) - return ret - return [] + kvmap: dict[str, str] = _redis_client.hgetall(query) + if kvmap: + res.add(res.types.KeyValue(kvmap=kvmap)) + elif " " in query: + qset, rest = query.split(" ", 1) + for row in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)): + res.add(res.types.KeyValue(kvmap={row[0]: row[1]})) + return res -def search_keys(query): +def search_keys(query) -> list[dict]: ret = [] for key in _redis_client.scan_iter(match='*{}*'.format(query)): key_type = _redis_client.type(key) @@ -98,7 +94,6 @@ def search_keys(query): res = dict(enumerate(_redis_client.lrange(key, 0, -1))) if res: - res['template'] = result_template res['redis_key'] = key ret.append(res) return ret diff --git a/searx/engines/solr.py b/searx/engines/solr.py index 4b80d5729..b23cbe92e 100644 --- a/searx/engines/solr.py +++ b/searx/engines/solr.py @@ -29,9 +29,10 @@ This is an example configuration for searching in the collection # pylint: disable=global-statement -from json import loads from urllib.parse import urlencode from searx.exceptions import SearxEngineAPIException +from searx.result_types import EngineResults +from searx.extended_types import SXNG_Response base_url = 'http://localhost:8983' @@ -72,27 +73,21 @@ def request(query, params): return params -def response(resp): - resp_json = __get_response(resp) - - results = [] - for result in resp_json['response']['docs']: - r = {key: str(value) for key, value in result.items()} - if len(r) == 0: - continue - r['template'] = 'key-value.html' - results.append(r) - - return results - - -def __get_response(resp): +def response(resp: SXNG_Response) -> EngineResults: try: - resp_json = loads(resp.text) + resp_json = resp.json() except Exception as e: raise SearxEngineAPIException("failed to parse response") from e - if 'error' in resp_json: - raise SearxEngineAPIException(resp_json['error']['msg']) + if "error" in resp_json: + raise SearxEngineAPIException(resp_json["error"]["msg"]) - return resp_json + res = EngineResults() + + for result in resp_json["response"]["docs"]: + kvmap = {key: str(value) for key, value in result.items()} + if not kvmap: + continue + res.add(res.types.KeyValue(kvmap=kvmap)) + + return res diff --git a/searx/engines/sqlite.py b/searx/engines/sqlite.py index e9694cf74..e3dd55829 100644 --- a/searx/engines/sqlite.py +++ b/searx/engines/sqlite.py @@ -2,6 +2,14 @@ """SQLite is a small, fast and reliable SQL database engine. It does not require any extra dependency. +Configuration +============= + +The engine has the following (additional) settings: + +- :py:obj:`result_type` + + Example ======= @@ -18,29 +26,32 @@ Query to test: ``!mediathekview concert`` .. code:: yaml - - name: mediathekview - engine: sqlite - disabled: False - categories: general - result_template: default.html - database: searx/data/filmliste-v2.db - query_str: >- - SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, - COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, - description AS content - FROM film - WHERE title LIKE :wildcard OR description LIKE :wildcard - ORDER BY duration DESC + - name: mediathekview + engine: sqlite + shortcut: mediathekview + categories: [general, videos] + result_type: MainResult + database: searx/data/filmliste-v2.db + query_str: >- + SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, + COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, + description AS content + FROM film + WHERE title LIKE :wildcard OR description LIKE :wildcard + ORDER BY duration DESC Implementations =============== """ - +import typing import sqlite3 import contextlib -engine_type = 'offline' +from searx.result_types import EngineResults +from searx.result_types import MainResult, KeyValue + +engine_type = "offline" database = "" """Filename of the SQLite DB.""" @@ -48,9 +59,11 @@ database = "" query_str = "" """SQL query that returns the result items.""" +result_type: typing.Literal["MainResult", "KeyValue"] = "KeyValue" +"""The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`.""" + limit = 10 paging = True -result_template = 'key-value.html' def init(engine_settings): @@ -80,9 +93,8 @@ def sqlite_cursor(): yield cursor -def search(query, params): - results = [] - +def search(query, params) -> EngineResults: + res = EngineResults() query_params = { 'query': query, 'wildcard': r'%' + query.replace(' ', r'%') + r'%', @@ -97,9 +109,11 @@ def search(query, params): col_names = [cn[0] for cn in cur.description] for row in cur.fetchall(): - item = dict(zip(col_names, map(str, row))) - item['template'] = result_template - logger.debug("append result --> %s", item) - results.append(item) + kvmap = dict(zip(col_names, map(str, row))) + if result_type == "MainResult": + item = MainResult(**kvmap) # type: ignore + else: + item = KeyValue(kvmap=kvmap) + res.add(item) - return results + return res diff --git a/searx/settings.yml b/searx/settings.yml index d57d5ea36..22adf6626 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1888,15 +1888,15 @@ engines: # For this demo of the sqlite engine download: # https://liste.mediathekview.de/filmliste-v2.db.bz2 # and unpack into searx/data/filmliste-v2.db - # Query to test: "!demo concert" + # Query to test: "!mediathekview concert" # - # - name: demo + # - name: mediathekview # engine: sqlite - # shortcut: demo - # categories: general - # result_template: default.html + # shortcut: mediathekview + # categories: [general, videos] + # result_type: MainResult # database: searx/data/filmliste-v2.db - # query_str: >- + # query_str: >- # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, # description AS content diff --git a/searx/templates/simple/result_templates/key-value.html b/searx/templates/simple/result_templates/key-value.html deleted file mode 100644 index dcab4377f..000000000 --- a/searx/templates/simple/result_templates/key-value.html +++ /dev/null @@ -1,11 +0,0 @@ -
{{ key|upper }}: {{ value }} | -