diff --git a/searx/data/engine_locales.json b/searx/data/engine_locales.json index fba8af3d3..9d18d4227 100644 --- a/searx/data/engine_locales.json +++ b/searx/data/engine_locales.json @@ -1,5 +1,6 @@ { "qwant": { + "all_locale": null, "languages": {}, "regions": { "bg-BG": "bg_BG", @@ -44,6 +45,7 @@ } }, "qwant images": { + "all_locale": null, "languages": {}, "regions": { "bg-BG": "bg_BG", @@ -88,6 +90,7 @@ } }, "qwant news": { + "all_locale": null, "languages": {}, "regions": { "ca-ES": "ca_ES", @@ -117,6 +120,7 @@ } }, "qwant videos": { + "all_locale": null, "languages": {}, "regions": { "bg-BG": "bg_BG", @@ -161,6 +165,7 @@ } }, "startpage": { + "all_locale": null, "languages": { "af": "afrikaans", "am": "amharic", @@ -176,7 +181,7 @@ "da": "dansk", "de": "deutsch", "el": "greek", - "en": "english_uk", + "en": "english", "eo": "esperanto", "es": "espanol", "et": "estonian", diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 17f67db1a..d3092e2c1 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -18,7 +18,7 @@ from typing import Dict, List, Optional, Any from os.path import realpath, dirname from babel.localedata import locale_identifiers -from searx import logger, settings +from searx import logger, settings, locales from searx.data import ENGINES_LANGUAGES, ENGINES_LOCALES from searx.utils import load_module, match_language @@ -72,6 +72,73 @@ class EngineLocales: } """ + all_locale: Optional[str] = None + """To which locale value SearXNG's ``all`` language is mapped (shown a "Default + language"). + """ + + def get_language(self, searxng_locale: str, default: Optional[str] = None): + """Return engine's language string that *best fits* to SearXNG's locale. + :param searxng_locale: SearXNG's internal representation of locale + selected by the user. + :param default: engine's default language + The *best fits* rules are implemented in + :py:obj:`locales.get_engine_locale`. Except for the special value ``all`` + which is determined from :py:obj`EngineTraits.all_language`. + """ + if searxng_locale == 'all' and self.all_locale is not None: + return self.all_locale + sxng_lang = searxng_locale.split('-')[0] + return locales.get_engine_locale(sxng_lang, self.languages, default=default) + + def get_region(self, searxng_locale: str, default: Optional[str] = None): + """Return engine's region string that best fits to SearXNG's locale. + :param searxng_locale: SearXNG's internal representation of locale + selected by the user. + :param default: engine's default region + The *best fits* rules are implemented in + :py:obj:`locales.get_engine_locale`. Except for the special value ``all`` + which is determined from :py:obj`EngineTraits.all_language`. + """ + if searxng_locale == 'all' and self.all_locale is not None: + return self.all_locale + return locales.get_engine_locale(searxng_locale, self.regions, default=default) + + def is_locale_supported(self, searxng_locale: str) -> bool: + """A *locale* (SearXNG's internal representation) is considered to be supported + by the engine if the *region* or the *language* is supported by the + engine. For verification the functions :py:func:`self.get_region` and + :py:func:`self.get_region` are used. + """ + return bool(self.get_region(searxng_locale) or self.get_language(searxng_locale)) + + @classmethod + def load( + cls, engine_locales_key: str, language: Optional[str] = None, region: Optional[str] = None + ) -> "EngineLocales": + # + engine_locales_value = {**ENGINES_LOCALES[engine_locales_key]} + + _msg = "settings.yml - engine: '%s' / %s: '%s' not supported" + + if language is not None: + el_languages = engine_locales_value['languages'] + if language not in el_languages: + raise ValueError(_msg % (engine_locales_key, 'language', language)) + engine_locales_value['languages'] = {language: el_languages[language]} + + if region is not None: + el_regions = engine_locales_value['regions'] + if region in el_regions: + raise ValueError(_msg % (engine_locales_key, 'region', region)) + engine_locales_value['regions'] = {region: el_regions[region]} + + return cls(**engine_locales_value) + + @classmethod + def exists(cls, engine_locales_key: str): + return engine_locales_key in ENGINES_LOCALES + class Engine: # pylint: disable=too-few-public-methods """This class is currently never initialized and only used for type hinting.""" @@ -210,9 +277,9 @@ def update_engine_attributes(engine: Engine, engine_setting: Dict[str, Any]): def set_engine_locales(engine: Engine): engine_locales_key = None - if engine.name in ENGINES_LOCALES: + if EngineLocales.exists(engine.name): engine_locales_key = engine.name - elif engine.engine in ENGINES_LOCALES: + elif EngineLocales.exists(engine.engine): # The key of the dictionary engine_data_dict is the *engine name* # configured in settings.xml. When multiple engines are configured in # settings.yml to use the same origin engine (python module) these @@ -222,9 +289,13 @@ def set_engine_locales(engine: Engine): else: return False - print(engine.name, ENGINES_LOCALES[engine_locales_key]) - engine.engine_locales = EngineLocales(**ENGINES_LOCALES[engine_locales_key]) + # + engine.engine_locales = EngineLocales.load( + engine_locales_key, getattr(engine, 'language', None), getattr(engine, 'region', None) + ) + # language_support + # NOTE: actually the value should be true, or the entry in engine_locales.json should not exists. engine.language_support = len(engine.engine_locales.regions) > 0 or len(engine.engine_locales.languages) > 0 return True diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py index 18256ec5a..075078abe 100644 --- a/searx/engines/qwant.py +++ b/searx/engines/qwant.py @@ -34,7 +34,6 @@ import babel from searx.exceptions import SearxEngineAPIException from searx.network import raise_for_httperror -from searx.locales import get_engine_locale # about about = { @@ -95,7 +94,7 @@ def request(query, params): ) # add quant's locale - q_locale = get_engine_locale(params['language'], engine_locales.regions, default='en_US') + q_locale = engine_locales.get_region(params['language'], 'en_US') params['url'] += '&locale=' + q_locale # add safesearch option diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 739a36b56..50a9b20f1 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -17,14 +17,12 @@ from lxml import html import babel from searx.network import get -from searx.locales import get_engine_locale from searx.utils import extract_text, eval_xpath from searx.exceptions import ( SearxEngineResponseException, SearxEngineCaptchaException, ) - # about about = { "website": 'https://startpage.com', @@ -116,10 +114,8 @@ def request(query, params): engine_region = 'all' engine_language = 'english_uk' if params['language'] != 'all': - engine_region = get_engine_locale(params['language'], engine_locales.regions, default='all') - engine_language = get_engine_locale( - params['language'].split('-')[0], engine_locales.languages, default='english_uk' - ) + engine_region = engine_locales.get_region(params['language'], 'all') + engine_language = engine_locales.get_language(params['language'], 'english_uk') logger.debug( 'selected language %s --> engine_language: %s // engine_region: %s', params['language'], @@ -375,8 +371,14 @@ def _fetch_engine_locales(resp, engine_locales): } ) + skip_eng_tags = { + 'english_uk', # SearXNG lang 'en' already maps to 'english' + } + for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'): engine_lang = option.get('value') + if engine_lang in skip_eng_tags: + continue name = extract_text(option).lower() lang_code = catalog_engine2code.get(engine_lang) diff --git a/searxng_extra/update/update_engine_locales.py b/searxng_extra/update/update_engine_locales.py index ccc0a0237..9b33bde5c 100755 --- a/searxng_extra/update/update_engine_locales.py +++ b/searxng_extra/update/update_engine_locales.py @@ -109,7 +109,6 @@ def fetch_engine_locales() -> Tuple[EngineLocalesDict, EngineLanguageDict]: % (engine_name, len(engine_data.languages), len(engine_data.regions)) ) elif fetch_languages is not None: - print(engine_name) resp = network.get(engine.supported_languages_url, headers=headers) # type: ignore engines_languages[engine_name] = fetch_languages(resp) print( @@ -425,6 +424,7 @@ def write_engine_data(file_name, engine_data_dict: EngineLocalesDict): engine_name: { 'regions': engine_data.regions, 'languages': engine_data.languages, + 'all_locale': engine_data.all_locale, } for engine_name, engine_data in engine_data_dict.items() }