Add engine locales (2/n)

searx/data/engine_locales.json

@@ -1,5 +1,6 @@
 {
   "qwant": {
+    "all_locale": null,
     "languages": {},
     "regions": {
       "bg-BG": "bg_BG",
@@ -44,6 +45,7 @@
     }
   },
   "qwant images": {
+    "all_locale": null,
     "languages": {},
     "regions": {
       "bg-BG": "bg_BG",
@@ -88,6 +90,7 @@
     }
   },
   "qwant news": {
+    "all_locale": null,
     "languages": {},
     "regions": {
       "ca-ES": "ca_ES",
@@ -117,6 +120,7 @@
     }
   },
   "qwant videos": {
+    "all_locale": null,
     "languages": {},
     "regions": {
       "bg-BG": "bg_BG",
@@ -161,6 +165,7 @@
     }
   },
   "startpage": {
+    "all_locale": null,
     "languages": {
       "af": "afrikaans",
       "am": "amharic",
@@ -176,7 +181,7 @@
       "da": "dansk",
       "de": "deutsch",
       "el": "greek",
-      "en": "english_uk",
+      "en": "english",
       "eo": "esperanto",
       "es": "espanol",
       "et": "estonian",

searx/engines/__init__.py

@@ -18,7 +18,7 @@ from typing import Dict, List, Optional, Any
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
-from searx import logger, settings
+from searx import logger, settings, locales
 from searx.data import ENGINES_LANGUAGES, ENGINES_LOCALES
 from searx.utils import load_module, match_language
 
@@ -72,6 +72,73 @@ class EngineLocales:
        }
     """
 
+    all_locale: Optional[str] = None
+    """To which locale value SearXNG's ``all`` language is mapped (shown a "Default
+    language").
+    """
+
+    def get_language(self, searxng_locale: str, default: Optional[str] = None):
+        """Return engine's language string that *best fits* to SearXNG's locale.
+        :param searxng_locale: SearXNG's internal representation of locale
+          selected by the user.
+        :param default: engine's default language
+        The *best fits* rules are implemented in
+        :py:obj:`locales.get_engine_locale`.  Except for the special value ``all``
+        which is determined from :py:obj`EngineTraits.all_language`.
+        """
+        if searxng_locale == 'all' and self.all_locale is not None:
+            return self.all_locale
+        sxng_lang = searxng_locale.split('-')[0]
+        return locales.get_engine_locale(sxng_lang, self.languages, default=default)
+
+    def get_region(self, searxng_locale: str, default: Optional[str] = None):
+        """Return engine's region string that best fits to SearXNG's locale.
+        :param searxng_locale: SearXNG's internal representation of locale
+          selected by the user.
+        :param default: engine's default region
+        The *best fits* rules are implemented in
+        :py:obj:`locales.get_engine_locale`.  Except for the special value ``all``
+        which is determined from :py:obj`EngineTraits.all_language`.
+        """
+        if searxng_locale == 'all' and self.all_locale is not None:
+            return self.all_locale
+        return locales.get_engine_locale(searxng_locale, self.regions, default=default)
+
+    def is_locale_supported(self, searxng_locale: str) -> bool:
+        """A *locale* (SearXNG's internal representation) is considered to be supported
+        by the engine if the *region* or the *language* is supported by the
+        engine.  For verification the functions :py:func:`self.get_region` and
+        :py:func:`self.get_region` are used.
+        """
+        return bool(self.get_region(searxng_locale) or self.get_language(searxng_locale))
+
+    @classmethod
+    def load(
+        cls, engine_locales_key: str, language: Optional[str] = None, region: Optional[str] = None
+    ) -> "EngineLocales":
+        #
+        engine_locales_value = {**ENGINES_LOCALES[engine_locales_key]}
+
+        _msg = "settings.yml - engine: '%s' / %s: '%s' not supported"
+
+        if language is not None:
+            el_languages = engine_locales_value['languages']
+            if language not in el_languages:
+                raise ValueError(_msg % (engine_locales_key, 'language', language))
+            engine_locales_value['languages'] = {language: el_languages[language]}
+
+        if region is not None:
+            el_regions = engine_locales_value['regions']
+            if region in el_regions:
+                raise ValueError(_msg % (engine_locales_key, 'region', region))
+            engine_locales_value['regions'] = {region: el_regions[region]}
+
+        return cls(**engine_locales_value)
+
+    @classmethod
+    def exists(cls, engine_locales_key: str):
+        return engine_locales_key in ENGINES_LOCALES
+
 
 class Engine:  # pylint: disable=too-few-public-methods
     """This class is currently never initialized and only used for type hinting."""
@@ -210,9 +277,9 @@ def update_engine_attributes(engine: Engine, engine_setting: Dict[str, Any]):
 def set_engine_locales(engine: Engine):
     engine_locales_key = None
 
-    if engine.name in ENGINES_LOCALES:
+    if EngineLocales.exists(engine.name):
         engine_locales_key = engine.name
-    elif engine.engine in ENGINES_LOCALES:
+    elif EngineLocales.exists(engine.engine):
         # The key of the dictionary engine_data_dict is the *engine name*
         # configured in settings.xml.  When multiple engines are configured in
         # settings.yml to use the same origin engine (python module) these
@@ -222,9 +289,13 @@ def set_engine_locales(engine: Engine):
     else:
         return False
 
-    print(engine.name, ENGINES_LOCALES[engine_locales_key])
+    #
-    engine.engine_locales = EngineLocales(**ENGINES_LOCALES[engine_locales_key])
+    engine.engine_locales = EngineLocales.load(
+        engine_locales_key, getattr(engine, 'language', None), getattr(engine, 'region', None)
+    )
+
     # language_support
+    # NOTE: actually the value should be true, or the entry in engine_locales.json should not exists.
     engine.language_support = len(engine.engine_locales.regions) > 0 or len(engine.engine_locales.languages) > 0
     return True
 

searx/engines/qwant.py

@@ -34,7 +34,6 @@ import babel
 
 from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
-from searx.locales import get_engine_locale
 
 # about
 about = {
@@ -95,7 +94,7 @@ def request(query, params):
     )
 
     # add quant's locale
-    q_locale = get_engine_locale(params['language'], engine_locales.regions, default='en_US')
+    q_locale = engine_locales.get_region(params['language'], 'en_US')
     params['url'] += '&locale=' + q_locale
 
     # add safesearch option

searx/engines/startpage.py

@@ -17,14 +17,12 @@ from lxml import html
 import babel
 
 from searx.network import get
-from searx.locales import get_engine_locale
 from searx.utils import extract_text, eval_xpath
 from searx.exceptions import (
     SearxEngineResponseException,
     SearxEngineCaptchaException,
 )
 
-
 # about
 about = {
     "website": 'https://startpage.com',
@@ -116,10 +114,8 @@ def request(query, params):
     engine_region = 'all'
     engine_language = 'english_uk'
     if params['language'] != 'all':
-        engine_region = get_engine_locale(params['language'], engine_locales.regions, default='all')
+        engine_region = engine_locales.get_region(params['language'], 'all')
-        engine_language = get_engine_locale(
+        engine_language = engine_locales.get_language(params['language'], 'english_uk')
-            params['language'].split('-')[0], engine_locales.languages, default='english_uk'
-        )
     logger.debug(
         'selected language %s --> engine_language: %s // engine_region: %s',
         params['language'],
@@ -375,8 +371,14 @@ def _fetch_engine_locales(resp, engine_locales):
         }
     )
 
+    skip_eng_tags = {
+        'english_uk',  # SearXNG lang 'en' already maps to 'english'
+    }
+
     for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'):
         engine_lang = option.get('value')
+        if engine_lang in skip_eng_tags:
+            continue
         name = extract_text(option).lower()
 
         lang_code = catalog_engine2code.get(engine_lang)

searxng_extra/update/update_engine_locales.py

@@ -109,7 +109,6 @@ def fetch_engine_locales() -> Tuple[EngineLocalesDict, EngineLanguageDict]:
                 % (engine_name, len(engine_data.languages), len(engine_data.regions))
             )
         elif fetch_languages is not None:
-            print(engine_name)
             resp = network.get(engine.supported_languages_url, headers=headers)  # type: ignore
             engines_languages[engine_name] = fetch_languages(resp)
             print(
@@ -425,6 +424,7 @@ def write_engine_data(file_name, engine_data_dict: EngineLocalesDict):
         engine_name: {
             'regions': engine_data.regions,
             'languages': engine_data.languages,
+            'all_locale': engine_data.all_locale,
         }
         for engine_name, engine_data in engine_data_dict.items()
     }