[fix] wikidata: increase wikidata queries timeout

The big queries for initializing and updating the currencies take longer than
the default of the wikidata engine, which is only 3sec.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -75,7 +75,7 @@ test.shell:
 # wrap ./manage script
 
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.traits data.useragents data.locales
+MANAGE += data.all data.traits data.useragents data.locales data.currencies
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx
 MANAGE += gecko.driver

searx/data/currencies.json

@@ -1557,7 +1557,7 @@
             "ar": "بيزو دومنيكاني",
             "bg": "Доминиканско песо",
             "ca": "peso dominicà",
-            "cs": "Dominikánské peso",
+            "cs": "dominikánské peso",
             "cy": "Peso Dominica",
             "de": "dominikanischer Peso",
             "en": "Dominican peso",
@@ -1582,7 +1582,7 @@
             "pt": "peso dominicano",
             "ro": "peso dominican",
             "ru": "доминиканское песо",
-            "sk": "Dominikánske peso",
+            "sk": "dominikánske peso",
             "sl": "dominikanski peso",
             "sr": "доминикански пезос",
             "sv": "Dominikansk peso",
@@ -1809,7 +1809,7 @@
             "fi": "Fidžin dollari",
             "fr": "dollar des Fidji",
             "ga": "dollar Fhidsí",
-            "gl": "Dólar fidxiano",
+            "gl": "Dólar fijiano",
             "he": "דולר פיג'י",
             "hr": "Fidžijski dolar",
             "hu": "fidzsi dollár",
@@ -1921,7 +1921,7 @@
             "ar": "لاري جورجي",
             "bg": "Грузинско лари",
             "ca": "lari",
-            "cs": "gruzínské lari",
+            "cs": "gruzínský lari",
             "da": "Lari",
             "de": "georgischer Lari",
             "en": "Georgian lari",
@@ -3308,7 +3308,7 @@
             "ar": "درهم مغربي",
             "bg": "Марокански дирхам",
             "ca": "dírham marroquí",
-            "cs": "Marocký dirham",
+            "cs": "marocký dirham",
             "cy": "Dirham Moroco",
             "de": "Marokkanischer Dirham",
             "en": "Moroccan dirham",
@@ -3776,6 +3776,7 @@
             "en": "Malaysian ringgit",
             "eo": "malajzia ringito",
             "es": "ringgit",
+            "et": "Ringgit",
             "eu": "Ringgit",
             "fi": "Malesian ringgit",
             "fr": "ringgit",
@@ -3885,12 +3886,13 @@
             "bg": "Нигерийска найра",
             "ca": "naira",
             "cs": "nigerijská naira",
+            "cy": "naira",
             "de": "Naira",
             "en": "naira",
             "eo": "niĝeria najro",
             "es": "naira",
             "fi": "Nigerian naira",
-            "fr": "Naira",
+            "fr": "naira",
             "ga": "naira",
             "gl": "Naira",
             "he": "נאירה",
@@ -4412,7 +4414,7 @@
             "ar": "غواراني باراغواي",
             "ca": "guaraní",
             "cs": "paraguayský guaraní",
-            "cy": "Gwarani Paragwâi",
+            "cy": "Gwarani Paragwái",
             "de": "Paraguayischer Guaraní",
             "en": "Paraguayan guaraní",
             "eo": "paragvaja gvaranio",
@@ -6284,6 +6286,7 @@
             "es": "Oro de Zimbabue",
             "fi": "Zimbabwen kulta",
             "fr": "or du Zimbabwe",
+            "hr": "zimbabveanski gold",
             "id": "Zimbabwe Gold",
             "it": "Zimbabwe Gold",
             "ja": "ジンバブエ・ゴールド",
@@ -6459,7 +6462,6 @@
         "NT$": "TWD",
         "NZ$": "NZD",
         "Nfk": "ERN",
-        "Noha heinen krasss": "EUR",
         "Nu": "BTN",
         "N₨": "NPR",
         "P": "BWP",
@@ -8146,7 +8148,6 @@
         "dólar dos barbados": "BBD",
         "dólar dos estados unidos": "USD",
         "dólar estadounidense": "USD",
-        "dólar fidxiano": "FJD",
         "dólar fijiano": "FJD",
         "dólar fixiano": "FJD",
         "dólar fiyiano": "FJD",
@@ -8832,7 +8833,7 @@
         "gvinejski franak": "GNF",
         "gvinejski frank": "GNF",
         "gvinėjos frankas": "GNF",
-        "gwarani paragwâi": "PYG",
+        "gwarani paragwái": "PYG",
         "gy$": "GYD",
         "gyd": "GYD",
         "ĝibraltara pundo": "GIP",
@@ -10186,6 +10187,8 @@
         "naira nigeriana": "NGN",
         "naira nigeriano": "NGN",
         "naira nigérian": "NGN",
+        "naïra": "NGN",
+        "naïra nigérian": "NGN",
         "najro": "NGN",
         "nakfa": "ERN",
         "nakfa eritrea": "ERN",
@@ -12442,6 +12445,7 @@
         "zimbabva dolaro": "ZWL",
         "zimbabva oro": "ZWG",
         "zimbabve altını": "ZWG",
+        "zimbabveanski gold": "ZWG",
         "zimbabvejski dolar": "ZWL",
         "zimbabvejski gold": "ZWG",
         "zimbabwaanse dollar": "ZWL",
@@ -13060,6 +13064,7 @@
         "джибутийский франк": "DJF",
         "джибутски франк": "DJF",
         "дин": "RSD",
+        "дин.": "RSD",
         "динар": [
             "TND",
             "RSD",
@@ -14304,7 +14309,9 @@
         "الروبل": "RUB",
         "الروبل الروسي": "RUB",
         "الروبية الهندية": "INR",
+        "الريال السعودي": "SAR",
         "الشيكل الإسرائيلي الجديد": "ILS",
+        "العملة السعودية": "SAR",
         "العملة الموريتانية": "MRU",
         "الفرنك الجيبوتي": "DJF",
         "الفرنك السويسري": "CHF",
@@ -14465,6 +14472,8 @@
         "روفيه": "MVR",
         "روفيه مالديفية": "MVR",
         "ريال": "YER",
+        "ريال السعودية": "SAR",
+        "ريال المملكة العربية السعودية": "SAR",
         "ريال إيراني": "IRR",
         "ريال برازيلي": "BRL",
         "ريال سعودي": "SAR",
@@ -14473,6 +14482,7 @@
         "ريال قطري": "QAR",
         "ريال كمبودي": "KHR",
         "ريال يمني": "YER",
+        "ريالات": "SAR",
         "رينغيت": "MYR",
         "رينغيت ماليزي": "MYR",
         "ریال": [
@@ -14525,6 +14535,8 @@
         "شيلينغ تنزاني": "TZS",
         "شيلينغ كيني": "KES",
         "شيلينغ كينيي": "KES",
+        "عملة السعودية": "SAR",
+        "عملة المملكة العربية السعودية": "SAR",
         "غواراني": "PYG",
         "غواراني باراغواي": "PYG",
         "غيلدر الأنتيل الهولندية": "ANG",
@@ -15778,7 +15790,6 @@
         "香港元": "HKD",
         "﷼": [
             "YER",
-            "SAR",
             "IRR"
         ],
         "﷼'": "YER"

searx/engines/wikidata.py

@@ -158,13 +158,13 @@ def get_label_for_entity(entity_id, language):
     return name
 
 
-def send_wikidata_query(query, method='GET'):
+def send_wikidata_query(query, method='GET', **kwargs):
     if method == 'GET':
         # query will be cached by wikidata
-        http_response = get(SPARQL_ENDPOINT_URL + '?' + urlencode({'query': query}), headers=get_headers())
+        http_response = get(SPARQL_ENDPOINT_URL + '?' + urlencode({'query': query}), headers=get_headers(), **kwargs)
     else:
         # query won't be cached by wikidata
-        http_response = post(SPARQL_ENDPOINT_URL, data={'query': query}, headers=get_headers())
+        http_response = post(SPARQL_ENDPOINT_URL, data={'query': query}, headers=get_headers(), **kwargs)
     if http_response.status_code != 200:
         logger.debug('SPARQL endpoint error %s', http_response.content.decode())
     logger.debug('request time %s', str(http_response.elapsed))
@@ -808,7 +808,7 @@ def init(engine_settings=None):  # pylint: disable=unused-argument
             if attribute.name not in WIKIDATA_PROPERTIES:
                 wikidata_property_names.append("wd:" + attribute.name)
     query = QUERY_PROPERTY_NAMES.replace('%ATTRIBUTES%', " ".join(wikidata_property_names))
-    jsonresponse = send_wikidata_query(query)
+    jsonresponse = send_wikidata_query(query, timeout=20)
     for result in jsonresponse.get('results', {}).get('bindings', {}):
         name = result['name']['value']
         lang = result['name']['xml:lang']

searxng_extra/update/update_currencies.py

@@ -92,8 +92,7 @@ def add_currency_label(db, label, iso4217, language):
 
 
 def wikidata_request_result_iterator(request):
-    wikidata.timeout = 30  # github CI has longer timeouts
+    result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
-    result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
     if result is not None:
         yield from result['results']['bindings']
 

utils/lib_sxng_data.sh

@@ -8,6 +8,7 @@ data.:
   traits    : update searx/data/engine_traits.json & searx/sxng_locales.py
   useragents: update searx/data/useragents.json with the most recent versions of Firefox
   locales   : update searx/data/locales.json from babel
+  currencies: update searx/data/currencies.json from wikidata
 EOF
 }
 
@@ -59,3 +60,12 @@ data.locales() {
     )
     dump_return $?
 }
+
+data.currencies(){
+    (   set -e
+        pyenv.activate
+        build_msg DATA "update searx/data/currencies.json"
+        python searxng_extra/update/update_currencies.py
+    )
+    dump_return $?
+}