Merge pull request #320 from dalf/currency

[enh] currency_convert engine : "1 dollars in euros"

searx/engines/currency_convert.py

@@ -1,11 +1,37 @@
 from datetime import datetime
 import re
+import os
+import json
+import unicodedata
+
 
 categories = []
 url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
 weight = 100
 
-parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)  # noqa
+parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W*in?\W*([^\.]+)\W*$', re.I)  # noqa
+
+db = 1
+
+
+def normalize_name(name):
+    name = name.lower().replace('-', ' ')
+    name = re.sub(' +', ' ', name)
+    return unicodedata.normalize('NFKD', u"" + name).lower()
+
+
+def name_to_iso4217(name):
+    global db
+
+    name = normalize_name(name)
+    currencies = db['names'].get(name, [name])
+    return currencies[0]
+
+
+def iso4217_to_name(iso4217, language):
+    global db
+
+    return db['iso4217'].get(iso4217, {}).get(language, iso4217)
 
 
 def request(query, params):
@@ -16,6 +42,8 @@ def request(query, params):
 
     ammount, from_currency, to_currency = m.groups()
     ammount = float(ammount)
+    from_currency = name_to_iso4217(from_currency.strip())
+    to_currency = name_to_iso4217(to_currency.strip())
 
     q = (from_currency + to_currency).upper()
 
@@ -23,6 +51,8 @@ def request(query, params):
     params['ammount'] = ammount
     params['from'] = from_currency
     params['to'] = to_currency
+    params['from_name'] = iso4217_to_name(from_currency, 'en')
+    params['to_name'] = iso4217_to_name(to_currency, 'en')
 
     return params
 
@@ -35,12 +65,14 @@ def response(resp):
     except:
         return results
 
-    answer = '{0} {1} = {2} {3} (1 {1} = {4} {3})'.format(
+    answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(
         resp.search_params['ammount'],
         resp.search_params['from'],
         resp.search_params['ammount'] * conversion_rate,
         resp.search_params['to'],
-        conversion_rate
+        conversion_rate,
+        resp.search_params['from_name'],
+        resp.search_params['to_name'],
     )
 
     now_date = datetime.now().strftime('%Y%m%d')
@@ -55,3 +87,15 @@ def response(resp):
     results.append({'answer': answer, 'url': url})
 
     return results
+
+
+def load():
+    global db
+
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    json_data = open(current_dir + "/../data/currencies.json").read()
+
+    db = json.loads(json_data)
+
+
+load()

searx/tests/engines/test_currency_convert.py

@@ -27,9 +27,11 @@ class TestCurrencyConvertEngine(SearxTestCase):
 
     def test_response(self):
         dicto = defaultdict(dict)
-        dicto['ammount'] = 10
+        dicto['ammount'] = float(10)
         dicto['from'] = "EUR"
         dicto['to'] = "USD"
+        dicto['from_name'] = "euro"
+        dicto['to_name'] = "United States dollar"
         response = mock.Mock(text='a,b,c,d', search_params=dicto)
         self.assertEqual(currency_convert.response(response), [])
 
@@ -38,7 +40,7 @@ class TestCurrencyConvertEngine(SearxTestCase):
         results = currency_convert.response(response)
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0]['answer'], '10 EUR = 5.0 USD (1 EUR = 0.5 USD)')
+        self.assertEqual(results[0]['answer'], '10.0 EUR = 5.0 USD, 1 EUR (euro) = 0.5 USD (United States dollar)')
         now_date = datetime.now().strftime('%Y%m%d')
         self.assertEqual(results[0]['url'], 'https://finance.yahoo.com/currency/converter-results/' +
-                                            now_date + '/10-eur-to-usd.html')
+                                            now_date + '/10.0-eur-to-usd.html')

utils/fetch_currencies.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+import unicodedata
+import string
+from urllib import urlencode
+from requests import get
+ 
+languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}
+ 
+url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)
+url_wmflabs_template = 'http://wdq.wmflabs.org/api?q=' 
+url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
+ 
+wmflabs_queries = [ 
+    'CLAIM[31:8142]', # all devise
+]
+ 
+db = {
+    'iso4217' : {
+        },
+    'names' : {
+        }
+}
+
+
+def remove_accents(data):
+    return unicodedata.normalize('NFKD', data).lower()
+        
+
+def normalize_name(name):
+    return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' '))
+
+
+def add_currency_name(name, iso4217):
+    global db
+
+    db_names = db['names']
+
+
+    if not isinstance(iso4217, basestring):
+        print "problem", name, iso4217
+        return
+
+    name = normalize_name(name)
+
+    if name == '':
+        print "name empty", iso4217
+        return
+
+    iso4217_set = db_names.get(name, None)
+    if iso4217_set is not None and iso4217 not in iso4217_set:
+        db_names[name].append(iso4217)
+    else:
+        db_names[name] = [ iso4217 ]
+
+
+def add_currency_label(label, iso4217, language):
+    global db
+
+    db['iso4217'][iso4217] = db['iso4217'].get(iso4217, {})
+    db['iso4217'][iso4217][language] = label
+
+
+def get_property_value(data, name):
+    prop = data.get('claims', {}).get(name, {})
+    if len(prop) == 0:
+        return None
+    
+    value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '') 
+    if value == '':
+        return None
+
+    return value
+    
+
+def parse_currency(data):
+    iso4217 = get_property_value(data, 'P498')
+        
+    if iso4217 is not None:
+        unit = get_property_value(data, 'P558')
+        if unit is not None:
+            add_currency_name(unit, iso4217)
+                
+        labels = data.get('labels', {})
+        for language in languages:
+            name = labels.get(language, {}).get('value', None)
+            if name != None:
+                add_currency_name(name, iso4217)
+                add_currency_label(name, iso4217, language)
+
+        aliases = data.get('aliases', {})
+        for language in aliases:
+            for i in range(0, len(aliases[language])):
+                alias = aliases[language][i].get('value', None)
+                add_currency_name(alias, iso4217)
+
+ 
+def fetch_data(wikidata_ids):
+    url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)}))
+    htmlresponse = get(url)
+    jsonresponse = json.loads(htmlresponse.content)
+    entities = jsonresponse.get('entities', {})
+ 
+    for pname in entities:
+        pvalue = entities.get(pname)
+        parse_currency(pvalue)
+ 
+ 
+def add_q(i):
+    return "Q" + str(i)
+ 
+ 
+def fetch_data_batch(wikidata_ids):
+    while len(wikidata_ids) > 0:
+        if len(wikidata_ids) > 50:
+            fetch_data(wikidata_ids[0:49])
+            wikidata_ids = wikidata_ids[50:]
+        else:
+            fetch_data(wikidata_ids)
+            wikidata_ids = []
+    
+ 
+def wdq_query(query):
+    url = url_wmflabs_template + query
+    htmlresponse = get(url)
+    jsonresponse = json.loads(htmlresponse.content)
+    qlist = map(add_q, jsonresponse.get('items', {}))
+    error = jsonresponse.get('status', {}).get('error', None)
+    if error != None and error != 'OK':
+        print "error for query '" + query + "' :" + error
+
+    fetch_data_batch(qlist)
+ 
+ 
+def wd_query(query, offset=0):
+    qlist = []
+ 
+    url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))
+    htmlresponse = get(url)
+    jsonresponse = json.loads(htmlresponse.content)
+    for r in jsonresponse.get('query', {}).get('search', {}):
+        qlist.append(r.get('title', ''))
+    fetch_data_batch(qlist)
+ 
+## fetch ##
+for q in wmflabs_queries:
+    wdq_query(q)
+
+# static 
+add_currency_name(u"euro", 'EUR')
+add_currency_name(u"euros", 'EUR')
+add_currency_name(u"dollar", 'USD')
+add_currency_name(u"dollars", 'USD')
+add_currency_name(u"peso", 'MXN')
+add_currency_name(u"pesos", 'MXN')
+
+# write
+f = open("currencies.json", "wb")
+json.dump(db, f, indent=4, encoding="utf-8")
+f.close()