Beanz/searxng
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[fix] merge infoboxes based on weight

Browse Source 
also minor changes in attributes and images from wikidata
This commit is contained in:
marc 2016-06-27 23:35:43 -05:00
parent c2e4014287
commit ad58b14be7
4 changed files with 50 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
searx/engines/wikidata.py
View File

@ -35,7 +35,7 @@ url_detail = wikidata_api\
url_map = 'https://www.openstreetmap.org/'\ url_map = 'https://www.openstreetmap.org/'\
+ '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M' + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500' url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
# xpaths # xpaths
wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title' wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title'
@ -162,6 +162,7 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# INFOBOX ATTRIBUTES (ROWS) # INFOBOX ATTRIBUTES (ROWS)
# DATES
# inception date # inception date
add_attribute(attributes, result, 'P571', date=True) add_attribute(attributes, result, 'P571', date=True)
# dissolution date # dissolution date
@ -170,11 +171,14 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
add_attribute(attributes, result, 'P580', date=True) add_attribute(attributes, result, 'P580', date=True)
# end date # end date
add_attribute(attributes, result, 'P582', date=True) add_attribute(attributes, result, 'P582', date=True)
# date of birth # date of birth
add_attribute(attributes, result, 'P569', date=True) add_attribute(attributes, result, 'P569', date=True)
# date of death # date of death
add_attribute(attributes, result, 'P570', date=True) add_attribute(attributes, result, 'P570', date=True)
# date of spacecraft launch
add_attribute(attributes, result, 'P619', date=True)
# date of spacecraft landing
add_attribute(attributes, result, 'P620', date=True)
# nationality # nationality
add_attribute(attributes, result, 'P27') add_attribute(attributes, result, 'P27')
@ -201,7 +205,7 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# area # area
add_attribute(attributes, result, 'P2046') add_attribute(attributes, result, 'P2046')
# currency # currency
add_attribute(attributes, result, 'P38') add_attribute(attributes, result, 'P38', trim=True)
# heigth (building) # heigth (building)
add_attribute(attributes, result, 'P2048') add_attribute(attributes, result, 'P2048')
@ -230,6 +234,10 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
add_attribute(attributes, result, 'P264') add_attribute(attributes, result, 'P264')
# publisher # publisher
add_attribute(attributes, result, 'P123') add_attribute(attributes, result, 'P123')
# original network
add_attribute(attributes, result, 'P449')
# distributor
add_attribute(attributes, result, 'P750')
# composer # composer
add_attribute(attributes, result, 'P86') add_attribute(attributes, result, 'P86')
# publication date # publication date
@ -266,6 +274,10 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
add_attribute(attributes, result, 'P112') add_attribute(attributes, result, 'P112')
# legal form (company/organization) # legal form (company/organization)
add_attribute(attributes, result, 'P1454') add_attribute(attributes, result, 'P1454')
# operator
add_attribute(attributes, result, 'P137')
# crew members (tripulation)
add_attribute(attributes, result, 'P1029')
# taxon # taxon
add_attribute(attributes, result, 'P225') add_attribute(attributes, result, 'P225')
# chemical formula # chemical formula
@ -300,8 +312,8 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
# only returns first match # only returns first match
def add_image(result): def add_image(result):
# P18: image, P154: logo, P242: map, P41: flag, P2716: collage, P2910: icon # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
property_ids = ['P18', 'P154', 'P242', 'P41', 'P2716', 'P2910'] property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
for property_id in property_ids: for property_id in property_ids:
image = result.xpath(property_xpath.replace('{propertyid}', property_id)) image = result.xpath(property_xpath.replace('{propertyid}', property_id))
@ -320,6 +332,7 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals
label = default_label label = default_label
else: else:
label = extract_text(attribute[0].xpath(label_xpath)) label = extract_text(attribute[0].xpath(label_xpath))
label = label[0].upper() + label[1:]
if date: if date:
trim = True trim = True
@ -369,6 +382,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
dom_element = dom_element[0] dom_element = dom_element[0]
if not default_label: if not default_label:
label = extract_text(dom_element.xpath(label_xpath)) label = extract_text(dom_element.xpath(label_xpath))
label = label[0].upper() + label[1:]
if link_type == 'geo': if link_type == 'geo':
links.append(get_geolink(dom_element)) links.append(get_geolink(dom_element))

18
searx/results.py
View File

@ -43,6 +43,19 @@ def compare_urls(url_a, url_b):
def merge_two_infoboxes(infobox1, infobox2): def merge_two_infoboxes(infobox1, infobox2):
# get engines weights
if hasattr(engines[infobox1['engine']], 'weight'):
weight1 = engines[infobox1['engine']].weight
else:
weight1 = 1
if hasattr(engines[infobox2['engine']], 'weight'):
weight2 = engines[infobox2['engine']].weight
else:
weight2 = 1
if weight2 > weight1:
infobox1['engine'] = infobox2['engine']
if 'urls' in infobox2: if 'urls' in infobox2:
urls1 = infobox1.get('urls', None) urls1 = infobox1.get('urls', None)
if urls1 is None: if urls1 is None:
@ -64,6 +77,8 @@ def merge_two_infoboxes(infobox1, infobox2):
img2 = infobox2.get('img_src') img2 = infobox2.get('img_src')
if img1 is None: if img1 is None:
infobox1['img_src'] = img2 infobox1['img_src'] = img2
elif weight2 > weight1:
infobox1['img_src'] = img2
if 'attributes' in infobox2: if 'attributes' in infobox2:
attributes1 = infobox1.get('attributes', None) attributes1 = infobox1.get('attributes', None)
@ -77,7 +92,8 @@ def merge_two_infoboxes(infobox1, infobox2):
attributeSet.add(attribute.get('label', None)) attributeSet.add(attribute.get('label', None))
for attribute in infobox2.get('attributes', []): for attribute in infobox2.get('attributes', []):
attributes1.append(attribute) if attribute.get('label', None) not in attributeSet:
attributes1.append(attribute)
if 'content' in infobox2: if 'content' in infobox2:
content1 = infobox1.get('content', None) content1 = infobox1.get('content', None)

2
searx/settings.yml
View File

@ -105,6 +105,7 @@ engines:
- name : ddg definitions - name : ddg definitions
engine : duckduckgo_definitions engine : duckduckgo_definitions
shortcut : ddd shortcut : ddd
weight : 2
disabled : True disabled : True
- name : digg - name : digg
@ -127,6 +128,7 @@ engines:
- name : wikidata - name : wikidata
engine : wikidata engine : wikidata
shortcut : wd shortcut : wd
weight : 2
- name : duckduckgo - name : duckduckgo
engine : duckduckgo engine : duckduckgo

22
tests/unit/engines/test_wikidata.py
View File

@ -95,14 +95,14 @@ class TestWikidataEngine(SearxTestCase):
results = wikidata.getDetail(response, "Q123", "yua", "yua_MX") results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")
self.assertEqual(len(results), 2) self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'official website') self.assertEqual(results[0]['title'], 'Official website')
self.assertEqual(results[0]['url'], 'https://officialsite.com') self.assertEqual(results[0]['url'], 'https://officialsite.com')
self.assertEqual(results[1]['infobox'], 'Test') self.assertEqual(results[1]['infobox'], 'Test')
self.assertEqual(results[1]['id'], None) self.assertEqual(results[1]['id'], None)
self.assertEqual(results[1]['content'], 'Description') self.assertEqual(results[1]['content'], 'Description')
self.assertEqual(results[1]['attributes'], []) self.assertEqual(results[1]['attributes'], [])
self.assertEqual(results[1]['urls'][0]['title'], 'official website') self.assertEqual(results[1]['urls'][0]['title'], 'Official website')
self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com') self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')
self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)') self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')
self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test') self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')
@ -141,7 +141,8 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
image_src = wikidata.add_image(html_etree) image_src = wikidata.add_image(html_etree)
self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500") self.assertEqual(image_src,
"https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
html = u""" html = u"""
<div> <div>
@ -196,7 +197,8 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
image_src = wikidata.add_image(html_etree) image_src = wikidata.add_image(html_etree)
self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500") self.assertEqual(image_src,
"https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
def test_add_attribute(self): def test_add_attribute(self):
html = u""" html = u"""
@ -234,7 +236,7 @@ class TestWikidataEngine(SearxTestCase):
wikidata.add_attribute(attributes, html_etree, "P27") wikidata.add_attribute(attributes, html_etree, "P27")
self.assertEqual(len(attributes), 1) self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "country of citizenship") self.assertEqual(attributes[0]["label"], "Country of citizenship")
self.assertEqual(attributes[0]["value"], "United Kingdom") self.assertEqual(attributes[0]["value"], "United Kingdom")
html = u""" html = u"""
@ -269,7 +271,7 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
wikidata.add_attribute(attributes, html_etree, "P569", date=True) wikidata.add_attribute(attributes, html_etree, "P569", date=True)
self.assertEqual(len(attributes), 1) self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "date of birth") self.assertEqual(attributes[0]["label"], "Date of birth")
self.assertEqual(attributes[0]["value"], "27 January 1832") self.assertEqual(attributes[0]["value"], "27 January 1832")
html = u""" html = u"""
@ -317,7 +319,7 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
wikidata.add_attribute(attributes, html_etree, "P6") wikidata.add_attribute(attributes, html_etree, "P6")
self.assertEqual(len(attributes), 1) self.assertEqual(len(attributes), 1)
self.assertEqual(attributes[0]["label"], "head of government") self.assertEqual(attributes[0]["label"], "Head of government")
self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister") self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
attributes = [] attributes = []
@ -355,7 +357,7 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
wikidata.add_url(urls, html_etree, 'P856') wikidata.add_url(urls, html_etree, 'P856')
self.assertEquals(len(urls), 1) self.assertEquals(len(urls), 1)
self.assertIn({'title': 'official website', 'url': 'https://searx.me/'}, urls) self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
urls = [] urls = []
results = [] results = []
wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results) wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)
@ -403,8 +405,8 @@ class TestWikidataEngine(SearxTestCase):
html_etree = fromstring(html) html_etree = fromstring(html)
wikidata.add_url(urls, html_etree, 'P856') wikidata.add_url(urls, html_etree, 'P856')
self.assertEquals(len(urls), 2) self.assertEquals(len(urls), 2)
self.assertIn({'title': 'official website', 'url': 'http://www.worldofwarcraft.com'}, urls) self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
self.assertIn({'title': 'official website', 'url': 'http://eu.battle.net/wow/en/'}, urls) self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)
def test_get_imdblink(self): def test_get_imdblink(self):
html = u""" html = u"""