add comments to google-engines
This commit is contained in:
		
							parent
							
								
									cdf74fe563
								
							
						
					
					
						commit
						144f89bf78
					
				| @ -1,37 +1,57 @@ | |||||||
| #!/usr/bin/env python | ## Google (Web) | ||||||
|  | #  | ||||||
|  | # @website     https://www.google.com | ||||||
|  | # @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! | ||||||
|  | #  | ||||||
|  | # @using-api   yes | ||||||
|  | # @results     JSON | ||||||
|  | # @stable      yes (but deprecated) | ||||||
|  | # @parse       url, title, content | ||||||
| 
 | 
 | ||||||
| from urllib import urlencode | from urllib import urlencode | ||||||
| from json import loads | from json import loads | ||||||
| 
 | 
 | ||||||
|  | # engine dependent config | ||||||
| categories = ['general'] | categories = ['general'] | ||||||
| 
 |  | ||||||
| url = 'https://ajax.googleapis.com/' |  | ||||||
| search_url = url + 'ajax/services/search/web?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}'  # noqa |  | ||||||
| 
 |  | ||||||
| paging = True | paging = True | ||||||
| language_support = True | language_support = True | ||||||
| 
 | 
 | ||||||
|  | # search-url | ||||||
|  | url = 'https://ajax.googleapis.com/' | ||||||
|  | search_url = url + 'ajax/services/search/web?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}'  # noqa | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
|  | # do search-request | ||||||
| def request(query, params): | def request(query, params): | ||||||
|     offset = (params['pageno'] - 1) * 8 |     offset = (params['pageno'] - 1) * 8 | ||||||
|  | 
 | ||||||
|     language = 'en-US' |     language = 'en-US' | ||||||
|     if params['language'] != 'all': |     if params['language'] != 'all': | ||||||
|         language = params['language'].replace('_', '-') |         language = params['language'].replace('_', '-') | ||||||
|  | 
 | ||||||
|     params['url'] = search_url.format(offset=offset, |     params['url'] = search_url.format(offset=offset, | ||||||
|                                       query=urlencode({'q': query}), |                                       query=urlencode({'q': query}), | ||||||
|                                       language=language) |                                       language=language) | ||||||
|  | 
 | ||||||
|     return params |     return params | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | # get response from search-request | ||||||
| def response(resp): | def response(resp): | ||||||
|     results = [] |     results = [] | ||||||
|  | 
 | ||||||
|     search_res = loads(resp.text) |     search_res = loads(resp.text) | ||||||
| 
 | 
 | ||||||
|  |     # return empty array if there are no results | ||||||
|     if not search_res.get('responseData', {}).get('results'): |     if not search_res.get('responseData', {}).get('results'): | ||||||
|         return [] |         return [] | ||||||
| 
 | 
 | ||||||
|  |     # parse results | ||||||
|     for result in search_res['responseData']['results']: |     for result in search_res['responseData']['results']: | ||||||
|  |         # append result | ||||||
|         results.append({'url': result['unescapedUrl'], |         results.append({'url': result['unescapedUrl'], | ||||||
|                         'title': result['titleNoFormatting'], |                         'title': result['titleNoFormatting'], | ||||||
|                         'content': result['content']}) |                         'content': result['content']}) | ||||||
|  | 
 | ||||||
|  |     # return results | ||||||
|     return results |     return results | ||||||
|  | |||||||
| @ -1,37 +1,58 @@ | |||||||
| #!/usr/bin/env python | ## Google (Images) | ||||||
|  | #  | ||||||
|  | # @website     https://www.google.com | ||||||
|  | # @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! | ||||||
|  | #  | ||||||
|  | # @using-api   yes | ||||||
|  | # @results     JSON | ||||||
|  | # @stable      yes (but deprecated) | ||||||
|  | # @parse       url, title, img_src | ||||||
| 
 | 
 | ||||||
| from urllib import urlencode | from urllib import urlencode | ||||||
| from json import loads | from json import loads | ||||||
| 
 | 
 | ||||||
|  | # engine dependent config | ||||||
| categories = ['images'] | categories = ['images'] | ||||||
|  | paging = True | ||||||
| 
 | 
 | ||||||
|  | # search-url | ||||||
| url = 'https://ajax.googleapis.com/' | url = 'https://ajax.googleapis.com/' | ||||||
| search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}'  # noqa | search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}'  # noqa | ||||||
| 
 | 
 | ||||||
| paging = True |  | ||||||
| 
 | 
 | ||||||
|  | # do search-request | ||||||
| def request(query, params): | def request(query, params): | ||||||
|     offset = (params['pageno'] - 1) * 8 |     offset = (params['pageno'] - 1) * 8 | ||||||
|  | 
 | ||||||
|     params['url'] = search_url.format(query=urlencode({'q': query}), |     params['url'] = search_url.format(query=urlencode({'q': query}), | ||||||
|                                       offset=offset) |                                       offset=offset) | ||||||
|  | 
 | ||||||
|     return params |     return params | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | # get response from search-request | ||||||
| def response(resp): | def response(resp): | ||||||
|     results = [] |     results = [] | ||||||
|  | 
 | ||||||
|     search_res = loads(resp.text) |     search_res = loads(resp.text) | ||||||
|     if not search_res.get('responseData'): | 
 | ||||||
|         return [] |     # return empty array if there are no results | ||||||
|     if not search_res['responseData'].get('results'): |     if not search_res.get('responseData', {}).get('results'): | ||||||
|         return [] |         return [] | ||||||
|  | 
 | ||||||
|  |     # parse results | ||||||
|     for result in search_res['responseData']['results']: |     for result in search_res['responseData']['results']: | ||||||
|         href = result['originalContextUrl'] |         href = result['originalContextUrl'] | ||||||
|         title = result['title'] |         title = result['title'] | ||||||
|         if not result['url']: |         if not result['url']: | ||||||
|             continue |             continue | ||||||
|  | 
 | ||||||
|  |         # append result | ||||||
|         results.append({'url': href, |         results.append({'url': href, | ||||||
|                         'title': title, |                         'title': title, | ||||||
|                         'content': '', |                         'content': '', | ||||||
|                         'img_src': result['url'], |                         'img_src': result['url'], | ||||||
|                         'template': 'images.html'}) |                         'template': 'images.html'}) | ||||||
|  | 
 | ||||||
|  |     # return results | ||||||
|     return results |     return results | ||||||
|  | |||||||
| @ -1,43 +1,62 @@ | |||||||
| #!/usr/bin/env python | ## Google (News) | ||||||
|  | #  | ||||||
|  | # @website     https://www.google.com | ||||||
|  | # @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! | ||||||
|  | #  | ||||||
|  | # @using-api   yes | ||||||
|  | # @results     JSON | ||||||
|  | # @stable      yes (but deprecated) | ||||||
|  | # @parse       url, title, content, publishedDate | ||||||
| 
 | 
 | ||||||
| from urllib import urlencode | from urllib import urlencode | ||||||
| from json import loads | from json import loads | ||||||
| from dateutil import parser | from dateutil import parser | ||||||
| 
 | 
 | ||||||
|  | # search-url | ||||||
| categories = ['news'] | categories = ['news'] | ||||||
| 
 |  | ||||||
| url = 'https://ajax.googleapis.com/' |  | ||||||
| search_url = url + 'ajax/services/search/news?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}'  # noqa |  | ||||||
| 
 |  | ||||||
| paging = True | paging = True | ||||||
| language_support = True | language_support = True | ||||||
| 
 | 
 | ||||||
|  | # engine dependent config | ||||||
|  | url = 'https://ajax.googleapis.com/' | ||||||
|  | search_url = url + 'ajax/services/search/news?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}'  # noqa | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
|  | # do search-request | ||||||
| def request(query, params): | def request(query, params): | ||||||
|     offset = (params['pageno'] - 1) * 8 |     offset = (params['pageno'] - 1) * 8 | ||||||
|  | 
 | ||||||
|     language = 'en-US' |     language = 'en-US' | ||||||
|     if params['language'] != 'all': |     if params['language'] != 'all': | ||||||
|         language = params['language'].replace('_', '-') |         language = params['language'].replace('_', '-') | ||||||
|  | 
 | ||||||
|     params['url'] = search_url.format(offset=offset, |     params['url'] = search_url.format(offset=offset, | ||||||
|                                       query=urlencode({'q': query}), |                                       query=urlencode({'q': query}), | ||||||
|                                       language=language) |                                       language=language) | ||||||
|  | 
 | ||||||
|     return params |     return params | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | # get response from search-request | ||||||
| def response(resp): | def response(resp): | ||||||
|     results = [] |     results = [] | ||||||
|  | 
 | ||||||
|     search_res = loads(resp.text) |     search_res = loads(resp.text) | ||||||
| 
 | 
 | ||||||
|  |     # return empty array if there are no results | ||||||
|     if not search_res.get('responseData', {}).get('results'): |     if not search_res.get('responseData', {}).get('results'): | ||||||
|         return [] |         return [] | ||||||
| 
 | 
 | ||||||
|  |     # parse results | ||||||
|     for result in search_res['responseData']['results']: |     for result in search_res['responseData']['results']: | ||||||
| 
 |         # parse publishedDate | ||||||
| # Mon, 10 Mar 2014 16:26:15 -0700 |  | ||||||
|         publishedDate = parser.parse(result['publishedDate']) |         publishedDate = parser.parse(result['publishedDate']) | ||||||
| 
 | 
 | ||||||
|  |         # append result | ||||||
|         results.append({'url': result['unescapedUrl'], |         results.append({'url': result['unescapedUrl'], | ||||||
|                         'title': result['titleNoFormatting'], |                         'title': result['titleNoFormatting'], | ||||||
|                         'publishedDate': publishedDate, |                         'publishedDate': publishedDate, | ||||||
|                         'content': result['content']}) |                         'content': result['content']}) | ||||||
|  | 
 | ||||||
|  |     # return results | ||||||
|     return results |     return results | ||||||
|  | |||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user
	 Thomas Pointhuber
						Thomas Pointhuber