Commit d1e6a858 authored by Ozzie Isaacs's avatar Ozzie Isaacs

Parallel requests of metadata provider

parent 10e212fc
...@@ -62,7 +62,7 @@ Please note that running the above install command can fail on some versions of ...@@ -62,7 +62,7 @@ Please note that running the above install command can fail on some versions of
## Requirements ## Requirements
python 3.x+ python 3.5+
Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata: Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata:
......
...@@ -26,7 +26,7 @@ class ComicVine(Metadata): ...@@ -26,7 +26,7 @@ class ComicVine(Metadata):
__name__ = "ComicVine" __name__ = "ComicVine"
__id__ = "comicvine" __id__ = "comicvine"
def search(self, query): def search(self, query, __):
val = list() val = list()
apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6" apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6"
if self.active: if self.active:
......
...@@ -26,7 +26,7 @@ class Google(Metadata): ...@@ -26,7 +26,7 @@ class Google(Metadata):
__name__ = "Google" __name__ = "Google"
__id__ = "google" __id__ = "google"
def search(self, query): def search(self, query, __):
if self.active: if self.active:
val = list() val = list()
result = requests.get("https://www.googleapis.com/books/v1/volumes?q="+query.replace(" ","+")) result = requests.get("https://www.googleapis.com/books/v1/volumes?q="+query.replace(" ","+"))
......
...@@ -17,7 +17,6 @@ ...@@ -17,7 +17,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from scholarly import scholarly from scholarly import scholarly
from flask import url_for
from cps.services.Metadata import Metadata from cps.services.Metadata import Metadata
...@@ -26,7 +25,7 @@ class scholar(Metadata): ...@@ -26,7 +25,7 @@ class scholar(Metadata):
__name__ = "Google Scholar" __name__ = "Google Scholar"
__id__ = "googlescholar" __id__ = "googlescholar"
def search(self, query): def search(self, query, generic_cover=""):
val = list() val = list()
if self.active: if self.active:
scholar_gen = scholarly.search_pubs(' '.join(query.split('+'))) scholar_gen = scholarly.search_pubs(' '.join(query.split('+')))
...@@ -45,7 +44,7 @@ class scholar(Metadata): ...@@ -45,7 +44,7 @@ class scholar(Metadata):
v['tags'] = "" v['tags'] = ""
v['ratings'] = 0 v['ratings'] = 0
v['series'] = "" v['series'] = ""
v['cover'] = url_for('static', filename='generic_cover.jpg') v['cover'] = generic_cover
v['url'] = publication.get('pub_url') or publication.get('eprint_url') or "", v['url'] = publication.get('pub_url') or publication.get('eprint_url') or "",
v['source'] = { v['source'] = {
"id": self.__id__, "id": self.__id__,
......
...@@ -22,8 +22,10 @@ import json ...@@ -22,8 +22,10 @@ import json
import importlib import importlib
import sys import sys
import inspect import inspect
import datetime
import concurrent.futures
from flask import Blueprint, request, Response from flask import Blueprint, request, Response, url_for
from flask_login import current_user from flask_login import current_user
from flask_login import login_required from flask_login import login_required
from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm.attributes import flag_modified
...@@ -32,6 +34,7 @@ from sqlalchemy.exc import OperationalError, InvalidRequestError ...@@ -32,6 +34,7 @@ from sqlalchemy.exc import OperationalError, InvalidRequestError
from . import constants, logger, ub from . import constants, logger, ub
from cps.services.Metadata import Metadata from cps.services.Metadata import Metadata
meta = Blueprint('metadata', __name__) meta = Blueprint('metadata', __name__)
log = logger.create() log = logger.create()
...@@ -101,7 +104,19 @@ def metadata_search(): ...@@ -101,7 +104,19 @@ def metadata_search():
data = list() data = list()
active = current_user.view_settings.get('metadata', {}) active = current_user.view_settings.get('metadata', {})
if query: if query:
for c in cl: a = datetime.datetime.now()
if active.get(c.__id__, True): static_cover = url_for('static', filename='generic_cover.jpg')
data.extend(c.search(query)) with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
meta = {executor.submit(c.search, query, static_cover): c for c in cl if active.get(c.__id__, True)}
for future in concurrent.futures.as_completed(meta):
data.extend(future.result())
b = datetime.datetime.now()
c = b - a
log.info(c.total_seconds())
return Response(json.dumps(data), mimetype='application/json') return Response(json.dumps(data), mimetype='application/json')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment