Commit 5f0660a4 authored by Ozzieisaacs's avatar Ozzieisaacs

Refactored helper.py and db.py

parent ec3a3a73
......@@ -37,8 +37,6 @@ from . import config_sql, logger, cache_buster, cli, ub, db
from .reverseproxy import ReverseProxied
from .server import WebServer
# import queue
# queue = queue.Queue()
mimetypes.init()
mimetypes.add_type('application/xhtml+xml', '.xhtml')
......
......@@ -39,7 +39,7 @@ from sqlalchemy.sql.expression import func
from . import constants, logger, helper, services
from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash
from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
from .gdriveutils import is_gdrive_ready, gdrive_support
from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano
......@@ -149,9 +149,9 @@ def configuration():
@admin_required
def view_configuration():
readColumn = calibre_db.session.query(db.Custom_Columns)\
.filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
.filter(and_(db.Custom_Columns.datatype == 'bool', db.Custom_Columns.mark_for_delete == 0)).all()
restrictColumns= calibre_db.session.query(db.Custom_Columns)\
.filter(and_(db.Custom_Columns.datatype == 'text',db.Custom_Columns.mark_for_delete == 0)).all()
.filter(and_(db.Custom_Columns.datatype == 'text', db.Custom_Columns.mark_for_delete == 0)).all()
return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn,
restrictColumns=restrictColumns,
title=_(u"UI Configuration"), page="uiconfig")
......@@ -878,7 +878,7 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support, do
@admin_required
def new_user():
content = ub.User()
languages = speaking_language()
languages = calibre_db.speaking_language()
translations = [LC('en')] + babel.list_translations()
kobo_support = feature_support['kobo'] and config.config_kobo_sync
if request.method == "POST":
......@@ -942,11 +942,11 @@ def edit_user(user_id):
flash(_(u"User not found"), category="error")
return redirect(url_for('admin.admin'))
downloads = list()
languages = speaking_language()
languages = calibre_db.speaking_language()
translations = babel.list_translations() + [LC('en')]
kobo_support = feature_support['kobo'] and config.config_kobo_sync
for book in content.downloads:
downloadbook = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
downloadbook = calibre_db.get_book(book.book_id)
if downloadbook:
downloads.append(downloadbook)
else:
......
......@@ -22,10 +22,9 @@ import sys
import os
import re
import ast
import json
from datetime import datetime
import threading
import time
import queue
from sqlalchemy import create_engine
from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
......@@ -33,12 +32,24 @@ from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
from sqlalchemy.orm import relationship, sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import OperationalError
from flask_login import current_user
from sqlalchemy.sql.expression import and_, true, false, text, func, or_
from babel import Locale as LC
from babel.core import UnknownLocaleError
from flask_babel import gettext as _
from . import logger, ub, isoLanguages
from .pagination import Pagination
try:
import unidecode
use_unidecode = True
except ImportError:
use_unidecode = False
from . import logger
# session = None
cc_exceptions = ['datetime', 'comments', 'composite', 'series']
cc_classes = {}
# engine = None
Base = declarative_base()
......@@ -327,6 +338,7 @@ class CalibreDB(threading.Thread):
self.session = None
self.queue = None
self.log = None
self.config = None
def add_queue(self,queue):
self.queue = queue
......@@ -356,6 +368,7 @@ class CalibreDB(threading.Thread):
self.queue.put('dummy')
def setup_db(self, config, app_db_path):
self.config = config
self.dispose()
# global engine
......@@ -441,6 +454,149 @@ class CalibreDB(threading.Thread):
self.session = Session()
return True
def get_book(self, book_id):
return self.session.query(Books).filter(Books.id == book_id).first()
def get_filtered_book(self, book_id, allow_show_archived=False):
return self.session.query(Books).filter(Books.id == book_id).\
filter(self.common_filters(allow_show_archived)).first()
def get_book_by_uuid(self, book_uuid):
return self.session.query(Books).filter(Books.uuid == book_uuid).first()
def get_book_format(self, book_id, format):
return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first()
# Language and content filters for displaying in the UI
def common_filters(self, allow_show_archived=False):
if not allow_show_archived:
archived_books = (
ub.session.query(ub.ArchivedBook)
.filter(ub.ArchivedBook.user_id == int(current_user.id))
.filter(ub.ArchivedBook.is_archived == True)
.all()
)
archived_book_ids = [archived_book.book_id for archived_book in archived_books]
archived_filter = Books.id.notin_(archived_book_ids)
else:
archived_filter = true()
if current_user.filter_language() != "all":
lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language())
else:
lang_filter = true()
negtags_list = current_user.list_denied_tags()
postags_list = current_user.list_allowed_tags()
neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list))
pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list))
if self.config.config_restricted_column:
pos_cc_list = current_user.allowed_column_value.split(',')
pos_content_cc_filter = true() if pos_cc_list == [''] else \
getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
neg_cc_list = current_user.denied_column_value.split(',')
neg_content_cc_filter = false() if neg_cc_list == [''] else \
getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
else:
pos_content_cc_filter = true()
neg_content_cc_filter = false()
return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
# Fill indexpage with all requested data from database
def fill_indexpage(self, page, database, db_filter, order, *join):
return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join):
if current_user.show_detail_random():
randm = self.session.query(Books) \
.filter(self.common_filters(allow_show_archived)) \
.order_by(func.random()) \
.limit(self.config.config_random_books)
else:
randm = false()
off = int(int(self.config.config_books_per_page) * (page - 1))
query = self.session.query(database) \
.join(*join, isouter=True) \
.filter(db_filter) \
.filter(self.common_filters(allow_show_archived))
pagination = Pagination(page, self.config.config_books_per_page,
len(query.all()))
entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all()
for book in entries:
book = self.order_authors(book)
return entries, randm, pagination
# Orders all Authors in the list according to authors sort
def order_authors(self, entry):
sort_authors = entry.author_sort.split('&')
authors_ordered = list()
error = False
for auth in sort_authors:
# ToDo: How to handle not found authorname
result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first()
if not result:
error = True
break
authors_ordered.append(result)
if not error:
entry.authors = authors_ordered
return entry
def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()):
query = query or ''
self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
entries = self.session.query(database).filter(tag_filter). \
filter(func.lower(database.name).ilike("%" + query + "%")).all()
json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
return json_dumps
def check_exists_book(self, authr, title):
self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
q = list()
authorterms = re.split(r'\s*&\s*', authr)
for authorterm in authorterms:
q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
return self.session.query(Books)\
.filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
# read search results from calibre-database and return it (function is used for feed and simple search
def get_search_results(self, term):
term.strip().lower()
self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
q = list()
authorterms = re.split("[, ]+", term)
for authorterm in authorterms:
q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
return self.session.query(Books).filter(self.common_filters()).filter(
or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
Books.authors.any(and_(*q)),
Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
func.lower(Books.title).ilike("%" + term + "%")
)).order_by(Books.sort).all()
# Creates for all stored languages a translated speaking name in the array for the UI
def speaking_language(self, languages=None):
from . import get_locale
if not languages:
languages = self.session.query(Languages) \
.join(books_languages_link) \
.join(Books) \
.filter(self.common_filters()) \
.group_by(text('books_languages_link.lang_code')).all()
for lang in languages:
try:
cur_l = LC.parse(lang.lang_code)
lang.name = cur_l.get_language_name(get_locale())
except UnknownLocaleError:
lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
return languages
def update_title_sort(self, config, conn=None):
# user defined sort function for calibre databases (Series, etc.)
def _title_sort(title):
......@@ -481,8 +637,13 @@ class CalibreDB(threading.Thread):
if table is not None:
Base.metadata.remove(table)
def reconnect_db(self, config, app_db_path):
self.session.close()
self.engine.dispose()
self.setup_db(config, app_db_path)
def lcase(self, s):
try:
return unidecode.unidecode(s.lower())
except Exception as e:
self.log.exception(e)
......@@ -35,7 +35,6 @@ from sqlalchemy.exc import OperationalError
from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
from . import config, get_locale, ub, worker, db
from . import calibre_db
from .helper import order_authors, common_filters
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
......@@ -176,7 +175,7 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
@login_required
def delete_book(book_id, book_format):
if current_user.role_delete_books():
book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
book = calibre_db.get_book(book_id)
if book:
try:
result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
......@@ -250,9 +249,7 @@ def delete_book(book_id, book_format):
def render_edit_book(book_id):
calibre_db.update_title_sort(config)
cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
book = calibre_db.session.query(db.Books)\
.filter(db.Books.id == book_id).filter(common_filters()).first()
book = calibre_db.get_filtered_book(book_id)
if not book:
flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
return redirect(url_for("web.index"))
......@@ -260,7 +257,7 @@ def render_edit_book(book_id):
for lang in book.languages:
lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
book = order_authors(book)
book = calibre_db.order_authors(book)
author_names = []
for authr in book.authors:
......@@ -447,8 +444,7 @@ def upload_single_file(request, book, book_id):
return redirect(url_for('web.show_book', book_id=book.id))
file_size = os.path.getsize(saved_filename)
is_format = calibre_db.session.query(db.Data).filter(db.Data.book == book_id).\
filter(db.Data.format == file_ext.upper()).first()
is_format = calibre_db.get_book_format(book_id, file_ext.upper())
# Format entry already exists, no need to update the database
if is_format:
......@@ -500,8 +496,7 @@ def edit_book(book_id):
# create the function for sorting...
calibre_db.update_title_sort(config)
book = calibre_db.session.query(db.Books)\
.filter(db.Books.id == book_id).filter(common_filters()).first()
book = calibre_db.get_filtered_book(book_id)
# Book not found
if not book:
......@@ -702,7 +697,7 @@ def upload():
series_index = meta.series_id
if title != _(u'Unknown') and authr != _(u'Unknown'):
entry = helper.check_exists_book(authr, title)
entry = calibre_db.check_exists_book(authr, title)
if entry:
log.info("Uploaded book probably exists in library")
flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
......@@ -807,7 +802,7 @@ def upload():
calibre_db.update_title_sort(config)
# Reread book. It's important not to filter the result, as it could have language which hide it from
# current users view (tags are not stored/extracted from metadata and could also be limited)
book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
book = calibre_db.get_book(book_id)
# upload book to gdrive if nesseccary and add "(bookid)" to folder name
if config.config_use_google_drive:
gdriveutils.updateGdriveCalibreFromLocal()
......
This diff is collapsed.
......@@ -111,10 +111,3 @@ def timestamptodate(date, fmt=None):
@jinjia.app_template_filter('yesno')
def yesno(value, yes, no):
return yes if value else no
'''@jinjia.app_template_filter('canread')
def canread(ext):
if isinstance(ext, db.Data):
ext = ext.format
return ext.lower() in EXTENSIONS_READER'''
......@@ -256,7 +256,7 @@ def HandleMetadataRequest(book_uuid):
if not current_app.wsgi_app.is_proxied:
log.debug('Kobo: Received unproxied request, changed request port to server port')
log.info("Kobo library metadata request received for book %s" % book_uuid)
book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
book = calibre_db.get_book_by_uuid(book_uuid)
if not book or not book.data:
log.info(u"Book %s not found in database", book_uuid)
return redirect_or_proxy_request()
......@@ -474,7 +474,7 @@ def add_items_to_shelf(items, shelf):
items_unknown_to_calibre.append(item)
continue
book = calibre_db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
book = calibre_db.get_book_by_uuid(item["RevisionId"])
if not book:
items_unknown_to_calibre.append(item)
continue
......@@ -545,7 +545,7 @@ def HandleTagRemoveItem(tag_id):
items_unknown_to_calibre.append(item)
continue
book = calibre_db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
book = calibre_db.get_book_by_uuid(item["RevisionId"])
if not book:
items_unknown_to_calibre.append(item)
continue
......@@ -613,7 +613,7 @@ def create_kobo_tag(shelf):
"Type": "UserTag"
}
for book_shelf in shelf.books:
book = calibre_db.session.query(db.Books).filter(db.Books.id == book_shelf.book_id).one_or_none()
book = calibre_db.get_book_by_uuid(book_shelf.book_id)
if not book:
log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database", book_shelf.book_id, shelf.id)
continue
......@@ -629,7 +629,7 @@ def create_kobo_tag(shelf):
@kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
@login_required
def HandleStateRequest(book_uuid):
book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
book = calibre_db.get_book_by_uuid(book_uuid)
if not book or not book.data:
log.info(u"Book %s not found in database", book_uuid)
return redirect_or_proxy_request()
......@@ -804,7 +804,7 @@ def TopLevelEndpoint():
@login_required
def HandleBookDeletionRequest(book_uuid):
log.info("Kobo book deletion request received for book %s" % book_uuid)
book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
book = calibre_db.get_book_by_uuid(book_uuid)
if not book:
log.info(u"Book %s not found in database", book_uuid)
return redirect_or_proxy_request()
......
This diff is collapsed.
......@@ -30,7 +30,6 @@ from sqlalchemy.sql.expression import func
from . import logger, ub, searched_ids, db, calibre_db
from .web import render_title_template
from .helper import common_filters
shelf = Blueprint('shelf', __name__)
......@@ -320,11 +319,11 @@ def show_shelf(shelf_type, shelf_id):
books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\
.order_by(ub.BookShelf.order.asc()).all()
for book in books_in_shelf:
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
cur_book = calibre_db.get_filtered_book(book.book_id)
if cur_book:
result.append(cur_book)
else:
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
cur_book = calibre_db.get_book(book.book_id)
if not cur_book:
log.info('Not existing book %s in %s deleted', book.book_id, shelf)
ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete()
......@@ -356,7 +355,7 @@ def order_shelf(shelf_id):
books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
.order_by(ub.BookShelf.order.asc()).all()
for book in books_in_shelf2:
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
cur_book = calibre_db.get_filtered_book(book.book_id)
if cur_book:
result.append({'title': cur_book.title,
'id': cur_book.id,
......@@ -364,7 +363,7 @@ def order_shelf(shelf_id):
'series': cur_book.series,
'series_index': cur_book.series_index})
else:
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
cur_book = calibre_db.get_book(book.book_id)
result.append({'title': _('Hidden Book'),
'id': cur_book.id,
'author': [],
......
This diff is collapsed.
......@@ -291,7 +291,7 @@ class WorkerThread(threading.Thread):
w_session = Session()
engine.execute("attach database '{}' as calibre;".format(dbpath))'''
file_path = self.queue[index]['file_path']
bookid = self.queue[index]['bookid']
book_id = self.queue[index]['bookid']
format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower()
format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower()
......@@ -299,15 +299,15 @@ class WorkerThread(threading.Thread):
# if it does - mark the conversion task as complete and return a success
# this will allow send to kindle workflow to continue to work
if os.path.isfile(file_path + format_new_ext):
log.info("Book id %d already converted to %s", bookid, format_new_ext)
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == bookid).first()
log.info("Book id %d already converted to %s", book_id, format_new_ext)
cur_book = calibre_db.get_book(book_id)
self.queue[index]['path'] = file_path
self.queue[index]['title'] = cur_book.title
self._handleSuccess()
return file_path + format_new_ext
else:
log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
bookid,
book_id,
format_new_ext)
if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
......@@ -324,13 +324,13 @@ class WorkerThread(threading.Thread):
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index)
if check == 0:
cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == bookid).first()
cur_book = calibre_db.get_book(book_id)
if os.path.isfile(file_path + format_new_ext):
# self.db_queue.join()
new_format = db.Data(name=cur_book.data[0].name,
book_format=self.queue[index]['settings']['new_book_format'].upper(),
book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
task = {'task':'add_format','id': bookid, 'format': new_format}
book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
task = {'task':'add_format','id': book_id, 'format': new_format}
self.db_queue.put(task)
# To Do how to handle error?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment