Commit e734bb12 authored by Ozzieisaacs's avatar Ozzieisaacs

Merge remote-tracking branch 'github/config_sql' into Develop

parents 499a66df 006e596c
......@@ -25,10 +25,6 @@ from __future__ import division, print_function, unicode_literals
import sys
import os
import mimetypes
try:
import cPickle
except ImportError:
import pickle as cPickle
from babel import Locale as LC
from babel import negotiate_locale
......@@ -38,8 +34,7 @@ from flask_login import LoginManager
from flask_babel import Babel
from flask_principal import Principal
from . import logger, cache_buster, ub
from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR
from . import logger, cache_buster, cli, config_sql, ub
from .reverseproxy import ReverseProxied
......@@ -68,16 +63,9 @@ lm.login_view = 'web.login'
lm.anonymous_user = ub.Anonymous
ub.init_db()
config = ub.Config()
from . import db
try:
with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f:
language_table = cPickle.load(f)
except cPickle.UnpicklingError as error:
print("Can't read file cps/translations/iso639.pickle: %s" % error)
sys.exit(1)
ub.init_db(cli.settingspath)
config = config_sql.load_configuration(ub.session)
from . import db, services
searched_ids = {}
......@@ -87,10 +75,8 @@ global_WorkerThread = WorkerThread()
from .server import WebServer
web_server = WebServer()
from .ldap_login import Ldap
ldap1 = Ldap()
babel = Babel()
_BABEL_TRANSLATIONS = set()
log = logger.create()
......@@ -109,30 +95,45 @@ def create_app():
Principal(app)
lm.init_app(app)
app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
web_server.init_app(app, config)
db.setup_db()
db.setup_db(config)
babel.init_app(app)
ldap1.init_app(app)
_BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())
_BABEL_TRANSLATIONS.add('en')
if services.ldap:
services.ldap.init_app(app, config)
if services.goodreads:
services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads)
global_WorkerThread.start()
return app
@babel.localeselector
def get_locale():
def negociate_locale():
# if a user is logged in, use the locale from the user settings
user = getattr(g, 'user', None)
# user = None
if user is not None and hasattr(user, "locale"):
if user.nickname != 'Guest': # if the account is the guest account bypass the config lang settings
return user.locale
translations = [str(item) for item in babel.list_translations()] + ['en']
preferred = list()
for x in request.accept_languages.values():
try:
preferred.append(str(LC.parse(x.replace('-', '_'))))
except (UnknownLocaleError, ValueError) as e:
log.warning('Could not parse locale "%s": %s', x, e)
preferred.append('en')
return negotiate_locale(preferred, translations)
preferred = set()
if request.accept_languages:
for x in request.accept_languages.values():
try:
preferred.add(str(LC.parse(x.replace('-', '_'))))
except (UnknownLocaleError, ValueError) as e:
log.warning('Could not parse locale "%s": %s', x, e)
# preferred.append('en')
return negotiate_locale(preferred or ['en'], _BABEL_TRANSLATIONS)
def get_locale():
return request._locale
@babel.timezoneselector
......
......@@ -69,8 +69,8 @@ def stats():
versions['pytz'] = 'v' + pytzVersion
versions['Requests'] = 'v' + requests.__version__
versions['pySqlite'] = 'v' + db.engine.dialect.dbapi.version
versions['Sqlite'] = 'v' + db.engine.dialect.dbapi.sqlite_version
versions['pySqlite'] = 'v' + db.session.bind.dialect.dbapi.version
versions['Sqlite'] = 'v' + db.session.bind.dialect.dbapi.sqlite_version
versions.update(converter.versioncheck())
versions.update(serverVersion)
versions['Python'] = sys.version
......
This diff is collapsed.
This diff is collapsed.
......@@ -74,7 +74,7 @@ SIDEBAR_PUBLISHER = 1 << 12
SIDEBAR_RATING = 1 << 13
SIDEBAR_FORMAT = 1 << 14
ADMIN_USER_ROLES = (ROLE_VIEWER << 1) - 1 - (ROLE_ANONYMOUS | ROLE_EDIT_SHELFS)
ADMIN_USER_ROLES = sum(r for r in ALL_ROLES.values()) & ~ROLE_EDIT_SHELFS & ~ROLE_ANONYMOUS
ADMIN_USER_SIDEBAR = (SIDEBAR_FORMAT << 1) - 1
UPDATE_STABLE = 0 << 0
......@@ -109,6 +109,9 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz'
def has_flag(value, bit_flag):
return bit_flag == (bit_flag & (value or 0))
def selected_roles(dictionary):
return sum(v for k, v in ALL_ROLES.items() if k in dictionary)
# :rtype: BookMeta
BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
......
......@@ -24,19 +24,14 @@ import re
from flask_babel import gettext as _
from . import config
from .subproc_wrapper import process_open
from .subproc_wrapper import process_wait
def versionKindle():
versions = _(u'not installed')
if os.path.exists(config.config_converterpath):
try:
p = process_open(config.config_converterpath)
# p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
for lines in process_wait(config.config_converterpath):
if re.search('Amazon kindlegen\(', lines):
versions = lines
except Exception:
......@@ -48,12 +43,7 @@ def versionCalibre():
versions = _(u'not installed')
if os.path.exists(config.config_converterpath):
try:
p = process_open([config.config_converterpath, '--version'])
# p = subprocess.Popen([ub.config.config_converterpath, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
for lines in process_wait([config.config_converterpath, '--version']):
if re.search('ebook-convert.*\(calibre', lines):
versions = lines
except Exception:
......
......@@ -30,24 +30,10 @@ from sqlalchemy import String, Integer, Boolean
from sqlalchemy.orm import relationship, sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_base
from . import config, ub
session = None
cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series']
cc_classes = None
engine = None
# user defined sort function for calibre databases (Series, etc.)
def title_sort(title):
# calibre sort stuff
title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
match = title_pat.search(title)
if match:
prep = match.group(1)
title = title.replace(prep, '') + ', ' + prep
return title.strip()
cc_classes = {}
Base = declarative_base()
......@@ -325,40 +311,45 @@ class Custom_Columns(Base):
return display_dict
def setup_db():
global engine
global session
global cc_classes
if config.config_calibre_dir is None or config.config_calibre_dir == u'':
content = ub.session.query(ub.Settings).first()
content.config_calibre_dir = None
content.db_configured = False
ub.session.commit()
config.loadSettings()
def update_title_sort(config, conn=None):
# user defined sort function for calibre databases (Series, etc.)
def _title_sort(title):
# calibre sort stuff
title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
match = title_pat.search(title)
if match:
prep = match.group(1)
title = title.replace(prep, '') + ', ' + prep
return title.strip()
conn = conn or session.connection().connection.connection
conn.create_function("title_sort", 1, _title_sort)
def setup_db(config):
dispose()
if not config.config_calibre_dir:
config.invalidate()
return False
dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
if not os.path.exists(dbpath):
config.invalidate()
return False
try:
if not os.path.exists(dbpath):
raise
engine = create_engine('sqlite:///{0}'.format(dbpath),
echo=False,
isolation_level="SERIALIZABLE",
connect_args={'check_same_thread': False})
conn = engine.connect()
except Exception:
content = ub.session.query(ub.Settings).first()
content.config_calibre_dir = None
content.db_configured = False
ub.session.commit()
config.loadSettings()
except:
config.invalidate()
return False
content = ub.session.query(ub.Settings).first()
content.db_configured = True
ub.session.commit()
config.loadSettings()
conn.connection.create_function('title_sort', 1, title_sort)
config.db_configured = True
update_title_sort(config, conn.connection)
# conn.connection.create_function('lower', 1, lcase)
# conn.connection.create_function('upper', 1, ucase)
......@@ -367,7 +358,6 @@ def setup_db():
cc_ids = []
books_custom_column_links = {}
cc_classes = {}
for row in cc:
if row.datatype not in cc_exceptions:
books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
......@@ -406,8 +396,38 @@ def setup_db():
backref='books'))
global session
Session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
session = Session()
return True
def dispose():
global session
engine = None
if session:
engine = session.bind
try: session.close()
except: pass
session = None
if engine:
try: engine.dispose()
except: pass
for attr in list(Books.__dict__.keys()):
if attr.startswith("custom_column_"):
delattr(Books, attr)
for db_class in cc_classes.values():
Base.metadata.remove(db_class.__table__)
cc_classes.clear()
for table in reversed(Base.metadata.sorted_tables):
name = table.key
if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
if table is not None:
Base.metadata.remove(table)
......@@ -33,7 +33,7 @@ from flask_babel import gettext as _
from flask_login import current_user
from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
from . import config, get_locale, db, ub, global_WorkerThread, language_table
from . import config, get_locale, db, ub, global_WorkerThread
from .helper import order_authors, common_filters
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required
......@@ -206,7 +206,7 @@ def delete_book(book_id, book_format):
def render_edit_book(book_id):
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.update_title_sort(config)
cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
book = db.session.query(db.Books)\
.filter(db.Books.id == book_id).filter(common_filters()).first()
......@@ -215,8 +215,8 @@ def render_edit_book(book_id):
flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
return redirect(url_for("web.index"))
for indx in range(0, len(book.languages)):
book.languages[indx].language_name = language_table[get_locale()][book.languages[indx].lang_code]
for lang in book.languages:
lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
book = order_authors(book)
......@@ -354,7 +354,7 @@ def upload_single_file(request, book, book_id):
db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
db.session.add(db_format)
db.session.commit()
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.update_title_sort(config)
# Queue uploader info
uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
......@@ -385,7 +385,7 @@ def edit_book(book_id):
return render_edit_book(book_id)
# create the function for sorting...
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.update_title_sort(config)
book = db.session.query(db.Books)\
.filter(db.Books.id == book_id).filter(common_filters()).first()
......@@ -484,17 +484,12 @@ def edit_book(book_id):
# handle book languages
input_languages = to_save["languages"].split(',')
input_languages = [x.strip().lower() for x in input_languages if x != '']
input_l = []
invers_lang_table = [x.lower() for x in language_table[get_locale()].values()]
for lang in input_languages:
try:
res = list(language_table[get_locale()].keys())[invers_lang_table.index(lang)]
input_l.append(res)
except ValueError:
log.error('%s is not a valid language', lang)
flash(_(u"%(langname)s is not a valid language", langname=lang), category="error")
modify_database_object(input_l, book.languages, db.Languages, db.session, 'languages')
unknown_languages = []
input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages)
for l in unknown_languages:
log.error('%s is not a valid language', l)
flash(_(u"%(langname)s is not a valid language", langname=l), category="error")
modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages')
# handle book ratings
if to_save["rating"].strip():
......@@ -546,7 +541,7 @@ def upload():
if request.method == 'POST' and 'btn-upload' in request.files:
for requested_file in request.files.getlist("btn-upload"):
# create the function for sorting...
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.update_title_sort(config)
db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
# check if file extension is correct
......@@ -659,7 +654,7 @@ def upload():
# save data to database, reread data
db.session.commit()
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.update_title_sort(config)
book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
# upload book to gdrive if nesseccary and add "(bookid)" to folder name
......
......@@ -39,7 +39,7 @@ try:
except ImportError:
pass
from . import logger, gdriveutils, config, ub, db
from . import logger, gdriveutils, config, db
from .web import admin_required
......@@ -94,12 +94,9 @@ def watch_gdrive():
try:
result = gdriveutils.watchChange(gdriveutils.Gdrive.Instance().drive, notification_id,
'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000)
settings = ub.session.query(ub.Settings).first()
settings.config_google_drive_watch_changes_response = json.dumps(result)
ub.session.merge(settings)
ub.session.commit()
settings = ub.session.query(ub.Settings).first()
config.loadSettings()
config.config_google_drive_watch_changes_response = json.dumps(result)
# after save(), config_google_drive_watch_changes_response will be a json object, not string
config.save()
except HttpError as e:
reason=json.loads(e.content)['error']['errors'][0]
if reason['reason'] == u'push.webhookUrlUnauthorized':
......@@ -121,11 +118,8 @@ def revoke_watch_gdrive():
last_watch_response['resourceId'])
except HttpError:
pass
settings = ub.session.query(ub.Settings).first()
settings.config_google_drive_watch_changes_response = None
ub.session.merge(settings)
ub.session.commit()
config.loadSettings()
config.config_google_drive_watch_changes_response = None
config.save()
return redirect(url_for('admin.configuration'))
......@@ -157,7 +151,7 @@ def on_received_watch_confirmation():
log.info('Setting up new DB')
# prevent error on windows, as os.rename does on exisiting files
move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath)
db.setup_db()
db.setup_db(config)
except Exception as e:
log.exception(e)
updateMetaData()
......
......@@ -19,6 +19,7 @@
from __future__ import division, print_function, unicode_literals
import os
import json
import shutil
from flask import Response, stream_with_context
......@@ -79,6 +80,9 @@ class Singleton:
except AttributeError:
self._instance = self._decorated()
return self._instance
except ImportError as e:
log.debug(e)
return None
def __call__(self):
raise TypeError('Singletons must be accessed through `Instance()`.')
......@@ -534,3 +538,51 @@ def do_gdrive_download(df, headers):
log.warning('An error occurred: %s', resp)
return
return Response(stream_with_context(stream()), headers=headers)
_SETTINGS_YAML_TEMPLATE = """
client_config_backend: settings
client_config_file: %(client_file)s
client_config:
client_id: %(client_id)s
client_secret: %(client_secret)s
redirect_uri: %(redirect_uri)s
save_credentials: True
save_credentials_backend: file
save_credentials_file: %(credential)s
get_refresh_token: True
oauth_scope:
- https://www.googleapis.com/auth/drive
"""
def update_settings(client_id, client_secret, redirect_uri):
if redirect_uri.endswith('/'):
redirect_uri = redirect_uri[:-1]
config_params = {
'client_file': CLIENT_SECRETS,
'client_id': client_id,
'client_secret': client_secret,
'redirect_uri': redirect_uri,
'credential': CREDENTIALS
}
with open(SETTINGS_YAML, 'w') as f:
f.write(_SETTINGS_YAML_TEMPLATE % config_params)
def get_error_text(client_secrets=None):
if not gdrive_support:
return 'Import of optional Google Drive requirements missing'
if not os.path.isfile(CLIENT_SECRETS):
return 'client_secrets.json is missing or not readable'
with open(CLIENT_SECRETS, 'r') as settings:
filedata = json.load(settings)
if 'web' not in filedata:
return 'client_secrets.json is not configured for web application'
if client_secrets:
client_secrets.update(filedata['web'])
......@@ -26,17 +26,16 @@ import json
import mimetypes
import random
import re
import requests
import shutil
import time
import unicodedata
from datetime import datetime, timedelta
from functools import reduce
from tempfile import gettempdir
import requests
from babel import Locale as LC
from babel.core import UnknownLocaleError
from babel.dates import format_datetime, format_timedelta
from babel.dates import format_datetime
from babel.units import format_unit
from flask import send_from_directory, make_response, redirect, abort
from flask_babel import gettext as _
......@@ -55,12 +54,6 @@ try:
except ImportError:
use_unidecode = False
try:
import Levenshtein
use_levenshtein = True
except ImportError:
use_levenshtein = False
try:
from PIL import Image
use_PIL = True
......@@ -71,7 +64,7 @@ from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLangua
from . import gdriveutils as gd
from .constants import STATIC_DIR as _STATIC_DIR
from .pagination import Pagination
from .subproc_wrapper import process_open
from .subproc_wrapper import process_wait
from .worker import STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
from .worker import TASK_EMAIL, TASK_CONVERT, TASK_UPLOAD, TASK_CONVERT_ANY
......@@ -110,7 +103,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
if os.path.exists(file_path + "." + old_book_format.lower()):
# read settings and append converter task to queue
if kindle_mail:
settings = ub.get_mail_settings()
settings = config.get_mail_settings()
settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail
settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
# text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
......@@ -128,7 +121,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
def send_test_mail(kindle_mail, user_name):
global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, ub.get_mail_settings(),
global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(),
kindle_mail, user_name, _(u"Test e-mail"),
_(u'This e-mail has been sent via Calibre-Web.'))
return
......@@ -145,7 +138,7 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
text += "Don't forget to change your password after first login.\r\n"
text += "Sincerely\r\n\r\n"
text += "Your Calibre-Web team"
global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, ub.get_mail_settings(),
global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(),
e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text)
return
......@@ -218,7 +211,7 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
for entry in iter(book.data):
if entry.format.upper() == book_format.upper():
result = entry.name + '.' + book_format.lower()
global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, ub.get_mail_settings(),
global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(),
kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title),
_(u'This e-mail has been sent via Calibre-Web.'))
return
......@@ -561,27 +554,23 @@ def do_download_file(book, book_format, data, headers):
def check_unrar(unrarLocation):
error = False
if os.path.exists(unrarLocation):
try:
if sys.version_info < (3, 0):
unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
p = process_open(unrarLocation)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
value=re.search('UNRAR (.*) freeware', lines)
if value:
version = value.group(1)
except OSError as e:
error = True
log.exception(e)
version =_(u'Error excecuting UnRar')
else:
version = _(u'Unrar binary file not found')
error=True
return (error, version)
if not unrarLocation:
return
if not os.path.exists(unrarLocation):
return 'Unrar binary file not found'
try:
if sys.version_info < (3, 0):
unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
for lines in process_wait(unrarLocation):
value = re.search('UNRAR (.*) freeware', lines)
if value:
version = value.group(1)
log.debug("unrar version %s", version)
except OSError as err:
log.exception(err)
return 'Error excecuting UnRar'
......@@ -605,7 +594,7 @@ def json_serial(obj):
def format_runtime(runtime):
retVal = ""
if runtime.days:
retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', '
retVal = format_unit(runtime.days, 'duration-day', length="long", locale=get_locale()) + ', '
mins, seconds = divmod(runtime.seconds, 60)
hours, minutes = divmod(mins, 60)
# ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ?
......@@ -630,7 +619,10 @@ def render_task_status(tasklist):
if 'starttime' not in task:
task['starttime'] = ""
task['runtime'] = format_runtime(task['formRuntime'])
if 'formRuntime' not in task:
task['runtime'] = ""
else:
task['runtime'] = format_runtime(task['formRuntime'])
# localize the task status
if isinstance( task['stat'], int ):
......@@ -754,28 +746,6 @@ def get_search_results(term):
func.lower(db.Books.title).ilike("%" + term + "%")
)).all()
def get_unique_other_books(library_books, author_books):
# Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates
# Note: Not all images will be shown, even though they're available on Goodreads.com.
# See https://www.goodreads.com/topic/show/18213769-goodreads-book-images
identifiers = reduce(lambda acc, book: acc + map(lambda identifier: identifier.val, book.identifiers),
library_books, [])
other_books = filter(lambda book: book.isbn not in identifiers and book.gid["#text"] not in identifiers,
author_books)
# Fuzzy match book titles
if use_levenshtein:
library_titles = reduce(lambda acc, book: acc + [book.title], library_books, [])
other_books = filter(lambda author_book: not filter(
lambda library_book:
# Remove items in parentheses before comparing
Levenshtein.ratio(re.sub(r"\(.*\)", "", author_book.title), library_book) > 0.7,
library_titles
), other_books)
return other_books
def get_cc_columns():
tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
if config.config_columns_to_ignore:
......@@ -802,10 +772,7 @@ def get_download_link(book_id, book_format):
file_name = book.authors[0].name + '_' + file_name
file_name = get_valid_filename(file_name)
headers = Headers()
try:
headers["Content-Type"] = mimetypes.types_map['.' + book_format]
except KeyError:
headers["Content-Type"] = "application/octet-stream"
headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (quote(file_name.encode('utf-8')),
book_format)
return do_download_file(book, book_format, data, headers)
......
......@@ -18,6 +18,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import sys
import os
try:
import cPickle
except ImportError:
import pickle as cPickle
from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR
try:
......@@ -33,14 +41,43 @@ except ImportError:
__version__ = "? (PyCountry)"
def _copy_fields(l):
l.part1 = l.alpha_2
l.part3 = l.alpha_3
l.part1 = getattr(l, 'alpha_2', None)
l.part3 = getattr(l, 'alpha_3', None)
return l
def get(name=None, part1=None, part3=None):
if (part3 is not None):
if part3 is not None:
return _copy_fields(pyc_languages.get(alpha_3=part3))
if (part1 is not None):
if part1 is not None:
return _copy_fields(pyc_languages.get(alpha_2=part1))
if (name is not None):
if name is not None:
return _copy_fields(pyc_languages.get(name=name))
try:
with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f:
_LANGUAGES = cPickle.load(f)
except cPickle.UnpicklingError as error:
print("Can't read file cps/translations/iso639.pickle: %s" % error)
sys.exit(1)
def get_language_names(locale):
return _LANGUAGES.get(locale)
def get_language_name(locale, lang_code):
return get_language_names(locale)[lang_code]
def get_language_codes(locale, language_names, remainder=None):
language_names = set(x.strip().lower() for x in language_names if x)
for k, v in get_language_names(locale).items():
v = v.lower()
if v in language_names:
language_names.remove(v)
yield k
if remainder is not None:
remainder.extend(language_names)
......@@ -71,11 +71,7 @@ def shortentitle_filter(s, nchar=20):
@jinjia.app_template_filter('mimetype')
def mimetype_filter(val):
try:
s = mimetypes.types_map['.' + val]
except Exception:
s = 'application/octet-stream'
return s
return mimetypes.types_map.get('.' + val, 'application/octet-stream')
@jinjia.app_template_filter('formatdate')
......
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2019 Krakinou
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import base64
try:
from flask_simpleldap import LDAP # , LDAPException
ldap_support = True
except ImportError:
ldap_support = False
from . import config, logger
log = logger.create()
class Ldap():
def __init__(self):
self.ldap = None
return
def init_app(self, app):
if ldap_support and config.config_login_type == 1:
app.config['LDAP_HOST'] = config.config_ldap_provider_url
app.config['LDAP_PORT'] = config.config_ldap_port
app.config['LDAP_SCHEMA'] = config.config_ldap_schema
app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\
+ ',' + config.config_ldap_dn
app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password)
if config.config_ldap_use_ssl:
app.config['LDAP_USE_SSL'] = True
if config.config_ldap_use_tls:
app.config['LDAP_USE_TLS'] = True
app.config['LDAP_REQUIRE_CERT'] = config.config_ldap_require_cert
if config.config_ldap_require_cert:
app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path
app.config['LDAP_BASE_DN'] = config.config_ldap_dn
app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object
if config.config_ldap_openldap:
app.config['LDAP_OPENLDAP'] = True
# app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org'
# app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)'
self.ldap = LDAP(app)
elif config.config_login_type == 1 and not ldap_support:
log.error('Cannot activate ldap support, did you run \'pip install --target vendor -r optional-requirements.txt\'?')
@classmethod
def ldap_supported(cls):
return ldap_support
......@@ -157,3 +157,8 @@ class StderrLogger(object):
self.buffer += message
except Exception:
self.log.debug("Logging Error")
# if debugging, start logging to stderr immediately
if os.environ.get('FLASK_DEBUG', None):
setup(LOG_TO_STDERR, logging.DEBUG)
......@@ -24,7 +24,6 @@
from __future__ import division, print_function, unicode_literals
import json
from functools import wraps
from oauth import OAuthBackend
from flask import session, request, make_response, abort
from flask import Blueprint, flash, redirect, url_for
......@@ -37,6 +36,7 @@ from sqlalchemy.orm.exc import NoResultFound
from . import constants, logger, config, app, ub
from .web import login_required
from .oauth import OAuthBackend
# from .web import github_oauth_required
......
......@@ -31,7 +31,7 @@ from flask_login import current_user
from sqlalchemy.sql.expression import func, text, or_, and_
from werkzeug.security import check_password_hash
from . import logger, config, db, ub, ldap1
from . import constants, logger, config, db, ub, services
from .helper import fill_indexpage, get_download_link, get_book_cover
from .pagination import Pagination
from .web import common_filters, get_search_results, render_read_books, download_required
......@@ -40,7 +40,6 @@ from .web import common_filters, get_search_results, render_read_books, download
opds = Blueprint('opds', __name__)
log = logger.create()
ldap_support = ldap1.ldap_supported()
def requires_basic_auth_if_no_ano(f):
......@@ -51,8 +50,8 @@ def requires_basic_auth_if_no_ano(f):
if not auth or not check_auth(auth.username, auth.password):
return authenticate()
return f(*args, **kwargs)
if config.config_login_type == 1 and ldap_support:
return ldap1.ldap.basic_auth_required(f)
if config.config_login_type == constants.LOGIN_LDAP and services.ldap:
return services.ldap.basic_auth_required(f)
return decorated
......
......@@ -197,6 +197,7 @@ class WebServer:
self.stop()
def stop(self, restart=False):
log.info("webserver stop (restart=%s)", restart)
self.restart = restart
if self.wsgiserver:
if _GEVENT:
......
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2019 pwr
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
from .. import logger
log = logger.create()
try: from . import goodreads
except ImportError as err:
log.warning("goodreads: %s", err)
goodreads = None
try: from . import simpleldap as ldap
except ImportError as err:
log.warning("simpleldap: %s", err)
ldap = None
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2018-2019 OzzieIsaacs, pwr
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import time
from functools import reduce
from goodreads.client import GoodreadsClient
try: import Levenshtein
except ImportError: Levenshtein = False
from .. import logger
log = logger.create()
_client = None # type: GoodreadsClient
# GoodReads TOS allows for 24h caching of data
_CACHE_TIMEOUT = 23 * 60 * 60 # 23 hours (in seconds)
_AUTHORS_CACHE = {}
def connect(key=None, secret=None, enabled=True):
global _client
if not enabled or not key or not secret:
_client = None
return
if _client:
# make sure the configuration has not changed since last we used the client
if _client.client_key != key or _client.client_secret != secret:
_client = None
if not _client:
_client = GoodreadsClient(key, secret)
def get_author_info(author_name):
now = time.time()
author_info = _AUTHORS_CACHE.get(author_name, None)
if author_info:
if now < author_info._timestamp + _CACHE_TIMEOUT:
return author_info
# clear expired entries
del _AUTHORS_CACHE[author_name]
if not _client:
log.warning("failed to get a Goodreads client")
return
try:
author_info = _client.find_author(author_name=author_name)
except Exception as ex:
# Skip goodreads, if site is down/inaccessible
log.warning('Goodreads website is down/inaccessible? %s', ex)
return
if author_info:
author_info._timestamp = now
_AUTHORS_CACHE[author_name] = author_info
return author_info
def get_other_books(author_info, library_books=None):
# Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates
# Note: Not all images will be shown, even though they're available on Goodreads.com.
# See https://www.goodreads.com/topic/show/18213769-goodreads-book-images
if not author_info:
return
identifiers = []
library_titles = []
if library_books:
identifiers = list(reduce(lambda acc, book: acc + [i.val for i in book.identifiers if i.val], library_books, []))
library_titles = [book.title for book in library_books]
for book in author_info.books:
if book.isbn in identifiers:
continue
if book.gid["#text"] in identifiers:
continue
if Levenshtein and library_titles:
goodreads_title = book._book_dict['title_without_series']
if any(Levenshtein.ratio(goodreads_title, title) > 0.7 for title in library_titles):
continue
yield book
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2018-2019 OzzieIsaacs, pwr
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import base64
from flask_simpleldap import LDAP
from ldap import SERVER_DOWN, INVALID_CREDENTIALS
from .. import constants, logger
log = logger.create()
_ldap = None
def init_app(app, config):
global _ldap
if config.config_login_type != constants.LOGIN_LDAP:
_ldap = None
return
app.config['LDAP_HOST'] = config.config_ldap_provider_url
app.config['LDAP_PORT'] = config.config_ldap_port
app.config['LDAP_SCHEMA'] = config.config_ldap_schema
app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\
+ ',' + config.config_ldap_dn
app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password)
app.config['LDAP_REQUIRE_CERT'] = bool(config.config_ldap_require_cert)
if config.config_ldap_require_cert:
app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path
app.config['LDAP_BASE_DN'] = config.config_ldap_dn
app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object
app.config['LDAP_USE_SSL'] = bool(config.config_ldap_use_ssl)
app.config['LDAP_USE_TLS'] = bool(config.config_ldap_use_tls)
app.config['LDAP_OPENLDAP'] = bool(config.config_ldap_openldap)
# app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org'
# app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)'
_ldap = LDAP(app)
def basic_auth_required(func):
return _ldap.basic_auth_required(func)
def bind_user(username, password):
'''Attempts a LDAP login.
:returns: True if login succeeded, False if login failed, None if server unavailable.
'''
try:
result = _ldap.bind_user(username, password)
log.debug("LDAP login '%s': %r", username, result)
return result is not None
except SERVER_DOWN as ex:
log.warning('LDAP Server down: %s', ex)
return None
except INVALID_CREDENTIALS as ex:
log.info("LDAP login '%s' failed: %s", username, ex)
return False
......@@ -43,3 +43,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro
exc_command = [x for x in command]
return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env)
def process_wait(command, serr=subprocess.PIPE):
'''Run command, wait for process to terminate, and return an iterator over lines of its output.'''
p = process_open(command, serr=serr)
p.wait()
for l in p.stdout.readlines():
if isinstance(l, bytes):
l = l.decode('utf-8')
yield l
......@@ -69,7 +69,7 @@
</div>
<div class="row">
<div class="col-xs-6 col-sm-6">{{_('Log level')}}</div>
<div class="col-xs-6 col-sm-6">{{config.get_Log_Level()}}</div>
<div class="col-xs-6 col-sm-6">{{config.get_log_level()}}</div>
</div>
<div class="row">
<div class="col-xs-6 col-sm-6">{{_('Port')}}</div>
......
......@@ -94,8 +94,8 @@
<input type="text" class="form-control" name="config_keyfile" id="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
</div>
<div class="form-group">
<label for="config_updater">{{_('Update channel')}}</label>
<select name="config_updater" id="config_updater" class="form-control">
<label for="config_updatechannel">{{_('Update channel')}}</label>
<select name="config_updatechannel" id="config_updatechannel" class="form-control">
<option value="0" {% if config.config_updatechannel == 0 %}selected{% endif %}>{{_('Stable')}}</option>
<!--option value="1" {% if config.config_updatechannel == 1 %}selected{% endif %}>{{_('Stable (Automatic)')}}</option-->
<option value="2" {% if config.config_updatechannel == 2 %}selected{% endif %}>{{_('Nightly')}}</option>
......@@ -327,10 +327,10 @@
<div class="col-sm-12">
<button type="submit" name="submit" class="btn btn-default">{{_('Submit')}}</button>
{% if not origin %}
{% if show_back_button %}
<a href="{{ url_for('admin.admin') }}" class="btn btn-default">{{_('Back')}}</a>
{% endif %}
{% if success %}
{% if show_login_button %}
<a href="{{ url_for('web.login') }}" name="login" class="btn btn-default">{{_('Login')}}</a>
{% endif %}
</div>
......
......@@ -38,7 +38,7 @@
</button>
<a class="navbar-brand" href="{{url_for('web.index')}}">{{instance}}</a>
</div>
{% if g.user.is_authenticated or g.user.is_anonymous %}
{% if g.user.is_authenticated or g.allow_anonymous %}
<form class="navbar-form navbar-left" role="search" action="{{url_for('web.search')}}" method="GET">
<div class="form-group input-group input-group-sm">
<label for="query" class="sr-only">{{_('Search')}}</label>
......@@ -50,13 +50,13 @@
</form>
{% endif %}
<div class="navbar-collapse collapse">
{% if g.user.is_authenticated or g.user.is_anonymous %}
{% if g.user.is_authenticated or g.allow_anonymous %}
<ul class="nav navbar-nav ">
<li><a href="{{url_for('web.advanced_search')}}"><span class="glyphicon glyphicon-search"></span><span class="hidden-sm"> {{_('Advanced Search')}}</span></a></li>
</ul>
{% endif %}
<ul class="nav navbar-nav navbar-right" id="main-nav">
{% if g.user.is_authenticated or g.user.is_anonymous %}
{% if g.user.is_authenticated or g.allow_anonymous %}
{% if g.user.role_upload() or g.user.role_admin()%}
{% if g.allow_upload %}
<li>
......@@ -115,7 +115,7 @@
{%endif%}
<div class="container-fluid">
<div class="row-fluid">
{% if g.user.is_authenticated or g.user.is_anonymous %}
{% if g.user.is_authenticated or g.allow_anonymous %}
<div class="col-sm-2">
<nav class="navigation">
<ul class="list-unstyled" id="scnd-nav" intent in-standard-append="nav.navigation" in-mobile-after="#main-nav" in-mobile-class="nav navbar-nav">
......@@ -128,7 +128,7 @@
{% endif %}
{% endfor %}
{% if g.user.is_authenticated or g.user.is_anonymous %}
{% if g.user.is_authenticated or allow_anonymous %}
<li class="nav-head hidden-xs public-shelves">{{_('Public Shelves')}}</li>
{% for shelf in g.public_shelfes %}
<li><a href="{{url_for('shelf.show_shelf', shelf_id=shelf.id)}}"><span class="glyphicon glyphicon-list public_shelf"></span>{{shelf.name|shortentitle(40)}}</a></li>
......
This diff is collapsed.
......@@ -22,7 +22,6 @@ import sys
import os
import datetime
import json
import requests
import shutil
import threading
import time
......@@ -30,6 +29,7 @@ import zipfile
from io import BytesIO
from tempfile import gettempdir
import requests
from babel.dates import format_datetime
from flask_babel import gettext as _
......@@ -58,16 +58,14 @@ class Updater(threading.Thread):
self.updateIndex = None
def get_current_version_info(self):
if config.get_update_channel == constants.UPDATE_STABLE:
if config.config_updatechannel == constants.UPDATE_STABLE:
return self._stable_version_info()
else:
return self._nightly_version_info()
return self._nightly_version_info()
def get_available_updates(self, request_method):
if config.get_update_channel == constants.UPDATE_STABLE:
if config.config_updatechannel == constants.UPDATE_STABLE:
return self._stable_available_updates(request_method)
else:
return self._nightly_available_updates(request_method)
return self._nightly_available_updates(request_method)
def run(self):
try:
......@@ -430,10 +428,9 @@ class Updater(threading.Thread):
return json.dumps(status)
def _get_request_path(self):
if config.get_update_channel == constants.UPDATE_STABLE:
if config.config_updatechannel == constants.UPDATE_STABLE:
return self.updateFile
else:
return _REPOSITORY_API_URL + '/zipball/master'
return _REPOSITORY_API_URL + '/zipball/master'
def _load_remote_data(self, repository_url):
status = {
......
This diff is collapsed.
......@@ -340,6 +340,8 @@ class WorkerThread(threading.Thread):
check = p.returncode
calibre_traceback = p.stderr.readlines()
for ele in calibre_traceback:
if sys.version_info < (3, 0):
ele = ele.decode('utf-8')
log.debug(ele.strip('\n'))
if not ele.startswith('Traceback') and not ele.startswith(' File'):
error_message = "Calibre failed with error: %s" % ele.strip('\n')
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment