Commit f79d5499 authored by Ozzieisaacs's avatar Ozzieisaacs

Fix duplicate user and email (now case insensitive #948)

Fix sorting in comics (#950)
Fix log error on Calibre converter error (#953)
Fix long running tasks (#954)
parent 66283c54
......@@ -34,7 +34,8 @@ from flask import send_from_directory, make_response, redirect, abort
from flask_babel import gettext as _
from flask_login import current_user
from babel.dates import format_datetime
from datetime import datetime
from babel.units import format_unit
from datetime import datetime, timedelta
import shutil
import requests
try:
......@@ -566,8 +567,33 @@ def json_serial(obj):
if isinstance(obj, (datetime)):
return obj.isoformat()
if isinstance(obj, (timedelta)):
return {
'__type__': 'timedelta',
'days': obj.days,
'seconds': obj.seconds,
'microseconds': obj.microseconds,
}
raise TypeError ("Type %s not serializable" % type(obj))
# helper function for displaying the runtime of tasks
def format_runtime(runtime):
retVal = ""
if runtime.days:
retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', '
mins, seconds = divmod(runtime.seconds, 60)
hours, minutes = divmod(mins, 60)
# ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ?
if hours:
retVal += '{:d}:{:02d}:{:02d}s'.format(hours, minutes, seconds)
elif minutes:
retVal += '{:2d}:{:02d}s'.format(minutes, seconds)
else:
retVal += '{:2d}s'.format(seconds)
return retVal
# helper function to apply localize status information in tasklist entries
def render_task_status(tasklist):
renderedtasklist=list()
......@@ -579,6 +605,8 @@ def render_task_status(tasklist):
if 'starttime' not in task:
task['starttime'] = ""
task['runtime'] = format_runtime(task['formRuntime'])
# localize the task status
if isinstance( task['stat'], int ):
if task['stat'] == worker.STAT_WAITING:
......
/* alphanum.js (C) Brian Huisman
* Based on the Alphanum Algorithm by David Koelle
* The Alphanum Algorithm is discussed at http://www.DaveKoelle.com
*
* Distributed under same license as original
*
* Released under the MIT License - https://opensource.org/licenses/MIT
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
/* ********************************************************************
* Alphanum sort() function version - case insensitive
* - Slower, but easier to modify for arrays of objects which contain
* string properties
*
*/
function alphanumCase(a, b) {
function chunkify(t) {
var tz = new Array();
var x = 0, y = -1, n = 0, i, j;
while (i = (j = t.charAt(x++)).charCodeAt(0)) {
var m = (i == 46 || (i >=48 && i <= 57));
if (m !== n) {
tz[++y] = "";
n = m;
}
tz[y] += j;
}
return tz;
}
var aa = chunkify(a.filename.toLowerCase());
var bb = chunkify(b.filename.toLowerCase());
for (x = 0; aa[x] && bb[x]; x++) {
if (aa[x] !== bb[x]) {
var c = Number(aa[x]), d = Number(bb[x]);
if (c == aa[x] && d == bb[x]) {
return c - d;
} else return (aa[x] > bb[x]) ? 1 : -1;
}
}
return aa.length - bb.length;
}
// ===========================================================================
/**
* archive.js
*
......
......@@ -1332,12 +1332,7 @@ var unrar = function(arrayBuffer) {
totalFilesInArchive = localFiles.length;
// now we have all information but things are unpacked
// TODO: unpack
localFiles = localFiles.sort(function(a, b) {
var aname = a.filename.toLowerCase();
var bname = b.filename.toLowerCase();
return aname > bname ? 1 : -1;
});
localFiles.sort(alphanumCase);
info(localFiles.map(function(a) {
return a.filename;
......
......@@ -115,6 +115,7 @@ var TarLocalFile = function(bstream) {
}
};
var untar = function(arrayBuffer) {
postMessage(new bitjs.archive.UnarchiveStartEvent());
currentFilename = "";
......@@ -127,14 +128,22 @@ var untar = function(arrayBuffer) {
var bstream = new bitjs.io.ByteStream(arrayBuffer);
postProgress();
// While we don't encounter an empty block, keep making TarLocalFiles.
/*
// go through whole file, read header of each block and memorize, filepointer
*/
while (bstream.peekNumber(4) !== 0) {
var oneLocalFile = new TarLocalFile(bstream);
var localFile = new TarLocalFile(bstream);
allLocalFiles.push(localFile);
postProgress();
}
// got all local files, now sort them
allLocalFiles.sort(alphanumCase);
allLocalFiles.forEach(function(oneLocalFile) {
// While we don't encounter an empty block, keep making TarLocalFiles.
if (oneLocalFile && oneLocalFile.isValid) {
// If we make it to this point and haven't thrown an error, we have successfully
// read in the data for a local file, so we can update the actual bytestream.
allLocalFiles.push(oneLocalFile);
totalUncompressedBytesInArchive += oneLocalFile.size;
// update progress
......@@ -145,7 +154,7 @@ var untar = function(arrayBuffer) {
postMessage(new bitjs.archive.UnarchiveExtractEvent(oneLocalFile));
postProgress();
}
}
});
totalFilesInArchive = allLocalFiles.length;
postProgress();
......
......@@ -72,23 +72,10 @@ var ZipLocalFile = function(bstream) {
this.filename = bstream.readString(this.fileNameLength);
}
info("Zip Local File Header:");
info(" version=" + this.version);
info(" general purpose=" + this.generalPurpose);
info(" compression method=" + this.compressionMethod);
info(" last mod file time=" + this.lastModFileTime);
info(" last mod file date=" + this.lastModFileDate);
info(" crc32=" + this.crc32);
info(" compressed size=" + this.compressedSize);
info(" uncompressed size=" + this.uncompressedSize);
info(" file name length=" + this.fileNameLength);
info(" extra field length=" + this.extraFieldLength);
info(" filename = '" + this.filename + "'");
this.extraField = null;
if (this.extraFieldLength > 0) {
this.extraField = bstream.readString(this.extraFieldLength);
info(" extra field=" + this.extraField);
this.extraField = bstream.readString(this.extraFieldLength);
info(" extra field=" + this.extraField);
}
// read in the compressed data
......@@ -107,6 +94,21 @@ var ZipLocalFile = function(bstream) {
this.compressedSize = bstream.readNumber(4);
this.uncompressedSize = bstream.readNumber(4);
}
// Now that we have all the bytes for this file, we can print out some information.
info("Zip Local File Header:");
info(" version=" + this.version);
info(" general purpose=" + this.generalPurpose);
info(" compression method=" + this.compressionMethod);
info(" last mod file time=" + this.lastModFileTime);
info(" last mod file date=" + this.lastModFileDate);
info(" crc32=" + this.crc32);
info(" compressed size=" + this.compressedSize);
info(" uncompressed size=" + this.uncompressedSize);
info(" file name length=" + this.fileNameLength);
info(" extra field length=" + this.extraFieldLength);
info(" filename = '" + this.filename + "'");
};
// determine what kind of compressed data we have and decompress
......@@ -132,6 +134,7 @@ ZipLocalFile.prototype.unzip = function() {
// Takes an ArrayBuffer of a zip file in
// returns null on error
// returns an array of DecompressedFile objects on success
// ToDo This function differs
var unzip = function(arrayBuffer) {
postMessage(new bitjs.archive.UnarchiveStartEvent());
......@@ -159,11 +162,7 @@ var unzip = function(arrayBuffer) {
totalFilesInArchive = localFiles.length;
// got all local files, now sort them
localFiles.sort(function(a, b) {
var aname = a.filename.toLowerCase();
var bname = b.filename.toLowerCase();
return aname > bname ? 1 : -1;
});
localFiles.sort(alphanumCase);
// archive extra data record
if (bstream.peekNumber(4) === zArchiveExtraDataSignature) {
......@@ -253,9 +252,9 @@ function getHuffmanCodes(bitLengths) {
}
// Reference: http://tools.ietf.org/html/rfc1951#page-8
var numLengths = bitLengths.length,
blCount = [],
MAX_BITS = 1;
var numLengths = bitLengths.length;
var blCount = [];
var MAX_BITS = 1;
// Step 1: count up how many codes of each length we have
for (var i = 0; i < numLengths; ++i) {
......@@ -274,8 +273,8 @@ function getHuffmanCodes(bitLengths) {
}
// Step 2: Find the numerical value of the smallest code for each code length
var nextCode = [],
code = 0;
var nextCode = [];
var code = 0;
for (var bits = 1; bits <= MAX_BITS; ++bits) {
var length2 = bits - 1;
// ensure undefined lengths are zero
......@@ -285,8 +284,8 @@ function getHuffmanCodes(bitLengths) {
}
// Step 3: Assign numerical values to all codes
var table = {},
tableLength = 0;
var table = {};
var tableLength = 0;
for (var n = 0; n < numLengths; ++n) {
var len = bitLengths[n];
if (len !== 0) {
......@@ -353,7 +352,8 @@ function getFixedDistanceTable() {
// extract one bit at a time until we find a matching Huffman Code
// then return that symbol
function decodeSymbol(bstream, hcTable) {
var code = 0, len = 0;
var code = 0;
var len = 0;
// loop until we match
for (;;) {
......@@ -364,7 +364,6 @@ function decodeSymbol(bstream, hcTable) {
// check against Huffman Code table and break if found
if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) {
break;
}
if (len > hcTable.maxLength) {
......@@ -500,10 +499,10 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
if (symbol === 256) {
break;
} else {
var lengthLookup = LengthLookupTable[symbol - 257],
length = lengthLookup[1] + bstream.readBits(lengthLookup[0]),
distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)],
distance = distLookup[1] + bstream.readBits(distLookup[0]);
var lengthLookup = LengthLookupTable[symbol - 257];
var length = lengthLookup[1] + bstream.readBits(lengthLookup[0]);
var distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)];
var distance = distLookup[1] + bstream.readBits(distLookup[0]);
// now apply length and distance appropriately and copy to output
......@@ -634,8 +633,8 @@ function inflate(compressedData, numDecompressedBytes) {
var distanceCodeLengths = literalCodeLengths.splice(numLiteralLengthCodes, numDistanceCodes);
// now generate the true Huffman Code tables using these code lengths
var hcLiteralTable = getHuffmanCodes(literalCodeLengths),
hcDistanceTable = getHuffmanCodes(distanceCodeLengths);
var hcLiteralTable = getHuffmanCodes(literalCodeLengths);
var hcDistanceTable = getHuffmanCodes(distanceCodeLengths);
blockSize = inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer);
} else {
// error
......
This diff is collapsed.
......@@ -3171,13 +3171,22 @@ def new_user():
return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
title=_(u"Add new user"))
content.password = generate_password_hash(to_save["password"])
content.nickname = to_save["nickname"]
if config.config_public_reg and not check_valid_domain(to_save["email"]):
flash(_(u"E-mail is not from valid domain"), category="error")
return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
title=_(u"Add new user"))
existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower())\
.first()
existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower())\
.first()
if not existing_user and not existing_email:
content.nickname = to_save["nickname"]
if config.config_public_reg and not check_valid_domain(to_save["email"]):
flash(_(u"E-mail is not from valid domain"), category="error")
return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
title=_(u"Add new user"))
else:
content.email = to_save["email"]
else:
content.email = to_save["email"]
flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
languages=languages, title=_(u"Add new user"), page="newuser")
try:
ub.session.add(content)
ub.session.commit()
......@@ -3362,14 +3371,24 @@ def edit_user(user_id):
if "locale" in to_save and to_save["locale"]:
content.locale = to_save["locale"]
if to_save["email"] and to_save["email"] != content.email:
content.email = to_save["email"]
existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
.first()
if not existing_email:
content.email = to_save["email"]
else:
flash(_(u"Found an existing account for this e-mail address."), category="error")
return render_title_template("user_edit.html", translations=translations, languages=languages,
new_user=0, content=content, downloads=downloads,
title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail:
content.kindle_mail = to_save["kindle_mail"]
try:
ub.session.commit()
flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success")
except IntegrityError:
except IntegrityError as e:
ub.session.rollback()
print(e)
flash(_(u"An unknown error occured."), category="error")
return render_title_template("user_edit.html", translations=translations, languages=languages, new_user=0,
content=content, downloads=downloads, title=_(u"Edit User %(nick)s",
......
......@@ -20,7 +20,7 @@
from __future__ import print_function
import smtplib
import threading
from datetime import datetime
from datetime import datetime, timedelta
import logging
import time
import socket
......@@ -221,8 +221,10 @@ class WorkerThread(threading.Thread):
if self.UIqueue[self.current]['stat'] == STAT_STARTED:
if self.queue[self.current]['taskType'] == TASK_EMAIL:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
self.UIqueue[self.current]['rt'] = self.UIqueue[self.current]['formRuntime'].days*24*60 \
+ self.UIqueue[self.current]['formRuntime'].seconds \
+ self.UIqueue[self.current]['formRuntime'].microseconds
return self.UIqueue
def _convert_any_format(self):
......@@ -259,7 +261,8 @@ class WorkerThread(threading.Thread):
self._handleSuccess()
return file_path + format_new_ext
else:
web.app.logger.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext)
web.app.logger.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
bookid, format_new_ext)
# check if converter-executable is existing
if not os.path.exists(web.ub.config.config_converterpath):
......@@ -300,7 +303,7 @@ class WorkerThread(threading.Thread):
if sys.version_info < (3, 0):
command = [x.encode(sys.getfilesystemencoding()) for x in command]
p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
except OSError as e:
self._handleError(_(u"Ebook-converter failed: %(error)s", error=e))
return
......@@ -328,6 +331,11 @@ class WorkerThread(threading.Thread):
# process returncode
check = p.returncode
calibre_traceback = p.stderr.readlines()
for ele in calibre_traceback:
web.app.logger.debug(ele.strip('\n'))
if not ele.startswith('Traceback') and not ele.startswith(' File'):
error_message = "Calibre failed with error: %s" % ele.strip('\n')
# kindlegen returncodes
# 0 = Info(prcgen):I1036: Mobi file built successfully
......@@ -481,31 +489,17 @@ class WorkerThread(threading.Thread):
self._handleError(u'Error sending email: ' + e.strerror)
return None
def _formatRuntime(self, runtime):
self.UIqueue[self.current]['rt'] = runtime.total_seconds()
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
def _handleError(self, error_message):
web.app.logger.error(error_message)
self.UIqueue[self.current]['stat'] = STAT_FAIL
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
self.UIqueue[self.current]['message'] = error_message
def _handleSuccess(self):
self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
# Enable logging of smtp lib debug output
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment