Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
75e66534
Commit
75e66534
authored
Aug 29, 2018
by
bodybybuddha
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
#584 & #587 Convert any book format to another format
parent
4f386c8a
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
201 additions
and
19 deletions
+201
-19
helper.py
cps/helper.py
+43
-3
web.py
cps/web.py
+21
-12
worker.py
cps/worker.py
+137
-4
No files found.
cps/helper.py
View file @
75e66534
...
@@ -136,9 +136,49 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id):
...
@@ -136,9 +136,49 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id):
# Convert existing book entry to new format
# Convert existing book entry to new format
#def convert_book_format(book_id, calibrepath, new_book_format, user_id):
def
convert_book_format
(
book_id
,
calibrepath
,
old_book_format
,
new_book_format
,
user_id
):
book
=
db
.
session
.
query
(
db
.
Books
)
.
filter
(
db
.
Books
.
id
==
book_id
)
.
first
()
# return
data
=
db
.
session
.
query
(
db
.
Data
)
.
filter
(
db
.
Data
.
book
==
book
.
id
)
.
filter
(
db
.
Data
.
format
==
old_book_format
)
.
first
()
if
not
data
:
error_message
=
_
(
u"
%(format)
s format not found for book id:
%(book)
d"
,
format
=
old_book_format
,
book
=
book_id
)
app
.
logger
.
error
(
"convert_book_format: "
+
error_message
)
return
error_message
if
ub
.
config
.
config_use_google_drive
:
df
=
gd
.
getFileFromEbooksFolder
(
book
.
path
,
data
.
name
+
"."
+
old_book_format
.
lower
())
if
df
:
datafile
=
os
.
path
.
join
(
calibrepath
,
book
.
path
,
data
.
name
+
"."
+
old_book_format
.
lower
())
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
calibrepath
,
book
.
path
)):
os
.
makedirs
(
os
.
path
.
join
(
calibrepath
,
book
.
path
))
df
.
GetContentFile
(
datafile
)
else
:
error_message
=
_
(
u"convert_book_format:
%(format)
s not found on gdrive:
%(fn)
s"
,
format
=
old_book_format
,
fn
=
data
.
name
+
"."
+
old_book_format
.
lower
())
return
error_message
file_path
=
os
.
path
.
join
(
calibrepath
,
book
.
path
,
data
.
name
)
if
os
.
path
.
exists
(
file_path
+
"."
+
old_book_format
.
lower
()):
# append converter to queue
settings
=
{
'old_book_format'
:
old_book_format
,
'new_book_format'
:
new_book_format
}
app
.
logger
.
debug
(
"Creating worker thread:"
)
app
.
logger
.
debug
(
"filepath: "
+
file_path
+
" "
+
"bookid: "
+
str
(
book
.
id
)
+
" "
+
"userid: "
+
str
(
user_id
)
+
" "
+
"taskmsg: "
+
_
(
u"Convert to
%(format)
s:
%(book)
s"
,
format
=
new_book_format
,
book
=
book
.
title
)
+
" "
+
"settings:old_book_format: "
+
settings
[
'old_book_format'
]
+
" "
+
"settings:new_book_format: "
+
settings
[
'new_book_format'
]
)
global_WorkerThread
.
add_convert_any
(
file_path
,
book
.
id
,
user_id
,
_
(
u"Convert to
%(format)
s:
%(book)
s"
,
format
=
new_book_format
,
book
=
book
.
title
),
settings
)
return
None
else
:
error_message
=
_
(
u"convert_book_format:
%(format)
s not found:
%(fn)
s"
,
format
=
old_book_format
,
fn
=
data
.
name
+
"."
+
old_book_format
.
lower
())
return
error_message
def
get_valid_filename
(
value
,
replace_whitespace
=
True
):
def
get_valid_filename
(
value
,
replace_whitespace
=
True
):
"""
"""
...
...
cps/web.py
View file @
75e66534
...
@@ -3177,18 +3177,20 @@ def edit_book(book_id):
...
@@ -3177,18 +3177,20 @@ def edit_book(book_id):
for
authr
in
book
.
authors
:
for
authr
in
book
.
authors
:
author_names
.
append
(
authr
.
name
.
replace
(
'|'
,
','
))
author_names
.
append
(
authr
.
name
.
replace
(
'|'
,
','
))
#Option for showing convertbook button
#
Option for showing convertbook button
if
config
.
config_ebookconverter
==
2
:
if
config
.
config_ebookconverter
==
2
:
display_convertbtn
=
True
display_convertbtn
=
True
else
:
else
:
display_convertbtn
=
False
display_convertbtn
=
False
#Determine what formats don't already exist
# Determine what formats don't already exist
allowed_conversion_formats
=
ALLOWED_EXTENSIONS
allowed_conversion_formats
=
ALLOWED_EXTENSIONS
.
copy
()
for
file
in
book
.
data
:
allowed_conversion_formats
.
remove
(
file
.
format
.
lower
())
app
.
logger
.
debug
(
allowed_conversion_formats
)
for
file
in
book
.
data
:
try
:
allowed_conversion_formats
.
remove
(
file
.
format
.
lower
())
except
Exception
:
app
.
logger
.
warning
(
file
.
format
.
lower
()
+
' already removed from list.'
)
# Show form
# Show form
if
request
.
method
!=
'POST'
:
if
request
.
method
!=
'POST'
:
...
@@ -3626,11 +3628,18 @@ def upload():
...
@@ -3626,11 +3628,18 @@ def upload():
@
login_required_if_no_ano
@
login_required_if_no_ano
@
edit_required
@
edit_required
def
convert_bookformat
(
book_id
):
def
convert_bookformat
(
book_id
):
# rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id)
app
.
logger
.
debug
(
'converting: book id: '
+
str
(
book_id
)
+
app
.
logger
.
debug
(
'got here'
)
' from: '
+
request
.
form
[
'book_format_from'
]
+
app
.
logger
.
debug
(
'book id:'
+
str
(
book_id
))
' to: '
+
request
.
form
[
'book_format_to'
])
app
.
logger
.
debug
(
'from format:'
+
request
.
form
[
'book_format_from'
])
rtn
=
helper
.
convert_book_format
(
book_id
,
config
.
config_calibre_dir
,
request
.
form
[
'book_format_from'
]
.
upper
(),
app
.
logger
.
debug
(
'to format:'
+
request
.
form
[
'book_format_to'
])
request
.
form
[
'book_format_to'
]
.
upper
(),
current_user
.
nickname
)
if
rtn
is
None
:
flash
(
_
(
u"Book successfully queued for converting to
%(book_format)
s"
,
book_format
=
request
.
form
[
'book_format_from'
]),
category
=
"success"
)
else
:
flash
(
_
(
u"There was an error sending this book:
%(res)
s"
,
res
=
rtn
),
category
=
"error"
)
return
redirect
(
url_for
(
"
index
"
))
return
redirect
(
url_for
(
"
get_tasks_status
"
))
cps/worker.py
View file @
75e66534
...
@@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3
...
@@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3
TASK_EMAIL
=
1
TASK_EMAIL
=
1
TASK_CONVERT
=
2
TASK_CONVERT
=
2
TASK_UPLOAD
=
3
TASK_UPLOAD
=
3
TASK_CONVERT_ANY
=
4
RET_FAIL
=
0
RET_FAIL
=
0
RET_SUCCESS
=
1
RET_SUCCESS
=
1
...
@@ -172,6 +173,8 @@ class WorkerThread(threading.Thread):
...
@@ -172,6 +173,8 @@ class WorkerThread(threading.Thread):
self
.
send_raw_email
()
self
.
send_raw_email
()
if
self
.
queue
[
self
.
current
][
'typ'
]
==
TASK_CONVERT
:
if
self
.
queue
[
self
.
current
][
'typ'
]
==
TASK_CONVERT
:
self
.
convert_mobi
()
self
.
convert_mobi
()
if
self
.
queue
[
self
.
current
][
'typ'
]
==
TASK_CONVERT_ANY
:
self
.
convert_any_format
()
# TASK_UPLOAD is handled implicitly
# TASK_UPLOAD is handled implicitly
self
.
current
+=
1
self
.
current
+=
1
else
:
else
:
...
@@ -308,7 +311,6 @@ class WorkerThread(threading.Thread):
...
@@ -308,7 +311,6 @@ class WorkerThread(threading.Thread):
self
.
_handleError
(
error_message
)
self
.
_handleError
(
error_message
)
return
return
def
add_convert
(
self
,
file_path
,
bookid
,
user_name
,
typ
,
settings
,
kindle_mail
):
def
add_convert
(
self
,
file_path
,
bookid
,
user_name
,
typ
,
settings
,
kindle_mail
):
addLock
=
threading
.
Lock
()
addLock
=
threading
.
Lock
()
addLock
.
acquire
()
addLock
.
acquire
()
...
@@ -325,7 +327,6 @@ class WorkerThread(threading.Thread):
...
@@ -325,7 +327,6 @@ class WorkerThread(threading.Thread):
self
.
last
=
len
(
self
.
queue
)
self
.
last
=
len
(
self
.
queue
)
addLock
.
release
()
addLock
.
release
()
def
add_email
(
self
,
subject
,
filepath
,
attachment
,
settings
,
recipient
,
user_name
,
typ
,
def
add_email
(
self
,
subject
,
filepath
,
attachment
,
settings
,
recipient
,
user_name
,
typ
,
text
=
_
(
u'This e-mail has been sent via Calibre-Web.'
)):
text
=
_
(
u'This e-mail has been sent via Calibre-Web.'
)):
# if more than 20 entries in the list, clean the list
# if more than 20 entries in the list, clean the list
...
@@ -357,8 +358,140 @@ class WorkerThread(threading.Thread):
...
@@ -357,8 +358,140 @@ class WorkerThread(threading.Thread):
self
.
id
+=
1
self
.
id
+=
1
self
.
last
=
len
(
self
.
queue
)
self
.
last
=
len
(
self
.
queue
)
addLock
.
release
()
addLock
.
release
()
def
convert_any_format
(
self
):
# convert book, and upload in case of google drive
self
.
queue
[
self
.
current
][
'status'
]
=
STAT_STARTED
self
.
UIqueue
[
self
.
current
][
'status'
]
=
_
(
'Started'
)
self
.
queue
[
self
.
current
][
'starttime'
]
=
datetime
.
now
()
self
.
UIqueue
[
self
.
current
][
'formStarttime'
]
=
self
.
queue
[
self
.
current
][
'starttime'
]
filename
=
self
.
convert_ebook_format
()
if
web
.
ub
.
config
.
config_use_google_drive
:
gd
.
updateGdriveCalibreFromLocal
()
def
add_convert_any
(
self
,
file_path
,
bookid
,
user_name
,
typ
,
settings
):
addLock
=
threading
.
Lock
()
addLock
.
acquire
()
if
self
.
last
>=
20
:
self
.
delete_completed_tasks
()
# progress, runtime, and status = 0
self
.
id
+=
1
self
.
queue
.
append
({
'file_path'
:
file_path
,
'bookid'
:
bookid
,
'starttime'
:
0
,
'status'
:
STAT_WAITING
,
'typ'
:
TASK_CONVERT_ANY
,
'settings'
:
settings
})
self
.
UIqueue
.
append
({
'user'
:
user_name
,
'formStarttime'
:
''
,
'progress'
:
" 0
%
"
,
'type'
:
typ
,
'runtime'
:
'0 s'
,
'status'
:
_
(
'Waiting'
),
'id'
:
self
.
id
}
)
self
.
id
+=
1
self
.
last
=
len
(
self
.
queue
)
addLock
.
release
()
def
convert_ebook_format
(
self
):
error_message
=
None
file_path
=
self
.
queue
[
self
.
current
][
'file_path'
]
bookid
=
self
.
queue
[
self
.
current
][
'bookid'
]
format_old_ext
=
u'.'
+
self
.
queue
[
self
.
current
][
'settings'
][
'old_book_format'
]
.
lower
()
format_new_ext
=
u'.'
+
self
.
queue
[
self
.
current
][
'settings'
][
'new_book_format'
]
.
lower
()
# check if converter-executable is existing
if
not
os
.
path
.
exists
(
web
.
ub
.
config
.
config_converterpath
):
self
.
_handleError
(
_
(
u"Convertertool
%(converter)
s not found"
,
converter
=
web
.
ub
.
config
.
config_converterpath
))
return
try
:
# check which converter to use kindlegen is "1" //backward compatibility assumption epub original format
if
web
.
ub
.
config
.
config_ebookconverter
==
1
:
command
=
[
web
.
ub
.
config
.
config_converterpath
,
u'"'
+
file_path
+
u'.epub"'
]
else
:
# Linux py2.7 encode as list without quotes no empty element for parameters
# linux py3.x no encode and as list without quotes no empty element for parameters
# windows py2.7 encode as string with quotes empty element for parameters is okay
# windows py 3.x no encode and as string with quotes empty element for parameters is okay
# separate handling for windows and linux
if
os
.
name
==
'nt'
:
command
=
web
.
ub
.
config
.
config_converterpath
+
u' "'
+
file_path
+
format_old_ext
+
u'" "'
+
\
file_path
+
format_new_ext
+
u'" '
+
web
.
ub
.
config
.
config_calibre
if
sys
.
version_info
<
(
3
,
0
):
command
=
command
.
encode
(
sys
.
getfilesystemencoding
())
else
:
command
=
[
web
.
ub
.
config
.
config_converterpath
,
(
file_path
+
format_old_ext
),
(
file_path
+
format_new_ext
)]
if
web
.
ub
.
config
.
config_calibre
:
command
.
append
(
web
.
ub
.
config
.
config_calibre
)
if
sys
.
version_info
<
(
3
,
0
):
command
=
[
x
.
encode
(
sys
.
getfilesystemencoding
())
for
x
in
command
]
p
=
subprocess
.
Popen
(
command
,
stdout
=
subprocess
.
PIPE
,
universal_newlines
=
True
)
except
OSError
as
e
:
self
.
_handleError
(
_
(
u"Ebook-converter failed:
%
s"
%
e
))
return
if
web
.
ub
.
config
.
config_ebookconverter
==
1
:
nextline
=
p
.
communicate
()[
0
]
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error
=
re
.
search
(
".*
\
(.*
\
):(E
\
d+):
\
s(.*)"
,
nextline
,
re
.
MULTILINE
)
# If error occoures, store error message for logfile
if
conv_error
:
error_message
=
_
(
u"Kindlegen failed with Error
%(error)
s. Message:
%(message)
s"
,
error
=
conv_error
.
group
(
1
),
message
=
conv_error
.
group
(
2
)
.
strip
())
web
.
app
.
logger
.
debug
(
"convert_kindlegen: "
+
nextline
)
else
:
while
p
.
poll
()
is
None
:
nextline
=
p
.
stdout
.
readline
()
if
os
.
name
==
'nt'
and
sys
.
version_info
<
(
3
,
0
):
nextline
=
nextline
.
decode
(
'windows-1252'
)
web
.
app
.
logger
.
debug
(
nextline
.
strip
(
'
\r\n
'
))
# parse progress string from calibre-converter
progress
=
re
.
search
(
"(
\
d+)
%
\
s.*"
,
nextline
)
if
progress
:
self
.
UIqueue
[
self
.
current
][
'progress'
]
=
progress
.
group
(
1
)
+
'
%
'
#process returncode
check
=
p
.
returncode
# kindlegen returncodes
# 0 = Info(prcgen):I1036: Mobi file built successfully
# 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
# 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
if
(
check
<
2
and
web
.
ub
.
config
.
config_ebookconverter
==
1
)
or
\
(
check
==
0
and
web
.
ub
.
config
.
config_ebookconverter
==
2
):
cur_book
=
web
.
db
.
session
.
query
(
web
.
db
.
Books
)
.
filter
(
web
.
db
.
Books
.
id
==
bookid
)
.
first
()
if
web
.
ub
.
config
.
config_ebookconverter
==
1
:
new_format
=
web
.
db
.
Data
(
name
=
cur_book
.
data
[
0
]
.
name
,
book_format
=
self
.
queue
[
self
.
current
][
'settings'
][
'new_book_format'
],
book
=
bookid
,
uncompressed_size
=
os
.
path
.
getsize
(
file_path
+
'.mobi'
))
else
:
new_format
=
web
.
db
.
Data
(
name
=
cur_book
.
data
[
0
]
.
name
,
book_format
=
self
.
queue
[
self
.
current
][
'settings'
][
'new_book_format'
],
book
=
bookid
,
uncompressed_size
=
os
.
path
.
getsize
(
file_path
+
format_new_ext
))
cur_book
.
data
.
append
(
new_format
)
web
.
db
.
session
.
commit
()
self
.
queue
[
self
.
current
][
'path'
]
=
cur_book
.
path
self
.
queue
[
self
.
current
][
'title'
]
=
cur_book
.
title
self
.
queue
[
self
.
current
][
'status'
]
=
STAT_FINISH_SUCCESS
self
.
UIqueue
[
self
.
current
][
'status'
]
=
_
(
'Finished'
)
self
.
UIqueue
[
self
.
current
][
'progress'
]
=
"100
%
"
self
.
UIqueue
[
self
.
current
][
'runtime'
]
=
self
.
_formatRuntime
(
datetime
.
now
()
-
self
.
queue
[
self
.
current
][
'starttime'
])
if
web
.
ub
.
config
.
config_ebookconverter
==
1
:
if
web
.
ub
.
config
.
config_use_google_drive
:
os
.
remove
(
file_path
+
u".epub"
)
return
file_path
+
'.mobi'
else
:
if
web
.
ub
.
config
.
config_use_google_drive
:
os
.
remove
(
file_path
+
format_old_ext
)
return
file_path
+
format_new_ext
else
:
web
.
app
.
logger
.
info
(
"ebook converter failed with error while converting book"
)
if
not
error_message
:
error_message
=
'Ebook converter failed with unknown error'
self
.
_handleError
(
error_message
)
return
def
send_raw_email
(
self
):
def
send_raw_email
(
self
):
self
.
queue
[
self
.
current
][
'starttime'
]
=
datetime
.
now
()
self
.
queue
[
self
.
current
][
'starttime'
]
=
datetime
.
now
()
self
.
UIqueue
[
self
.
current
][
'formStarttime'
]
=
self
.
queue
[
self
.
current
][
'starttime'
]
self
.
UIqueue
[
self
.
current
][
'formStarttime'
]
=
self
.
queue
[
self
.
current
][
'starttime'
]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment