Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
63634961
Commit
63634961
authored
Jul 14, 2019
by
Daniel Pavel
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
cleaner worker api
the worker thread now stops on its own
parent
a836df9a
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
66 additions
and
47 deletions
+66
-47
__init__.py
cps/__init__.py
+2
-8
editbooks.py
cps/editbooks.py
+5
-5
helper.py
cps/helper.py
+19
-18
server.py
cps/server.py
+1
-2
web.py
cps/web.py
+4
-4
worker.py
cps/worker.py
+35
-10
No files found.
cps/__init__.py
View file @
63634961
...
...
@@ -34,8 +34,9 @@ from flask_login import LoginManager
from
flask_babel
import
Babel
from
flask_principal
import
Principal
from
.
import
logger
,
cache_buster
,
cli
,
config_sql
,
ub
from
.
import
logger
,
cache_buster
,
cli
,
config_sql
,
ub
,
db
,
services
from
.reverseproxy
import
ReverseProxied
from
.server
import
WebServer
mimetypes
.
init
()
...
...
@@ -66,14 +67,8 @@ lm.anonymous_user = ub.Anonymous
ub
.
init_db
(
cli
.
settingspath
)
# pylint: disable=no-member
config
=
config_sql
.
load_configuration
(
ub
.
session
)
from
.
import
db
,
services
searched_ids
=
{}
from
.worker
import
WorkerThread
global_WorkerThread
=
WorkerThread
()
from
.server
import
WebServer
web_server
=
WebServer
()
babel
=
Babel
()
...
...
@@ -109,7 +104,6 @@ def create_app():
if
services
.
goodreads
:
services
.
goodreads
.
connect
(
config
.
config_goodreads_api_key
,
config
.
config_goodreads_api_secret
,
config
.
config_use_goodreads
)
global_WorkerThread
.
start
()
return
app
@
babel
.
localeselector
...
...
cps/editbooks.py
View file @
63634961
...
...
@@ -30,12 +30,12 @@ from uuid import uuid4
from
flask
import
Blueprint
,
request
,
flash
,
redirect
,
url_for
,
abort
,
Markup
,
Response
from
flask_babel
import
gettext
as
_
from
flask_login
import
current_user
from
flask_login
import
current_user
,
login_required
from
.
import
constants
,
logger
,
isoLanguages
,
gdriveutils
,
uploader
,
helper
from
.
import
config
,
get_locale
,
db
,
ub
,
global_WorkerThread
from
.
import
config
,
get_locale
,
db
,
ub
,
worker
from
.helper
import
order_authors
,
common_filters
from
.web
import
login_required_if_no_ano
,
render_title_template
,
edit_required
,
upload_required
,
login_required
from
.web
import
login_required_if_no_ano
,
render_title_template
,
edit_required
,
upload_required
editbook
=
Blueprint
(
'editbook'
,
__name__
)
...
...
@@ -358,7 +358,7 @@ def upload_single_file(request, book, book_id):
# Queue uploader info
uploadText
=
_
(
u"File format
%(ext)
s added to
%(book)
s"
,
ext
=
file_ext
.
upper
(),
book
=
book
.
title
)
global_WorkerThread
.
add_upload
(
current_user
.
nickname
,
worker
.
add_upload
(
current_user
.
nickname
,
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book
.
id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
...
...
@@ -667,7 +667,7 @@ def upload():
if
error
:
flash
(
error
,
category
=
"error"
)
uploadText
=
_
(
u"File
%(file)
s uploaded"
,
file
=
book
.
title
)
global_WorkerThread
.
add_upload
(
current_user
.
nickname
,
worker
.
add_upload
(
current_user
.
nickname
,
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book
.
id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
# create data for displaying display Full language name instead of iso639.part3language
...
...
cps/helper.py
View file @
63634961
...
...
@@ -60,7 +60,7 @@ try:
except
ImportError
:
use_PIL
=
False
from
.
import
logger
,
config
,
g
lobal_WorkerThread
,
get_locale
,
db
,
ub
,
isoLanguages
from
.
import
logger
,
config
,
g
et_locale
,
db
,
ub
,
isoLanguages
,
worker
from
.
import
gdriveutils
as
gd
from
.constants
import
STATIC_DIR
as
_STATIC_DIR
from
.pagination
import
Pagination
...
...
@@ -112,7 +112,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
text
=
(
u"
%
s ->
%
s:
%
s"
%
(
old_book_format
,
new_book_format
,
book
.
title
))
settings
[
'old_book_format'
]
=
old_book_format
settings
[
'new_book_format'
]
=
new_book_format
global_WorkerThread
.
add_convert
(
file_path
,
book
.
id
,
user_id
,
text
,
settings
,
kindle_mail
)
worker
.
add_convert
(
file_path
,
book
.
id
,
user_id
,
text
,
settings
,
kindle_mail
)
return
None
else
:
error_message
=
_
(
u"
%(format)
s not found:
%(fn)
s"
,
...
...
@@ -121,9 +121,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
def
send_test_mail
(
kindle_mail
,
user_name
):
global_WorkerThread
.
add_email
(
_
(
u'Calibre-Web test e-mail'
),
None
,
None
,
config
.
get_mail_settings
()
,
kindle_mail
,
user_name
,
_
(
u"Test e-mail"
)
,
_
(
u'This e-mail has been sent via Calibre-Web.'
))
worker
.
add_email
(
_
(
u'Calibre-Web test e-mail'
),
None
,
None
,
config
.
get_mail_settings
(),
kindle_mail
,
user_name
,
_
(
u"Test e-mail"
),
_
(
u'This e-mail has been sent via Calibre-Web.'
))
return
...
...
@@ -138,8 +138,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
text
+=
"Don't forget to change your password after first login.
\r\n
"
text
+=
"Sincerely
\r\n\r\n
"
text
+=
"Your Calibre-Web team"
global_WorkerThread
.
add_email
(
_
(
u'Get Started with Calibre-Web'
),
None
,
None
,
config
.
get_mail_settings
(),
e_mail
,
None
,
_
(
u"Registration e-mail for user:
%(name)
s"
,
name
=
user_name
),
text
)
worker
.
add_email
(
_
(
u'Get Started with Calibre-Web'
),
None
,
None
,
config
.
get_mail_settings
(),
e_mail
,
None
,
_
(
u"Registration e-mail for user:
%(name)
s"
,
name
=
user_name
),
text
)
return
...
...
@@ -207,15 +208,15 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
if
convert
:
# returns None if success, otherwise errormessage
return
convert_book_format
(
book_id
,
calibrepath
,
u'epub'
,
book_format
.
lower
(),
user_id
,
kindle_mail
)
else
:
for
entry
in
iter
(
book
.
data
):
if
entry
.
format
.
upper
()
==
book_format
.
upper
():
result
=
entry
.
name
+
'.'
+
book_format
.
lower
()
global_WorkerThread
.
add_email
(
_
(
u"Send to Kindle"
),
book
.
path
,
result
,
config
.
get_mail_settings
()
,
kindle_mail
,
user_id
,
_
(
u"E-mail:
%(book)
s"
,
book
=
book
.
title
)
,
_
(
u'This e-mail has been sent via Calibre-Web.'
))
return
return
_
(
u"The requested file could not be read. Maybe wrong permissions?"
)
for
entry
in
iter
(
book
.
data
):
if
entry
.
format
.
upper
()
==
book_format
.
upper
():
converted_file_name
=
entry
.
name
+
'.'
+
book_format
.
lower
()
worker
.
add_email
(
_
(
u"Send to Kindle"
),
book
.
path
,
converted_file_name
,
config
.
get_mail_settings
(),
kindle_mail
,
user_id
,
_
(
u"E-mail:
%(book)
s"
,
book
=
book
.
title
),
_
(
u'This e-mail has been sent via Calibre-Web.'
))
return
return
_
(
u"The requested file could not be read. Maybe wrong permissions?"
)
def
get_valid_filename
(
value
,
replace_whitespace
=
True
):
...
...
@@ -232,7 +233,7 @@ def get_valid_filename(value, replace_whitespace=True):
value
=
value
.
replace
(
u'§'
,
u'SS'
)
value
=
value
.
replace
(
u'ß'
,
u'ss'
)
value
=
unicodedata
.
normalize
(
'NFKD'
,
value
)
re_slugify
=
re
.
compile
(
'[
\
W
\
s-]'
,
re
.
UNICODE
)
re_slugify
=
re
.
compile
(
r
'[\W\s-]'
,
re
.
UNICODE
)
if
isinstance
(
value
,
str
):
# Python3 str, Python2 unicode
value
=
re_slugify
.
sub
(
''
,
value
)
.
strip
()
else
:
...
...
@@ -254,7 +255,7 @@ def get_valid_filename(value, replace_whitespace=True):
def
get_sorted_author
(
value
):
try
:
if
','
not
in
value
:
regexes
=
[
"^(JR|SR)
\
.?$"
,
"^I{1,3}
\
.?$"
,
"^IV
\
.?$"
]
regexes
=
[
r"^(JR|SR)\.?$"
,
r"^I{1,3}\.?$"
,
r
"^IV\.?$"
]
combined
=
"("
+
")|("
.
join
(
regexes
)
+
")"
value
=
value
.
split
(
" "
)
if
re
.
match
(
combined
,
value
[
-
1
]
.
upper
()):
...
...
cps/server.py
View file @
63634961
...
...
@@ -38,7 +38,7 @@ except ImportError:
VERSION
=
{
'Tornado'
:
'v'
+
_version
}
_GEVENT
=
False
from
.
import
logger
,
global_WorkerThread
from
.
import
logger
log
=
logger
.
create
()
...
...
@@ -179,7 +179,6 @@ class WebServer:
return
False
finally
:
self
.
wsgiserver
=
None
global_WorkerThread
.
stop
()
if
not
self
.
restart
:
log
.
info
(
"Performing shutdown of Calibre-Web"
)
...
...
cps/web.py
View file @
63634961
...
...
@@ -41,8 +41,8 @@ from werkzeug.exceptions import default_exceptions
from
werkzeug.datastructures
import
Headers
from
werkzeug.security
import
generate_password_hash
,
check_password_hash
from
.
import
constants
,
logger
,
isoLanguages
,
services
from
.
import
global_WorkerThread
,
searched_ids
,
lm
,
babel
,
db
,
ub
,
config
,
negociate_locale
,
get_locale
,
app
from
.
import
constants
,
logger
,
isoLanguages
,
services
,
worker
from
.
import
searched_ids
,
lm
,
babel
,
db
,
ub
,
config
,
negociate_locale
,
get_locale
,
app
from
.gdriveutils
import
getFileFromEbooksFolder
,
do_gdrive_download
from
.helper
import
common_filters
,
get_search_results
,
fill_indexpage
,
speaking_language
,
check_valid_domain
,
\
order_authors
,
get_typeahead
,
render_task_status
,
json_serial
,
get_cc_columns
,
\
...
...
@@ -245,7 +245,7 @@ def before_request():
@
web
.
route
(
"/ajax/emailstat"
)
@
login_required
def
get_email_status_json
():
tasks
=
global_WorkerThread
.
get_taskstatus
()
tasks
=
worker
.
get_taskstatus
()
answer
=
render_task_status
(
tasks
)
js
=
json
.
dumps
(
answer
,
default
=
json_serial
)
response
=
make_response
(
js
)
...
...
@@ -760,7 +760,7 @@ def category_list():
@
login_required
def
get_tasks_status
():
# if current user admin, show all email, otherwise only own emails
tasks
=
global_WorkerThread
.
get_taskstatus
()
tasks
=
worker
.
get_taskstatus
()
answer
=
render_task_status
(
tasks
)
return
render_title_template
(
'tasks.html'
,
entries
=
answer
,
title
=
_
(
u"Tasks"
),
page
=
"tasks"
)
...
...
cps/worker.py
View file @
63634961
...
...
@@ -25,7 +25,7 @@ import smtplib
import
socket
import
time
import
threading
from
datetime
import
datetime
,
timedelta
from
datetime
import
datetime
try
:
from
StringIO
import
StringIO
...
...
@@ -66,6 +66,13 @@ RET_FAIL = 0
RET_SUCCESS
=
1
def
_get_main_thread
():
for
t
in
threading
.
enumerate
():
if
t
.
__class__
.
__name__
==
'_MainThread'
:
return
t
raise
Exception
(
"main thread not found?!"
)
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to
def
get_attachment
(
bookpath
,
filename
):
...
...
@@ -173,19 +180,19 @@ class email_SSL(emailbase, smtplib.SMTP_SSL):
class
WorkerThread
(
threading
.
Thread
):
def
__init__
(
self
):
self
.
_stopevent
=
threading
.
Event
()
threading
.
Thread
.
__init__
(
self
)
self
.
status
=
0
self
.
current
=
0
self
.
last
=
0
self
.
queue
=
list
()
self
.
UIqueue
=
list
()
self
.
asyncSMTP
=
None
self
.
asyncSMTP
=
None
self
.
id
=
0
# Main thread loop starting the different tasks
def
run
(
self
):
while
not
self
.
_stopevent
.
isSet
():
main_thread
=
_get_main_thread
()
while
main_thread
.
is_alive
():
doLock
=
threading
.
Lock
()
doLock
.
acquire
()
if
self
.
current
!=
self
.
last
:
...
...
@@ -200,10 +207,8 @@ class WorkerThread(threading.Thread):
self
.
current
+=
1
else
:
doLock
.
release
()
time
.
sleep
(
1
)
def
stop
(
self
):
self
.
_stopevent
.
set
()
if
main_thread
.
is_alive
():
time
.
sleep
(
1
)
def
get_send_status
(
self
):
if
self
.
asyncSMTP
:
...
...
@@ -317,7 +322,7 @@ class WorkerThread(threading.Thread):
nextline
=
p
.
communicate
()[
0
]
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error
=
re
.
search
(
".*
\
(.*
\
):(E
\
d+):
\
s(.*)"
,
nextline
,
re
.
MULTILINE
)
conv_error
=
re
.
search
(
r
".*\(.*\):(E\d+):\s(.*)"
,
nextline
,
re
.
MULTILINE
)
# If error occoures, store error message for logfile
if
conv_error
:
error_message
=
_
(
u"Kindlegen failed with Error
%(error)
s. Message:
%(message)
s"
,
...
...
@@ -332,7 +337,7 @@ class WorkerThread(threading.Thread):
nextline
=
nextline
.
decode
(
'utf-8'
)
log
.
debug
(
nextline
.
strip
(
'
\r\n
'
))
# parse progress string from calibre-converter
progress
=
re
.
search
(
"(
\
d+)
%
\
s.*"
,
nextline
)
progress
=
re
.
search
(
r
"(\d+)
%
\s.*"
,
nextline
)
if
progress
:
self
.
UIqueue
[
self
.
current
][
'progress'
]
=
progress
.
group
(
1
)
+
'
%
'
...
...
@@ -511,3 +516,23 @@ class WorkerThread(threading.Thread):
self
.
UIqueue
[
self
.
current
][
'stat'
]
=
STAT_FINISH_SUCCESS
self
.
UIqueue
[
self
.
current
][
'progress'
]
=
"100
%
"
self
.
UIqueue
[
self
.
current
][
'formRuntime'
]
=
datetime
.
now
()
-
self
.
queue
[
self
.
current
][
'starttime'
]
_worker
=
WorkerThread
()
_worker
.
start
()
def
get_taskstatus
():
return
_worker
.
get_taskstatus
()
def
add_email
(
subject
,
filepath
,
attachment
,
settings
,
recipient
,
user_name
,
taskMessage
,
text
):
return
_worker
.
add_email
(
subject
,
filepath
,
attachment
,
settings
,
recipient
,
user_name
,
taskMessage
,
text
)
def
add_upload
(
user_name
,
taskMessage
):
return
_worker
.
add_upload
(
user_name
,
taskMessage
)
def
add_convert
(
file_path
,
bookid
,
user_name
,
taskMessage
,
settings
,
kindle_mail
=
None
):
return
_worker
.
add_convert
(
file_path
,
bookid
,
user_name
,
taskMessage
,
settings
,
kindle_mail
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment