Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
2533c9c1
Commit
2533c9c1
authored
Aug 23, 2020
by
blitzmann
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Continue converting tasks - email and upload tasks
parent
f10f0dad
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
268 additions
and
20 deletions
+268
-20
editbooks.py
cps/editbooks.py
+6
-4
helper.py
cps/helper.py
+9
-8
worker.py
cps/services/worker.py
+8
-6
email.py
cps/tasks/email.py
+221
-0
upload.py
cps/tasks/upload.py
+18
-0
web.py
cps/web.py
+6
-2
No files found.
cps/editbooks.py
View file @
2533c9c1
...
@@ -35,6 +35,8 @@ from sqlalchemy.exc import OperationalError
...
@@ -35,6 +35,8 @@ from sqlalchemy.exc import OperationalError
from
.
import
constants
,
logger
,
isoLanguages
,
gdriveutils
,
uploader
,
helper
from
.
import
constants
,
logger
,
isoLanguages
,
gdriveutils
,
uploader
,
helper
from
.
import
config
,
get_locale
,
ub
,
worker
,
db
from
.
import
config
,
get_locale
,
ub
,
worker
,
db
from
.
import
calibre_db
from
.
import
calibre_db
from
.services.worker
import
WorkerThread
from
.tasks.upload
import
TaskUpload
from
.web
import
login_required_if_no_ano
,
render_title_template
,
edit_required
,
upload_required
from
.web
import
login_required_if_no_ano
,
render_title_template
,
edit_required
,
upload_required
...
@@ -509,8 +511,8 @@ def upload_single_file(request, book, book_id):
...
@@ -509,8 +511,8 @@ def upload_single_file(request, book, book_id):
# Queue uploader info
# Queue uploader info
uploadText
=
_
(
u"File format
%(ext)
s added to
%(book)
s"
,
ext
=
file_ext
.
upper
(),
book
=
book
.
title
)
uploadText
=
_
(
u"File format
%(ext)
s added to
%(book)
s"
,
ext
=
file_ext
.
upper
(),
book
=
book
.
title
)
worker
.
add_upload
(
current_user
.
nickname
,
WorkerThread
.
add
(
current_user
.
nickname
,
TaskUpload
(
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book
.
id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book
.
id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
)
return
uploader
.
process
(
return
uploader
.
process
(
saved_filename
,
*
os
.
path
.
splitext
(
requested_file
.
filename
),
saved_filename
,
*
os
.
path
.
splitext
(
requested_file
.
filename
),
...
@@ -854,8 +856,8 @@ def upload():
...
@@ -854,8 +856,8 @@ def upload():
if
error
:
if
error
:
flash
(
error
,
category
=
"error"
)
flash
(
error
,
category
=
"error"
)
uploadText
=
_
(
u"File
%(file)
s uploaded"
,
file
=
title
)
uploadText
=
_
(
u"File
%(file)
s uploaded"
,
file
=
title
)
worker
.
add_upload
(
current_user
.
nickname
,
WorkerThread
.
add
(
current_user
.
nickname
,
TaskUpload
(
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book_id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
"<a href=
\"
"
+
url_for
(
'web.show_book'
,
book_id
=
book_id
)
+
"
\"
>"
+
uploadText
+
"</a>"
)
)
if
len
(
request
.
files
.
getlist
(
"btn-upload"
))
<
2
:
if
len
(
request
.
files
.
getlist
(
"btn-upload"
))
<
2
:
if
current_user
.
role_edit
()
or
current_user
.
role_admin
():
if
current_user
.
role_edit
()
or
current_user
.
role_admin
():
...
...
cps/helper.py
View file @
2533c9c1
...
@@ -68,6 +68,7 @@ from .subproc_wrapper import process_wait
...
@@ -68,6 +68,7 @@ from .subproc_wrapper import process_wait
from
.worker
import
STAT_WAITING
,
STAT_FAIL
,
STAT_STARTED
,
STAT_FINISH_SUCCESS
from
.worker
import
STAT_WAITING
,
STAT_FAIL
,
STAT_STARTED
,
STAT_FINISH_SUCCESS
from
.worker
import
TASK_EMAIL
,
TASK_CONVERT
,
TASK_UPLOAD
,
TASK_CONVERT_ANY
from
.worker
import
TASK_EMAIL
,
TASK_CONVERT
,
TASK_UPLOAD
,
TASK_CONVERT_ANY
from
.services.worker
import
WorkerThread
from
.services.worker
import
WorkerThread
from
.tasks.email
import
TaskEmail
from
.
import
tasks
from
.
import
tasks
...
@@ -115,9 +116,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
...
@@ -115,9 +116,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
def
send_test_mail
(
kindle_mail
,
user_name
):
def
send_test_mail
(
kindle_mail
,
user_name
):
worker
.
add_e
mail
(
_
(
u'Calibre-Web test e-mail'
),
None
,
None
,
WorkerThread
.
add
(
user_name
,
TaskE
mail
(
_
(
u'Calibre-Web test e-mail'
),
None
,
None
,
config
.
get_mail_settings
(),
kindle_mail
,
user_name
,
config
.
get_mail_settings
(),
kindle_mail
,
_
(
u"Test e-mail"
)
,
_
(
u"Test e-mail"
),
_
(
u'This e-mail has been sent via Calibre-Web.'
))
_
(
u'This e-mail has been sent via Calibre-Web.'
)
))
return
return
...
@@ -132,9 +133,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
...
@@ -132,9 +133,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
text
+=
"Don't forget to change your password after first login.
\r\n
"
text
+=
"Don't forget to change your password after first login.
\r\n
"
text
+=
"Sincerely
\r\n\r\n
"
text
+=
"Sincerely
\r\n\r\n
"
text
+=
"Your Calibre-Web team"
text
+=
"Your Calibre-Web team"
worker
.
add_e
mail
(
_
(
u'Get Started with Calibre-Web'
),
None
,
None
,
WorkerThread
.
add
(
None
,
TaskE
mail
(
_
(
u'Get Started with Calibre-Web'
),
None
,
None
,
config
.
get_mail_settings
(),
e_mail
,
None
,
config
.
get_mail_settings
(),
e_mail
,
None
,
_
(
u"Registration e-mail for user:
%(name)
s"
,
name
=
user_name
),
text
)
_
(
u"Registration e-mail for user:
%(name)
s"
,
name
=
user_name
),
text
)
)
return
return
...
@@ -226,9 +227,9 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
...
@@ -226,9 +227,9 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
for
entry
in
iter
(
book
.
data
):
for
entry
in
iter
(
book
.
data
):
if
entry
.
format
.
upper
()
==
book_format
.
upper
():
if
entry
.
format
.
upper
()
==
book_format
.
upper
():
converted_file_name
=
entry
.
name
+
'.'
+
book_format
.
lower
()
converted_file_name
=
entry
.
name
+
'.'
+
book_format
.
lower
()
worker
.
add_e
mail
(
_
(
u"Send to Kindle"
),
book
.
path
,
converted_file_name
,
WorkerThread
.
add
(
user_id
,
TaskE
mail
(
_
(
u"Send to Kindle"
),
book
.
path
,
converted_file_name
,
config
.
get_mail_settings
(),
kindle_mail
,
user_id
,
config
.
get_mail_settings
(),
kindle_mail
,
_
(
u"E-mail:
%(book)
s"
,
book
=
book
.
title
),
_
(
u'This e-mail has been sent via Calibre-Web.'
))
_
(
u"E-mail:
%(book)
s"
,
book
=
book
.
title
),
_
(
u'This e-mail has been sent via Calibre-Web.'
))
)
return
return
return
_
(
u"The requested file could not be read. Maybe wrong permissions?"
)
return
_
(
u"The requested file could not be read. Maybe wrong permissions?"
)
...
...
cps/services/worker.py
View file @
2533c9c1
...
@@ -67,7 +67,7 @@ class ImprovedQueue(queue.Queue):
...
@@ -67,7 +67,7 @@ class ImprovedQueue(queue.Queue):
#Class for all worker tasks in the background
#Class for all worker tasks in the background
class
WorkerThread
(
threading
.
Thread
):
class
WorkerThread
(
threading
.
Thread
):
_
_
instance
=
None
_instance
=
None
@
classmethod
@
classmethod
def
getInstance
(
cls
):
def
getInstance
(
cls
):
...
@@ -112,11 +112,12 @@ class WorkerThread(threading.Thread):
...
@@ -112,11 +112,12 @@ class WorkerThread(threading.Thread):
with
self
.
doLock
:
with
self
.
doLock
:
self
.
finished
.
append
((
user
,
item
))
self
.
finished
.
append
((
user
,
item
))
try
:
# sometimes tasks (like Upload) don't actually have work to do and are created as already finished
item
.
start
(
self
)
if
item
.
stat
is
STAT_WAITING
:
print
(
item
)
try
:
except
Exception
as
e
:
item
.
start
(
self
)
log
.
exception
(
e
)
except
Exception
as
e
:
log
.
exception
(
e
)
self
.
queue
.
task_done
()
self
.
queue
.
task_done
()
...
@@ -161,6 +162,7 @@ class CalibreTask(metaclass=abc.ABCMeta):
...
@@ -161,6 +162,7 @@ class CalibreTask(metaclass=abc.ABCMeta):
def
start
(
self
,
*
args
):
def
start
(
self
,
*
args
):
self
.
start_time
=
datetime
.
now
()
self
.
start_time
=
datetime
.
now
()
self
.
stat
=
STAT_STARTED
self
.
run
(
*
args
)
self
.
run
(
*
args
)
self
.
end_time
=
datetime
.
now
()
self
.
end_time
=
datetime
.
now
()
...
...
cps/tasks/email.py
0 → 100644
View file @
2533c9c1
from
__future__
import
division
,
print_function
,
unicode_literals
import
sys
import
os
import
smtplib
import
threading
import
socket
try
:
from
StringIO
import
StringIO
from
email.MIMEBase
import
MIMEBase
from
email.MIMEMultipart
import
MIMEMultipart
from
email.MIMEText
import
MIMEText
except
ImportError
:
from
io
import
StringIO
from
email.mime.base
import
MIMEBase
from
email.mime.multipart
import
MIMEMultipart
from
email.mime.text
import
MIMEText
from
email
import
encoders
from
email.utils
import
formatdate
,
make_msgid
from
email.generator
import
Generator
from
cps.services.worker
import
CalibreTask
from
cps
import
logger
,
config
from
cps
import
gdriveutils
log
=
logger
.
create
()
chunksize
=
8192
# Class for sending email with ability to get current progress
class
EmailBase
():
transferSize
=
0
progress
=
0
def
data
(
self
,
msg
):
self
.
transferSize
=
len
(
msg
)
(
code
,
resp
)
=
smtplib
.
SMTP
.
data
(
self
,
msg
)
self
.
progress
=
0
return
(
code
,
resp
)
def
send
(
self
,
strg
):
"""Send `strg' to the server."""
log
.
debug
(
'send:
%
r'
,
strg
[:
300
])
if
hasattr
(
self
,
'sock'
)
and
self
.
sock
:
try
:
if
self
.
transferSize
:
lock
=
threading
.
Lock
()
lock
.
acquire
()
self
.
transferSize
=
len
(
strg
)
lock
.
release
()
for
i
in
range
(
0
,
self
.
transferSize
,
chunksize
):
if
isinstance
(
strg
,
bytes
):
self
.
sock
.
send
((
strg
[
i
:
i
+
chunksize
]))
else
:
self
.
sock
.
send
((
strg
[
i
:
i
+
chunksize
])
.
encode
(
'utf-8'
))
lock
.
acquire
()
self
.
progress
=
i
lock
.
release
()
else
:
self
.
sock
.
sendall
(
strg
.
encode
(
'utf-8'
))
except
socket
.
error
:
self
.
close
()
raise
smtplib
.
SMTPServerDisconnected
(
'Server not connected'
)
else
:
raise
smtplib
.
SMTPServerDisconnected
(
'please run connect() first'
)
@
classmethod
def
_print_debug
(
self
,
*
args
):
log
.
debug
(
args
)
def
getTransferStatus
(
self
):
if
self
.
transferSize
:
lock2
=
threading
.
Lock
()
lock2
.
acquire
()
value
=
int
((
float
(
self
.
progress
)
/
float
(
self
.
transferSize
))
*
100
)
lock2
.
release
()
return
str
(
value
)
+
'
%
'
else
:
return
"100
%
"
# Class for sending email with ability to get current progress, derived from emailbase class
class
Email
(
EmailBase
,
smtplib
.
SMTP
):
def
__init__
(
self
,
*
args
,
**
kwargs
):
smtplib
.
SMTP
.
__init__
(
self
,
*
args
,
**
kwargs
)
# Class for sending ssl encrypted email with ability to get current progress, , derived from emailbase class
class
EmailSSL
(
EmailBase
,
smtplib
.
SMTP_SSL
):
def
__init__
(
self
,
*
args
,
**
kwargs
):
smtplib
.
SMTP_SSL
.
__init__
(
self
,
*
args
,
**
kwargs
)
class
TaskEmail
(
CalibreTask
):
def
__init__
(
self
,
subject
,
filepath
,
attachment
,
settings
,
recipient
,
taskMessage
,
text
,
internal
=
False
):
super
()
.
__init__
(
taskMessage
)
self
.
subject
=
subject
self
.
attachment
=
attachment
self
.
settings
=
settings
self
.
filepath
=
filepath
self
.
recipent
=
recipient
self
.
text
=
text
self
.
results
=
dict
()
def
run
(
self
,
worker_thread
):
# create MIME message
msg
=
MIMEMultipart
()
msg
[
'Subject'
]
=
self
.
subject
msg
[
'Message-Id'
]
=
make_msgid
(
'calibre-web'
)
msg
[
'Date'
]
=
formatdate
(
localtime
=
True
)
text
=
self
.
text
msg
.
attach
(
MIMEText
(
text
.
encode
(
'UTF-8'
),
'plain'
,
'UTF-8'
))
if
self
.
attachment
:
result
=
self
.
get_attachment
(
self
.
filepath
,
self
.
attachment
)
if
result
:
msg
.
attach
(
result
)
else
:
self
.
_handleError
(
u"Attachment not found"
)
return
msg
[
'From'
]
=
self
.
settings
[
"mail_from"
]
msg
[
'To'
]
=
self
.
recipent
use_ssl
=
int
(
self
.
settings
.
get
(
'mail_use_ssl'
,
0
))
try
:
# convert MIME message to string
fp
=
StringIO
()
gen
=
Generator
(
fp
,
mangle_from_
=
False
)
gen
.
flatten
(
msg
)
msg
=
fp
.
getvalue
()
# send email
timeout
=
600
# set timeout to 5mins
# redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
# _print_debug function
if
sys
.
version_info
<
(
3
,
0
):
org_smtpstderr
=
smtplib
.
stderr
smtplib
.
stderr
=
logger
.
StderrLogger
(
'worker.smtp'
)
if
use_ssl
==
2
:
self
.
asyncSMTP
=
EmailSSL
(
self
.
settings
[
"mail_server"
],
self
.
settings
[
"mail_port"
],
timeout
=
timeout
)
else
:
self
.
asyncSMTP
=
Email
(
self
.
settings
[
"mail_server"
],
self
.
settings
[
"mail_port"
],
timeout
=
timeout
)
# link to logginglevel
if
logger
.
is_debug_enabled
():
self
.
asyncSMTP
.
set_debuglevel
(
1
)
if
use_ssl
==
1
:
self
.
asyncSMTP
.
starttls
()
if
self
.
settings
[
"mail_password"
]:
self
.
asyncSMTP
.
login
(
str
(
self
.
settings
[
"mail_login"
]),
str
(
self
.
settings
[
"mail_password"
]))
self
.
asyncSMTP
.
sendmail
(
self
.
settings
[
"mail_from"
],
self
.
recipent
,
msg
)
self
.
asyncSMTP
.
quit
()
self
.
_handleSuccess
()
if
sys
.
version_info
<
(
3
,
0
):
smtplib
.
stderr
=
org_smtpstderr
except
(
MemoryError
)
as
e
:
log
.
exception
(
e
)
self
.
_handleError
(
u'MemoryError sending email: '
+
str
(
e
))
return
None
except
(
smtplib
.
SMTPException
,
smtplib
.
SMTPAuthenticationError
)
as
e
:
if
hasattr
(
e
,
"smtp_error"
):
text
=
e
.
smtp_error
.
decode
(
'utf-8'
)
.
replace
(
"
\n
"
,
'. '
)
elif
hasattr
(
e
,
"message"
):
text
=
e
.
message
else
:
log
.
exception
(
e
)
text
=
''
self
.
_handleError
(
u'Smtplib Error sending email: '
+
text
)
return
None
except
(
socket
.
error
)
as
e
:
self
.
_handleError
(
u'Socket Error sending email: '
+
e
.
strerror
)
return
None
def
_get_attachment
(
bookpath
,
filename
):
"""Get file as MIMEBase message"""
calibrepath
=
config
.
config_calibre_dir
if
config
.
config_use_google_drive
:
df
=
gdriveutils
.
getFileFromEbooksFolder
(
bookpath
,
filename
)
if
df
:
datafile
=
os
.
path
.
join
(
calibrepath
,
bookpath
,
filename
)
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
calibrepath
,
bookpath
)):
os
.
makedirs
(
os
.
path
.
join
(
calibrepath
,
bookpath
))
df
.
GetContentFile
(
datafile
)
else
:
return
None
file_
=
open
(
datafile
,
'rb'
)
data
=
file_
.
read
()
file_
.
close
()
os
.
remove
(
datafile
)
else
:
try
:
file_
=
open
(
os
.
path
.
join
(
calibrepath
,
bookpath
,
filename
),
'rb'
)
data
=
file_
.
read
()
file_
.
close
()
except
IOError
as
e
:
log
.
exception
(
e
)
log
.
error
(
u'The requested file could not be read. Maybe wrong permissions?'
)
return
None
attachment
=
MIMEBase
(
'application'
,
'octet-stream'
)
attachment
.
set_payload
(
data
)
encoders
.
encode_base64
(
attachment
)
attachment
.
add_header
(
'Content-Disposition'
,
'attachment'
,
filename
=
filename
)
return
attachment
@
property
def
name
(
self
):
return
"Email"
cps/tasks/upload.py
0 → 100644
View file @
2533c9c1
from
__future__
import
division
,
print_function
,
unicode_literals
from
datetime
import
datetime
from
cps.services.worker
import
CalibreTask
,
STAT_FINISH_SUCCESS
class
TaskUpload
(
CalibreTask
):
def
__init__
(
self
,
taskMessage
):
super
()
.
__init__
(
taskMessage
)
self
.
start_time
=
self
.
end_time
=
datetime
.
now
()
self
.
stat
=
STAT_FINISH_SUCCESS
def
run
(
self
,
worker_thread
):
"""Upload task doesn't have anything to do, it's simply a way to add information to the task list"""
pass
@
property
def
name
(
self
):
return
"Upload"
cps/web.py
View file @
2533c9c1
...
@@ -41,6 +41,9 @@ from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
...
@@ -41,6 +41,9 @@ from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
from
sqlalchemy.sql.expression
import
text
,
func
,
true
,
false
,
not_
,
and_
,
or_
from
sqlalchemy.sql.expression
import
text
,
func
,
true
,
false
,
not_
,
and_
,
or_
from
werkzeug.exceptions
import
default_exceptions
,
InternalServerError
from
werkzeug.exceptions
import
default_exceptions
,
InternalServerError
from
sqlalchemy.sql.functions
import
coalesce
from
sqlalchemy.sql.functions
import
coalesce
from
.services.worker
import
WorkerThread
try
:
try
:
from
werkzeug.exceptions
import
FailedDependency
from
werkzeug.exceptions
import
FailedDependency
except
ImportError
:
except
ImportError
:
...
@@ -48,7 +51,7 @@ except ImportError:
...
@@ -48,7 +51,7 @@ except ImportError:
from
werkzeug.datastructures
import
Headers
from
werkzeug.datastructures
import
Headers
from
werkzeug.security
import
generate_password_hash
,
check_password_hash
from
werkzeug.security
import
generate_password_hash
,
check_password_hash
from
.
import
constants
,
logger
,
isoLanguages
,
services
,
worker
,
worker2
,
cli
from
.
import
constants
,
logger
,
isoLanguages
,
services
,
worker
,
cli
from
.
import
searched_ids
,
lm
,
babel
,
db
,
ub
,
config
,
get_locale
,
app
from
.
import
searched_ids
,
lm
,
babel
,
db
,
ub
,
config
,
get_locale
,
app
from
.
import
calibre_db
from
.
import
calibre_db
from
.gdriveutils
import
getFileFromEbooksFolder
,
do_gdrive_download
from
.gdriveutils
import
getFileFromEbooksFolder
,
do_gdrive_download
...
@@ -383,7 +386,8 @@ def import_ldap_users():
...
@@ -383,7 +386,8 @@ def import_ldap_users():
@
web
.
route
(
"/ajax/emailstat"
)
@
web
.
route
(
"/ajax/emailstat"
)
@
login_required
@
login_required
def
get_email_status_json
():
def
get_email_status_json
():
tasks
=
worker2
.
_worker2
.
tasks
tasks
=
WorkerThread
.
getInstance
()
.
tasks
return
jsonify
(
render_task_status
(
tasks
))
return
jsonify
(
render_task_status
(
tasks
))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment