Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
8dc11e89
Commit
8dc11e89
authored
May 02, 2021
by
Ozzieisaacs
Browse files
Options
Browse Files
Download
Plain Diff
Merge remote-tracking branch 'koko_sync/feature/kobo-shelf' into Develop
parents
b34672ed
24bbf226
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
154 additions
and
47 deletions
+154
-47
kobo.py
cps/kobo.py
+108
-31
shelf.py
cps/shelf.py
+27
-16
shelf_edit.html
cps/templates/shelf_edit.html
+8
-0
ub.py
cps/ub.py
+11
-0
No files found.
cps/kobo.py
View file @
8dc11e89
...
...
@@ -42,7 +42,7 @@ from flask import (
from
flask_login
import
current_user
from
werkzeug.datastructures
import
Headers
from
sqlalchemy
import
func
from
sqlalchemy.sql.expression
import
and_
from
sqlalchemy.sql.expression
import
and_
,
or_
from
sqlalchemy.exc
import
StatementError
import
requests
...
...
@@ -81,6 +81,7 @@ CONNECTION_SPECIFIC_HEADERS = [
"transfer-encoding"
,
]
def
get_kobo_activated
():
return
config
.
config_kobo_sync
...
...
@@ -151,30 +152,45 @@ def HandleSyncRequest():
# in case of external changes (e.g: adding a book through Calibre).
calibre_db
.
reconnect_db
(
config
,
ub
.
app_DB_path
)
if
sync_token
.
books_last_id
>
-
1
:
only_kobo_shelves
=
(
calibre_db
.
session
.
query
(
ub
.
Shelf
)
.
filter
(
ub
.
Shelf
.
user_id
==
current_user
.
id
)
.
filter
(
ub
.
Shelf
.
kobo_sync
)
.
count
()
)
>
0
if
only_kobo_shelves
:
changed_entries
=
(
calibre_db
.
session
.
query
(
db
.
Books
,
ub
.
ArchivedBook
.
last_modified
,
ub
.
ArchivedBook
.
is_archived
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>=
sync_token
.
books_last_modified
)
.
filter
(
db
.
Books
.
id
>
sync_token
.
books_last_id
)
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
.
limit
(
SYNC_ITEM_LIMIT
)
calibre_db
.
session
.
query
(
db
.
Books
,
ub
.
ArchivedBook
.
last_modified
,
ub
.
BookShelf
.
date_added
,
ub
.
ArchivedBook
.
is_archived
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
or_
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
,
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
))
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
id
)
.
order_by
(
ub
.
ArchivedBook
.
last_modified
)
.
join
(
ub
.
BookShelf
,
db
.
Books
.
id
==
ub
.
BookShelf
.
book_id
)
.
join
(
ub
.
Shelf
)
.
filter
(
ub
.
Shelf
.
kobo_sync
)
.
distinct
()
)
else
:
changed_entries
=
(
calibre_db
.
session
.
query
(
db
.
Books
,
ub
.
ArchivedBook
.
last_modified
,
ub
.
ArchivedBook
.
is_archived
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
)
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
.
limit
(
SYNC_ITEM_LIMIT
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
)
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
)
if
sync_token
.
books_last_id
>
-
1
:
changed_entries
=
changed_entries
.
filter
(
db
.
Books
.
id
>
sync_token
.
books_last_id
)
reading_states_in_new_entitlements
=
[]
for
book
in
changed_entries
:
for
book
in
changed_entries
.
limit
(
SYNC_ITEM_LIMIT
)
:
formats
=
[
data
.
format
for
data
in
book
.
Books
.
data
]
if
not
'KEPUB'
in
formats
and
config
.
config_kepubifypath
and
'EPUB'
in
formats
:
helper
.
convert_book_format
(
book
.
Books
.
id
,
config
.
config_calibre_dir
,
'EPUB'
,
'KEPUB'
,
current_user
.
name
)
...
...
@@ -190,7 +206,14 @@ def HandleSyncRequest():
new_reading_state_last_modified
=
max
(
new_reading_state_last_modified
,
kobo_reading_state
.
last_modified
)
reading_states_in_new_entitlements
.
append
(
book
.
Books
.
id
)
if
book
.
Books
.
timestamp
>
sync_token
.
books_last_created
:
ts_created
=
book
.
Books
.
timestamp
try
:
ts_created
=
max
(
ts_created
,
book
.
date_added
)
except
AttributeError
:
pass
if
ts_created
>
sync_token
.
books_last_created
:
sync_results
.
append
({
"NewEntitlement"
:
entitlement
})
else
:
sync_results
.
append
({
"ChangedEntitlement"
:
entitlement
})
...
...
@@ -198,7 +221,14 @@ def HandleSyncRequest():
new_books_last_modified
=
max
(
book
.
Books
.
last_modified
,
new_books_last_modified
)
new_books_last_created
=
max
(
book
.
Books
.
timestamp
,
new_books_last_created
)
try
:
new_books_last_modified
=
max
(
new_books_last_modified
,
book
.
date_added
)
except
AttributeError
:
pass
new_books_last_created
=
max
(
ts_created
,
new_books_last_created
)
max_change
=
(
changed_entries
.
from_self
()
...
...
@@ -222,11 +252,36 @@ def HandleSyncRequest():
books_last_id
=
-
1
# generate reading state data
changed_reading_states
=
ub
.
session
.
query
(
ub
.
KoboReadingState
)
if
only_kobo_shelves
:
changed_reading_states
=
(
changed_reading_states
.
join
(
ub
.
BookShelf
,
ub
.
KoboReadingState
.
book_id
==
ub
.
BookShelf
.
book_id
)
.
join
(
ub
.
Shelf
)
.
filter
(
ub
.
Shelf
.
kobo_sync
,
or_
(
func
.
datetime
(
ub
.
KoboReadingState
.
last_modified
)
>
sync_token
.
reading_state_last_modified
,
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
)
)
)
.
distinct
()
else
:
changed_reading_states
=
(
changed_reading_states
.
filter
(
func
.
datetime
(
ub
.
KoboReadingState
.
last_modified
)
>
sync_token
.
reading_state_last_modified
)
)
changed_reading_states
=
(
ub
.
session
.
query
(
ub
.
KoboReadingState
)
.
filter
(
and_
(
func
.
datetime
(
ub
.
KoboReadingState
.
last_modified
)
>
sync_token
.
reading_state_last_modified
,
ub
.
KoboReadingState
.
user_id
==
current_user
.
id
,
ub
.
KoboReadingState
.
book_id
.
notin_
(
reading_states_in_new_entitlements
))))
changed_reading_states
.
filter
(
and_
(
ub
.
KoboReadingState
.
user_id
==
current_user
.
id
,
ub
.
KoboReadingState
.
book_id
.
notin_
(
reading_states_in_new_entitlements
)
)
)
)
for
kobo_reading_state
in
changed_reading_states
.
all
():
book
=
calibre_db
.
session
.
query
(
db
.
Books
)
.
filter
(
db
.
Books
.
id
==
kobo_reading_state
.
book_id
)
.
one_or_none
()
if
book
:
...
...
@@ -237,7 +292,7 @@ def HandleSyncRequest():
})
new_reading_state_last_modified
=
max
(
new_reading_state_last_modified
,
kobo_reading_state
.
last_modified
)
sync_shelves
(
sync_token
,
sync_results
)
sync_shelves
(
sync_token
,
sync_results
,
only_kobo_shelves
=
only_kobo_shelves
)
sync_token
.
books_last_created
=
new_books_last_created
sync_token
.
books_last_modified
=
new_books_last_modified
...
...
@@ -392,7 +447,7 @@ def get_metadata(book):
book_uuid
=
book
.
uuid
metadata
=
{
"Categories"
:
[
"00000000-0000-0000-0000-000000000001"
,],
"Categories"
:
[
"00000000-0000-0000-0000-000000000001"
,
],
# "Contributors": get_author(book),
"CoverImageId"
:
book_uuid
,
"CrossRevisionId"
:
book_uuid
,
...
...
@@ -599,13 +654,14 @@ def HandleTagRemoveItem(tag_id):
# Add new, changed, or deleted shelves to the sync_results.
# Note: Public shelves that aren't owned by the user aren't supported.
def
sync_shelves
(
sync_token
,
sync_results
):
def
sync_shelves
(
sync_token
,
sync_results
,
only_kobo_shelves
=
False
):
new_tags_last_modified
=
sync_token
.
tags_last_modified
for
shelf
in
ub
.
session
.
query
(
ub
.
ShelfArchive
)
.
filter
(
func
.
datetime
(
ub
.
ShelfArchive
.
last_modified
)
>
sync_token
.
tags_last_modified
,
ub
.
ShelfArchive
.
user_id
==
current_user
.
id
):
for
shelf
in
ub
.
session
.
query
(
ub
.
ShelfArchive
)
.
filter
(
func
.
datetime
(
ub
.
ShelfArchive
.
last_modified
)
>
sync_token
.
tags_last_modified
,
ub
.
ShelfArchive
.
user_id
==
current_user
.
id
):
new_tags_last_modified
=
max
(
shelf
.
last_modified
,
new_tags_last_modified
)
sync_results
.
append
({
"DeletedTag"
:
{
"Tag"
:
{
...
...
@@ -615,8 +671,29 @@ def sync_shelves(sync_token, sync_results):
}
})
for
shelf
in
ub
.
session
.
query
(
ub
.
Shelf
)
.
filter
(
func
.
datetime
(
ub
.
Shelf
.
last_modified
)
>
sync_token
.
tags_last_modified
,
ub
.
Shelf
.
user_id
==
current_user
.
id
):
extra_filters
=
[]
if
only_kobo_shelves
:
for
shelf
in
ub
.
session
.
query
(
ub
.
Shelf
)
.
filter
(
func
.
datetime
(
ub
.
Shelf
.
last_modified
)
>
sync_token
.
tags_last_modified
,
ub
.
Shelf
.
user_id
==
current_user
.
id
,
not
ub
.
Shelf
.
kobo_sync
):
sync_results
.
append
({
"DeletedTag"
:
{
"Tag"
:
{
"Id"
:
shelf
.
uuid
,
"LastModified"
:
convert_to_kobo_timestamp_string
(
shelf
.
last_modified
)
}
}
})
extra_filters
.
append
(
ub
.
Shelf
.
kobo_sync
)
for
shelf
in
ub
.
session
.
query
(
ub
.
Shelf
)
.
outerjoin
(
ub
.
BookShelf
)
.
filter
(
or_
(
func
.
datetime
(
ub
.
Shelf
.
last_modified
)
>
sync_token
.
tags_last_modified
,
ub
.
BookShelf
.
date_added
>
sync_token
.
tags_last_modified
),
ub
.
Shelf
.
user_id
==
current_user
.
id
,
*
extra_filters
)
.
distinct
()
.
order_by
(
func
.
datetime
(
ub
.
Shelf
.
last_modified
)
.
asc
()):
if
not
shelf_lib
.
check_shelf_view_permissions
(
shelf
):
continue
...
...
cps/shelf.py
View file @
8dc11e89
...
...
@@ -21,20 +21,20 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
division
,
print_function
,
unicode_literals
from
datetime
import
datetime
import
sys
from
datetime
import
datetime
from
flask
import
Blueprint
,
request
,
flash
,
redirec
t
,
url_for
from
flask
import
Blueprint
,
flash
,
redirect
,
reques
t
,
url_for
from
flask_babel
import
gettext
as
_
from
flask_login
import
login_required
,
current_user
from
flask_login
import
current_user
,
login_required
from
sqlalchemy.exc
import
InvalidRequestError
,
OperationalError
from
sqlalchemy.sql.expression
import
func
,
true
from
sqlalchemy.exc
import
OperationalError
,
InvalidRequestError
from
.
import
logger
,
ub
,
calibre_db
,
d
b
from
.
import
calibre_db
,
config
,
db
,
logger
,
u
b
from
.render_template
import
render_title_template
from
.usermanagement
import
login_required_if_no_ano
shelf
=
Blueprint
(
'shelf'
,
__name__
)
log
=
logger
.
create
()
...
...
@@ -246,6 +246,13 @@ def create_edit_shelf(shelf, title, page, shelf_id=False):
shelf
.
is_public
=
1
else
:
shelf
.
is_public
=
0
if
config
.
config_kobo_sync
:
if
"kobo_sync"
in
to_save
:
shelf
.
kobo_sync
=
True
else
:
shelf
.
kobo_sync
=
False
if
check_shelf_is_unique
(
shelf
,
to_save
,
shelf_id
):
shelf
.
name
=
to_save
[
"title"
]
# shelf.last_modified = datetime.utcnow()
...
...
@@ -271,7 +278,11 @@ def create_edit_shelf(shelf, title, page, shelf_id=False):
ub
.
session
.
rollback
()
log
.
debug_or_exception
(
ex
)
flash
(
_
(
u"There was an error"
),
category
=
"error"
)
return
render_title_template
(
'shelf_edit.html'
,
shelf
=
shelf
,
title
=
title
,
page
=
page
)
return
render_title_template
(
'shelf_edit.html'
,
shelf
=
shelf
,
title
=
title
,
page
=
page
,
kobo_sync_enabled
=
config
.
config_kobo_sync
)
def
check_shelf_is_unique
(
shelf
,
to_save
,
shelf_id
=
False
):
...
...
@@ -362,8 +373,8 @@ def order_shelf(shelf_id):
shelf
=
ub
.
session
.
query
(
ub
.
Shelf
)
.
filter
(
ub
.
Shelf
.
id
==
shelf_id
)
.
first
()
result
=
list
()
if
shelf
and
check_shelf_view_permissions
(
shelf
):
result
=
calibre_db
.
session
.
query
(
db
.
Books
)
\
.
join
(
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
\
result
=
calibre_db
.
session
.
query
(
db
.
Books
)
\
.
join
(
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
\
.
add_columns
(
calibre_db
.
common_filters
()
.
label
(
"visible"
))
\
.
filter
(
ub
.
BookShelf
.
shelf
==
shelf_id
)
.
order_by
(
ub
.
BookShelf
.
order
.
asc
())
.
all
()
return
render_title_template
(
'shelf_order.html'
,
entries
=
result
,
...
...
@@ -372,7 +383,7 @@ def order_shelf(shelf_id):
def
change_shelf_order
(
shelf_id
,
order
):
result
=
calibre_db
.
session
.
query
(
db
.
Books
)
.
join
(
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
)
\
result
=
calibre_db
.
session
.
query
(
db
.
Books
)
.
join
(
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
)
\
.
filter
(
ub
.
BookShelf
.
shelf
==
shelf_id
)
.
order_by
(
*
order
)
.
all
()
for
index
,
entry
in
enumerate
(
result
):
book
=
ub
.
session
.
query
(
ub
.
BookShelf
)
.
filter
(
ub
.
BookShelf
.
shelf
==
shelf_id
)
\
...
...
@@ -412,13 +423,13 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param):
page
=
'shelfdown.html'
result
,
__
,
pagination
=
calibre_db
.
fill_indexpage
(
page_no
,
pagesize
,
db
.
Books
,
ub
.
BookShelf
.
shelf
==
shelf_id
,
[
ub
.
BookShelf
.
order
.
asc
()],
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
)
db
.
Books
,
ub
.
BookShelf
.
shelf
==
shelf_id
,
[
ub
.
BookShelf
.
order
.
asc
()],
ub
.
BookShelf
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
)
# delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web
wrong_entries
=
calibre_db
.
session
.
query
(
ub
.
BookShelf
)
\
.
join
(
db
.
Books
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
\
wrong_entries
=
calibre_db
.
session
.
query
(
ub
.
BookShelf
)
\
.
join
(
db
.
Books
,
ub
.
BookShelf
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
\
.
filter
(
db
.
Books
.
id
==
None
)
.
all
()
for
entry
in
wrong_entries
:
log
.
info
(
'Not existing book {} in {} deleted'
.
format
(
entry
.
book_id
,
shelf
))
...
...
cps/templates/shelf_edit.html
View file @
8dc11e89
...
...
@@ -14,6 +14,14 @@
</label>
</div>
{% endif %}
{% if kobo_sync_enabled %}
<div
class=
"checkbox"
>
<label>
<input
type=
"checkbox"
name=
"kobo_sync"
{%
if
shelf
.
kobo_sync =
=
1
%}
checked
{%
endif
%}
>
{{ _('Synchronize with Kobo device') }}
</label>
</div>
{% endif %}
<button
type=
"submit"
class=
"btn btn-default"
id=
"submit"
>
{{_('Save')}}
</button>
{% if shelf.id != None %}
<a
href=
"{{ url_for('shelf.show_shelf', shelf_id=shelf.id) }}"
class=
"btn btn-default"
>
{{_('Cancel')}}
</a>
...
...
cps/ub.py
View file @
8dc11e89
...
...
@@ -270,6 +270,7 @@ class Shelf(Base):
name
=
Column
(
String
)
is_public
=
Column
(
Integer
,
default
=
0
)
user_id
=
Column
(
Integer
,
ForeignKey
(
'user.id'
))
kobo_sync
=
Column
(
Boolean
,
default
=
False
)
books
=
relationship
(
"BookShelf"
,
backref
=
"ub_shelf"
,
cascade
=
"all, delete-orphan"
,
lazy
=
"dynamic"
)
created
=
Column
(
DateTime
,
default
=
datetime
.
datetime
.
utcnow
)
last_modified
=
Column
(
DateTime
,
default
=
datetime
.
datetime
.
utcnow
,
onupdate
=
datetime
.
datetime
.
utcnow
)
...
...
@@ -502,6 +503,7 @@ def migrate_shelfs(engine, session):
conn
.
execute
(
"ALTER TABLE shelf ADD column 'created' DATETIME"
)
conn
.
execute
(
"ALTER TABLE shelf ADD column 'last_modified' DATETIME"
)
conn
.
execute
(
"ALTER TABLE book_shelf_link ADD column 'date_added' DATETIME"
)
conn
.
execute
(
"ALTER TABLE shelf ADD column 'kobo_sync' BOOLEAN DEFAULT false"
)
for
shelf
in
session
.
query
(
Shelf
)
.
all
():
shelf
.
uuid
=
str
(
uuid
.
uuid4
())
shelf
.
created
=
datetime
.
datetime
.
now
()
...
...
@@ -509,6 +511,15 @@ def migrate_shelfs(engine, session):
for
book_shelf
in
session
.
query
(
BookShelf
)
.
all
():
book_shelf
.
date_added
=
datetime
.
datetime
.
now
()
session
.
commit
()
try
:
session
.
query
(
exists
()
.
where
(
Shelf
.
kobo_sync
))
.
scalar
()
except
exc
.
OperationalError
:
with
engine
.
connect
()
as
conn
:
conn
.
execute
(
"ALTER TABLE shelf ADD column 'kobo_sync' BOOLEAN DEFAULT false"
)
session
.
commit
()
try
:
session
.
query
(
exists
()
.
where
(
BookShelf
.
order
))
.
scalar
()
except
exc
.
OperationalError
:
# Database is not compatible, some columns are missing
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment