Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
c25afdc2
Commit
c25afdc2
authored
Dec 06, 2020
by
Ozzieisaacs
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Chunked sync
parent
c25f6d7c
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
55 additions
and
41 deletions
+55
-41
kobo.py
cps/kobo.py
+47
-39
SyncToken.py
cps/services/SyncToken.py
+8
-2
No files found.
cps/kobo.py
View file @
c25afdc2
...
...
@@ -43,6 +43,7 @@ from flask_login import current_user
from
werkzeug.datastructures
import
Headers
from
sqlalchemy
import
func
from
sqlalchemy.sql.expression
import
and_
,
or_
from
sqlalchemy.orm
import
load_only
from
sqlalchemy.exc
import
StatementError
import
requests
...
...
@@ -56,6 +57,8 @@ KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]}
KOBO_STOREAPI_URL
=
"https://storeapi.kobo.com"
KOBO_IMAGEHOST_URL
=
"https://kbimages1-a.akamaihd.net"
SYNC_ITEM_LIMIT
=
5
kobo
=
Blueprint
(
"kobo"
,
__name__
,
url_prefix
=
"/kobo/<auth_token>"
)
kobo_auth
.
disable_failed_auth_redirect_for_blueprint
(
kobo
)
kobo_auth
.
register_url_value_preprocessor
(
kobo
)
...
...
@@ -142,68 +145,70 @@ def HandleSyncRequest():
new_books_last_modified
=
sync_token
.
books_last_modified
new_books_last_created
=
sync_token
.
books_last_created
new_reading_state_last_modified
=
sync_token
.
reading_state_last_modified
new_archived_last_modified
=
datetime
.
datetime
.
min
sync_results
=
[]
# We reload the book database so that the user get's a fresh view of the library
# in case of external changes (e.g: adding a book through Calibre).
calibre_db
.
reconnect_db
(
config
,
ub
.
app_DB_path
)
archived_books
=
(
ub
.
session
.
query
(
ub
.
ArchivedBook
)
.
filter
(
ub
.
ArchivedBook
.
user_id
==
int
(
current_user
.
id
))
.
all
()
)
# We join-in books that have had their Archived bit recently modified in order to either:
# * Restore them to the user's device.
# * Delete them from the user's device.
# (Ideally we would use a join for this logic, however cross-database joins don't look trivial in SqlAlchemy.)
recently_restored_or_archived_books
=
[]
archived_book_ids
=
{}
new_archived_last_modified
=
datetime
.
datetime
.
min
for
archived_book
in
archived_books
:
if
archived_book
.
last_modified
>
sync_token
.
archive_last_modified
:
recently_restored_or_archived_books
.
append
(
archived_book
.
book_id
)
if
archived_book
.
is_archived
:
archived_book_ids
[
archived_book
.
book_id
]
=
True
new_archived_last_modified
=
max
(
new_archived_last_modified
,
archived_book
.
last_modified
)
# sqlite gives unexpected results when performing the last_modified comparison without the datetime cast.
# It looks like it's treating the db.Books.last_modified field as a string and may fail
# the comparison because of the +00:00 suffix.
changed_entries
=
(
calibre_db
.
session
.
query
(
db
.
Books
)
.
join
(
db
.
Data
)
.
filter
(
or_
(
func
.
datetime
(
db
.
Books
.
last_modified
)
>
sync_token
.
books_last_modified
,
db
.
Books
.
id
.
in_
(
recently_restored_or_archived_books
))
)
calibre_db
.
session
.
query
(
db
.
Books
,
ub
.
ArchivedBook
.
last_modified
,
ub
.
ArchivedBook
.
is_archived
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>=
sync_token
.
books_last_modified
)
.
filter
(
db
.
Books
.
id
>
sync_token
.
books_last_id
)
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
all
()
# .filter(ub.ArchivedBook.is_archived == 0)
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
.
limit
(
SYNC_ITEM_LIMIT
)
)
reading_states_in_new_entitlements
=
[]
for
book
in
changed_entries
:
kobo_reading_state
=
get_or_create_reading_state
(
book
.
id
)
kobo_reading_state
=
get_or_create_reading_state
(
book
.
Books
.
id
)
entitlement
=
{
"BookEntitlement"
:
create_book_entitlement
(
book
,
archived
=
(
book
.
id
in
archived_book_ids
)),
"BookMetadata"
:
get_metadata
(
book
),
"BookEntitlement"
:
create_book_entitlement
(
book
.
Books
,
archived
=
(
book
.
is_archived
==
True
)),
"BookMetadata"
:
get_metadata
(
book
.
Books
),
}
if
kobo_reading_state
.
last_modified
>
sync_token
.
reading_state_last_modified
:
entitlement
[
"ReadingState"
]
=
get_kobo_reading_state_response
(
book
,
kobo_reading_state
)
entitlement
[
"ReadingState"
]
=
get_kobo_reading_state_response
(
book
.
Books
,
kobo_reading_state
)
new_reading_state_last_modified
=
max
(
new_reading_state_last_modified
,
kobo_reading_state
.
last_modified
)
reading_states_in_new_entitlements
.
append
(
book
.
id
)
reading_states_in_new_entitlements
.
append
(
book
.
Books
.
id
)
if
book
.
timestamp
>
sync_token
.
books_last_created
:
if
book
.
Books
.
timestamp
>
sync_token
.
books_last_created
:
sync_results
.
append
({
"NewEntitlement"
:
entitlement
})
else
:
sync_results
.
append
({
"ChangedEntitlement"
:
entitlement
})
new_books_last_modified
=
max
(
book
.
last_modified
,
new_books_last_modified
book
.
Books
.
last_modified
,
new_books_last_modified
)
new_books_last_created
=
max
(
book
.
timestamp
,
new_books_last_created
)
new_books_last_created
=
max
(
book
.
Books
.
timestamp
,
new_books_last_created
)
max_change
=
(
changed_entries
.
from_self
()
.
filter
(
ub
.
ArchivedBook
.
is_archived
)
.
order_by
(
func
.
datetime
(
ub
.
ArchivedBook
.
last_modified
)
.
desc
())
.
first
()
)
if
max_change
:
max_change
=
max_change
.
last_modified
else
:
max_change
=
new_archived_last_modified
new_archived_last_modified
=
max
(
new_archived_last_modified
,
max_change
)
# no. of books returned
book_count
=
changed_entries
.
count
()
# last entry:
if
book_count
:
books_last_id
=
changed_entries
.
all
()[
-
1
]
.
Books
.
id
or
-
1
else
:
books_last_id
=
-
1
# generate reading state data
changed_reading_states
=
(
ub
.
session
.
query
(
ub
.
KoboReadingState
)
.
filter
(
and_
(
func
.
datetime
(
ub
.
KoboReadingState
.
last_modified
)
>
sync_token
.
reading_state_last_modified
,
...
...
@@ -225,11 +230,12 @@ def HandleSyncRequest():
sync_token
.
books_last_modified
=
new_books_last_modified
sync_token
.
archive_last_modified
=
new_archived_last_modified
sync_token
.
reading_state_last_modified
=
new_reading_state_last_modified
sync_token
.
books_last_id
=
books_last_id
return
generate_sync_response
(
sync_token
,
sync_results
)
return
generate_sync_response
(
sync_token
,
sync_results
,
book_count
)
def
generate_sync_response
(
sync_token
,
sync_results
):
def
generate_sync_response
(
sync_token
,
sync_results
,
set_cont
=
False
):
extra_headers
=
{}
if
config
.
config_kobo_proxy
:
# Merge in sync results from the official Kobo store.
...
...
@@ -245,6 +251,8 @@ def generate_sync_response(sync_token, sync_results):
except
Exception
as
e
:
log
.
error
(
"Failed to receive or parse response from Kobo's sync endpoint: "
+
str
(
e
))
if
set_cont
:
extra_headers
[
"x-kobo-sync"
]
=
"continue"
sync_token
.
to_headers
(
extra_headers
)
response
=
make_response
(
jsonify
(
sync_results
),
extra_headers
)
...
...
cps/services/SyncToken.py
View file @
c25afdc2
...
...
@@ -85,6 +85,7 @@ class SyncToken:
"archive_last_modified"
:
{
"type"
:
"string"
},
"reading_state_last_modified"
:
{
"type"
:
"string"
},
"tags_last_modified"
:
{
"type"
:
"string"
},
"books_last_id"
:
{
"type"
:
"integer"
,
"optional"
:
True
}
},
}
...
...
@@ -96,6 +97,7 @@ class SyncToken:
archive_last_modified
=
datetime
.
min
,
reading_state_last_modified
=
datetime
.
min
,
tags_last_modified
=
datetime
.
min
,
books_last_id
=-
1
):
self
.
raw_kobo_store_token
=
raw_kobo_store_token
self
.
books_last_created
=
books_last_created
...
...
@@ -103,6 +105,7 @@ class SyncToken:
self
.
archive_last_modified
=
archive_last_modified
self
.
reading_state_last_modified
=
reading_state_last_modified
self
.
tags_last_modified
=
tags_last_modified
self
.
books_last_id
=
books_last_id
@
staticmethod
def
from_headers
(
headers
):
...
...
@@ -137,6 +140,7 @@ class SyncToken:
archive_last_modified
=
get_datetime_from_json
(
data_json
,
"archive_last_modified"
)
reading_state_last_modified
=
get_datetime_from_json
(
data_json
,
"reading_state_last_modified"
)
tags_last_modified
=
get_datetime_from_json
(
data_json
,
"tags_last_modified"
)
books_last_id
=
data_json
[
"books_last_id"
]
except
TypeError
:
log
.
error
(
"SyncToken timestamps don't parse to a datetime."
)
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
)
...
...
@@ -147,7 +151,8 @@ class SyncToken:
books_last_modified
=
books_last_modified
,
archive_last_modified
=
archive_last_modified
,
reading_state_last_modified
=
reading_state_last_modified
,
tags_last_modified
=
tags_last_modified
tags_last_modified
=
tags_last_modified
,
books_last_id
=
books_last_id
)
def
set_kobo_store_header
(
self
,
store_headers
):
...
...
@@ -170,7 +175,8 @@ class SyncToken:
"books_last_created"
:
to_epoch_timestamp
(
self
.
books_last_created
),
"archive_last_modified"
:
to_epoch_timestamp
(
self
.
archive_last_modified
),
"reading_state_last_modified"
:
to_epoch_timestamp
(
self
.
reading_state_last_modified
),
"tags_last_modified"
:
to_epoch_timestamp
(
self
.
tags_last_modified
)
"tags_last_modified"
:
to_epoch_timestamp
(
self
.
tags_last_modified
),
"books_last_id"
:
self
.
books_last_id
},
}
return
b64encode_json
(
token
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment