Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
5edde53f
Commit
5edde53f
authored
Oct 03, 2021
by
Ozzie Isaacs
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improved sync for kobo with additional table
parent
1c15e10a
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
53 additions
and
29 deletions
+53
-29
kobo.py
cps/kobo.py
+34
-16
SyncToken.py
cps/services/SyncToken.py
+13
-13
ub.py
cps/ub.py
+6
-0
No files found.
cps/kobo.py
View file @
5edde53f
...
...
@@ -170,9 +170,14 @@ def HandleSyncRequest():
ub
.
ArchivedBook
.
is_archived
)
changed_entries
=
(
changed_entries
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
or_
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
,
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
))
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
filter
(
calibre_db
.
common_filters
())
.
join
(
ub
.
KoboSyncedBooks
,
ub
.
KoboSyncedBooks
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
.
filter
(
or_
(
ub
.
KoboSyncedBooks
.
user_id
!=
current_user
.
id
,
ub
.
KoboSyncedBooks
.
book_id
==
None
))
#.filter(or_(db.Books.last_modified > sync_token.books_last_modified,
# ub.BookShelf.date_added > sync_token.books_last_modified))
.
filter
(
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
)
#?? or also or from above
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
filter
(
calibre_db
.
common_filters
())
.
order_by
(
db
.
Books
.
id
)
.
order_by
(
ub
.
ArchivedBook
.
last_modified
)
.
join
(
ub
.
BookShelf
,
db
.
Books
.
id
==
ub
.
BookShelf
.
book_id
)
...
...
@@ -190,15 +195,17 @@ def HandleSyncRequest():
ub
.
ArchivedBook
.
is_archived
)
changed_entries
=
(
changed_entries
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
)
.
join
(
ub
.
KoboSyncedBooks
,
ub
.
KoboSyncedBooks
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
.
filter
(
or_
(
ub
.
KoboSyncedBooks
.
user_id
!=
current_user
.
id
,
ub
.
KoboSyncedBooks
.
book_id
==
None
))
.
filter
(
calibre_db
.
common_filters
())
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
)
if
sync_token
.
books_last_id
>
-
1
:
changed_entries
=
changed_entries
.
filter
(
db
.
Books
.
id
>
sync_token
.
books_last_id
)
#
if sync_token.books_last_id > -1:
#
changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id)
reading_states_in_new_entitlements
=
[]
if
sqlalchemy_version2
:
...
...
@@ -206,6 +213,7 @@ def HandleSyncRequest():
else
:
books
=
changed_entries
.
limit
(
SYNC_ITEM_LIMIT
)
for
book
in
books
:
add_synced_books
(
book
.
Books
.
id
)
formats
=
[
data
.
format
for
data
in
book
.
Books
.
data
]
if
not
'KEPUB'
in
formats
and
config
.
config_kepubifypath
and
'EPUB'
in
formats
:
helper
.
convert_book_format
(
book
.
Books
.
id
,
config
.
config_calibre_dir
,
'EPUB'
,
'KEPUB'
,
current_user
.
name
)
...
...
@@ -263,11 +271,11 @@ def HandleSyncRequest():
entries
=
calibre_db
.
session
.
execute
(
changed_entries
)
.
all
()
book_count
=
len
(
entries
)
else
:
entries
=
changed_entries
.
all
()
#
entries = changed_entries.all()
book_count
=
changed_entries
.
count
()
# last entry:
books_last_id
=
entries
[
-
1
]
.
Books
.
id
or
-
1
if
book_count
else
-
1
# sync_cont
= entries[-1].Books.id or -1 if book_count else -1
log
.
debug
(
"Remaining books to Sync: {}"
.
format
(
book_count
))
# generate reading state data
changed_reading_states
=
ub
.
session
.
query
(
ub
.
KoboReadingState
)
...
...
@@ -305,7 +313,7 @@ def HandleSyncRequest():
sync_token
.
books_last_modified
=
new_books_last_modified
sync_token
.
archive_last_modified
=
new_archived_last_modified
sync_token
.
reading_state_last_modified
=
new_reading_state_last_modified
sync_token
.
books_last_id
=
books_last_id
#
sync_token.books_last_id = books_last_id
return
generate_sync_response
(
sync_token
,
sync_results
,
book_count
)
...
...
@@ -330,7 +338,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False):
extra_headers
[
"x-kobo-sync"
]
=
"continue"
sync_token
.
to_headers
(
extra_headers
)
log
.
debug
(
"Kobo Sync Content: {}"
.
format
(
sync_results
))
#
log.debug("Kobo Sync Content: {}".format(sync_results))
response
=
make_response
(
jsonify
(
sync_results
),
extra_headers
)
return
response
...
...
@@ -838,6 +846,16 @@ def get_ub_read_status(kobo_read_status):
}
return
string_to_enum_map
[
kobo_read_status
]
def
add_synced_books
(
book_id
):
synced_book
=
ub
.
KoboSyncedBooks
()
synced_book
.
user_id
=
current_user
.
id
synced_book
.
book_id
=
book_id
ub
.
session
.
add
(
synced_book
)
try
:
ub
.
session
.
commit
()
except
Exception
:
ub
.
session
.
rollback
()
def
get_or_create_reading_state
(
book_id
):
book_read
=
ub
.
session
.
query
(
ub
.
ReadBook
)
.
filter
(
ub
.
ReadBook
.
book_id
==
book_id
,
...
...
cps/services/SyncToken.py
View file @
5edde53f
...
...
@@ -85,8 +85,8 @@ class SyncToken:
"books_last_created"
:
{
"type"
:
"string"
},
"archive_last_modified"
:
{
"type"
:
"string"
},
"reading_state_last_modified"
:
{
"type"
:
"string"
},
"tags_last_modified"
:
{
"type"
:
"string"
}
,
"books_last_id"
:
{
"type"
:
"integer"
,
"optional"
:
True
}
"tags_last_modified"
:
{
"type"
:
"string"
}
#
"books_last_id": {"type": "integer", "optional": True}
},
}
...
...
@@ -97,8 +97,8 @@ class SyncToken:
books_last_modified
=
datetime
.
min
,
archive_last_modified
=
datetime
.
min
,
reading_state_last_modified
=
datetime
.
min
,
tags_last_modified
=
datetime
.
min
,
books_last_id
=-
1
tags_last_modified
=
datetime
.
min
#
books_last_id=-1
):
# nosec
self
.
raw_kobo_store_token
=
raw_kobo_store_token
self
.
books_last_created
=
books_last_created
...
...
@@ -106,7 +106,7 @@ class SyncToken:
self
.
archive_last_modified
=
archive_last_modified
self
.
reading_state_last_modified
=
reading_state_last_modified
self
.
tags_last_modified
=
tags_last_modified
self
.
books_last_id
=
books_last_id
#
self.books_last_id = books_last_id
@
staticmethod
def
from_headers
(
headers
):
...
...
@@ -141,12 +141,12 @@ class SyncToken:
archive_last_modified
=
get_datetime_from_json
(
data_json
,
"archive_last_modified"
)
reading_state_last_modified
=
get_datetime_from_json
(
data_json
,
"reading_state_last_modified"
)
tags_last_modified
=
get_datetime_from_json
(
data_json
,
"tags_last_modified"
)
books_last_id
=
data_json
[
"books_last_id"
]
#
books_last_id = data_json["books_last_id"]
except
TypeError
:
log
.
error
(
"SyncToken timestamps don't parse to a datetime."
)
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
)
except
KeyError
:
books_last_id
=
-
1
#
except KeyError:
#
books_last_id = -1
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
,
...
...
@@ -155,7 +155,7 @@ class SyncToken:
archive_last_modified
=
archive_last_modified
,
reading_state_last_modified
=
reading_state_last_modified
,
tags_last_modified
=
tags_last_modified
,
books_last_id
=
books_last_id
#
books_last_id=books_last_id
)
def
set_kobo_store_header
(
self
,
store_headers
):
...
...
@@ -179,16 +179,16 @@ class SyncToken:
"archive_last_modified"
:
to_epoch_timestamp
(
self
.
archive_last_modified
),
"reading_state_last_modified"
:
to_epoch_timestamp
(
self
.
reading_state_last_modified
),
"tags_last_modified"
:
to_epoch_timestamp
(
self
.
tags_last_modified
),
"books_last_id"
:
self
.
books_last_id
#
"books_last_id":self.books_last_id
},
}
return
b64encode_json
(
token
)
def
__str__
(
self
):
return
"{},{},{},{},{},{}
,{}
"
.
format
(
self
.
raw_kobo_store_token
,
return
"{},{},{},{},{},{}"
.
format
(
self
.
raw_kobo_store_token
,
self
.
books_last_created
,
self
.
books_last_modified
,
self
.
archive_last_modified
,
self
.
reading_state_last_modified
,
self
.
tags_last_modified
,
self
.
books_last_id
)
self
.
tags_last_modified
)
#
self.books_last_id)
cps/ub.py
View file @
5edde53f
...
...
@@ -419,6 +419,12 @@ class ArchivedBook(Base):
last_modified
=
Column
(
DateTime
,
default
=
datetime
.
datetime
.
utcnow
)
class
KoboSyncedBooks
(
Base
):
__tablename__
=
'kobo_synced_books'
id
=
Column
(
Integer
,
primary_key
=
True
,
autoincrement
=
True
)
user_id
=
Column
(
Integer
,
ForeignKey
(
'user.id'
))
book_id
=
Column
(
Integer
)
# The Kobo ReadingState API keeps track of 4 timestamped entities:
# ReadingState, StatusInfo, Statistics, CurrentBookmark
# Which we map to the following 4 tables:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment