Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
douban-api-proxy
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
captainwong
douban-api-proxy
Commits
5edde53f
Commit
5edde53f
authored
3 years ago
by
Ozzie Isaacs
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improved sync for kobo with additional table
parent
1c15e10a
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
53 additions
and
29 deletions
+53
-29
kobo.py
cps/kobo.py
+34
-16
SyncToken.py
cps/services/SyncToken.py
+13
-13
ub.py
cps/ub.py
+6
-0
No files found.
cps/kobo.py
View file @
5edde53f
...
@@ -170,9 +170,14 @@ def HandleSyncRequest():
...
@@ -170,9 +170,14 @@ def HandleSyncRequest():
ub
.
ArchivedBook
.
is_archived
)
ub
.
ArchivedBook
.
is_archived
)
changed_entries
=
(
changed_entries
changed_entries
=
(
changed_entries
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
or_
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
,
.
join
(
ub
.
KoboSyncedBooks
,
ub
.
KoboSyncedBooks
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
))
.
filter
(
or_
(
ub
.
KoboSyncedBooks
.
user_id
!=
current_user
.
id
,
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
filter
(
calibre_db
.
common_filters
())
ub
.
KoboSyncedBooks
.
book_id
==
None
))
#.filter(or_(db.Books.last_modified > sync_token.books_last_modified,
# ub.BookShelf.date_added > sync_token.books_last_modified))
.
filter
(
ub
.
BookShelf
.
date_added
>
sync_token
.
books_last_modified
)
#?? or also or from above
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
filter
(
calibre_db
.
common_filters
())
.
order_by
(
db
.
Books
.
id
)
.
order_by
(
db
.
Books
.
id
)
.
order_by
(
ub
.
ArchivedBook
.
last_modified
)
.
order_by
(
ub
.
ArchivedBook
.
last_modified
)
.
join
(
ub
.
BookShelf
,
db
.
Books
.
id
==
ub
.
BookShelf
.
book_id
)
.
join
(
ub
.
BookShelf
,
db
.
Books
.
id
==
ub
.
BookShelf
.
book_id
)
...
@@ -190,15 +195,17 @@ def HandleSyncRequest():
...
@@ -190,15 +195,17 @@ def HandleSyncRequest():
ub
.
ArchivedBook
.
is_archived
)
ub
.
ArchivedBook
.
is_archived
)
changed_entries
=
(
changed_entries
changed_entries
=
(
changed_entries
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
join
(
db
.
Data
)
.
outerjoin
(
ub
.
ArchivedBook
,
db
.
Books
.
id
==
ub
.
ArchivedBook
.
book_id
)
.
filter
(
db
.
Books
.
last_modified
>
sync_token
.
books_last_modified
)
.
join
(
ub
.
KoboSyncedBooks
,
ub
.
KoboSyncedBooks
.
book_id
==
db
.
Books
.
id
,
isouter
=
True
)
.
filter
(
or_
(
ub
.
KoboSyncedBooks
.
user_id
!=
current_user
.
id
,
ub
.
KoboSyncedBooks
.
book_id
==
None
))
.
filter
(
calibre_db
.
common_filters
())
.
filter
(
calibre_db
.
common_filters
())
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
filter
(
db
.
Data
.
format
.
in_
(
KOBO_FORMATS
))
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
last_modified
)
.
order_by
(
db
.
Books
.
id
)
.
order_by
(
db
.
Books
.
id
)
)
)
if
sync_token
.
books_last_id
>
-
1
:
#
if sync_token.books_last_id > -1:
changed_entries
=
changed_entries
.
filter
(
db
.
Books
.
id
>
sync_token
.
books_last_id
)
#
changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id)
reading_states_in_new_entitlements
=
[]
reading_states_in_new_entitlements
=
[]
if
sqlalchemy_version2
:
if
sqlalchemy_version2
:
...
@@ -206,6 +213,7 @@ def HandleSyncRequest():
...
@@ -206,6 +213,7 @@ def HandleSyncRequest():
else
:
else
:
books
=
changed_entries
.
limit
(
SYNC_ITEM_LIMIT
)
books
=
changed_entries
.
limit
(
SYNC_ITEM_LIMIT
)
for
book
in
books
:
for
book
in
books
:
add_synced_books
(
book
.
Books
.
id
)
formats
=
[
data
.
format
for
data
in
book
.
Books
.
data
]
formats
=
[
data
.
format
for
data
in
book
.
Books
.
data
]
if
not
'KEPUB'
in
formats
and
config
.
config_kepubifypath
and
'EPUB'
in
formats
:
if
not
'KEPUB'
in
formats
and
config
.
config_kepubifypath
and
'EPUB'
in
formats
:
helper
.
convert_book_format
(
book
.
Books
.
id
,
config
.
config_calibre_dir
,
'EPUB'
,
'KEPUB'
,
current_user
.
name
)
helper
.
convert_book_format
(
book
.
Books
.
id
,
config
.
config_calibre_dir
,
'EPUB'
,
'KEPUB'
,
current_user
.
name
)
...
@@ -263,11 +271,11 @@ def HandleSyncRequest():
...
@@ -263,11 +271,11 @@ def HandleSyncRequest():
entries
=
calibre_db
.
session
.
execute
(
changed_entries
)
.
all
()
entries
=
calibre_db
.
session
.
execute
(
changed_entries
)
.
all
()
book_count
=
len
(
entries
)
book_count
=
len
(
entries
)
else
:
else
:
entries
=
changed_entries
.
all
()
#
entries = changed_entries.all()
book_count
=
changed_entries
.
count
()
book_count
=
changed_entries
.
count
()
# last entry:
# last entry:
books_last_id
=
entries
[
-
1
]
.
Books
.
id
or
-
1
if
book_count
else
-
1
# sync_cont
= entries[-1].Books.id or -1 if book_count else -1
log
.
debug
(
"Remaining books to Sync: {}"
.
format
(
book_count
))
# generate reading state data
# generate reading state data
changed_reading_states
=
ub
.
session
.
query
(
ub
.
KoboReadingState
)
changed_reading_states
=
ub
.
session
.
query
(
ub
.
KoboReadingState
)
...
@@ -305,7 +313,7 @@ def HandleSyncRequest():
...
@@ -305,7 +313,7 @@ def HandleSyncRequest():
sync_token
.
books_last_modified
=
new_books_last_modified
sync_token
.
books_last_modified
=
new_books_last_modified
sync_token
.
archive_last_modified
=
new_archived_last_modified
sync_token
.
archive_last_modified
=
new_archived_last_modified
sync_token
.
reading_state_last_modified
=
new_reading_state_last_modified
sync_token
.
reading_state_last_modified
=
new_reading_state_last_modified
sync_token
.
books_last_id
=
books_last_id
#
sync_token.books_last_id = books_last_id
return
generate_sync_response
(
sync_token
,
sync_results
,
book_count
)
return
generate_sync_response
(
sync_token
,
sync_results
,
book_count
)
...
@@ -330,7 +338,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False):
...
@@ -330,7 +338,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False):
extra_headers
[
"x-kobo-sync"
]
=
"continue"
extra_headers
[
"x-kobo-sync"
]
=
"continue"
sync_token
.
to_headers
(
extra_headers
)
sync_token
.
to_headers
(
extra_headers
)
log
.
debug
(
"Kobo Sync Content: {}"
.
format
(
sync_results
))
#
log.debug("Kobo Sync Content: {}".format(sync_results))
response
=
make_response
(
jsonify
(
sync_results
),
extra_headers
)
response
=
make_response
(
jsonify
(
sync_results
),
extra_headers
)
return
response
return
response
...
@@ -838,6 +846,16 @@ def get_ub_read_status(kobo_read_status):
...
@@ -838,6 +846,16 @@ def get_ub_read_status(kobo_read_status):
}
}
return
string_to_enum_map
[
kobo_read_status
]
return
string_to_enum_map
[
kobo_read_status
]
def
add_synced_books
(
book_id
):
synced_book
=
ub
.
KoboSyncedBooks
()
synced_book
.
user_id
=
current_user
.
id
synced_book
.
book_id
=
book_id
ub
.
session
.
add
(
synced_book
)
try
:
ub
.
session
.
commit
()
except
Exception
:
ub
.
session
.
rollback
()
def
get_or_create_reading_state
(
book_id
):
def
get_or_create_reading_state
(
book_id
):
book_read
=
ub
.
session
.
query
(
ub
.
ReadBook
)
.
filter
(
ub
.
ReadBook
.
book_id
==
book_id
,
book_read
=
ub
.
session
.
query
(
ub
.
ReadBook
)
.
filter
(
ub
.
ReadBook
.
book_id
==
book_id
,
...
...
This diff is collapsed.
Click to expand it.
cps/services/SyncToken.py
View file @
5edde53f
...
@@ -85,8 +85,8 @@ class SyncToken:
...
@@ -85,8 +85,8 @@ class SyncToken:
"books_last_created"
:
{
"type"
:
"string"
},
"books_last_created"
:
{
"type"
:
"string"
},
"archive_last_modified"
:
{
"type"
:
"string"
},
"archive_last_modified"
:
{
"type"
:
"string"
},
"reading_state_last_modified"
:
{
"type"
:
"string"
},
"reading_state_last_modified"
:
{
"type"
:
"string"
},
"tags_last_modified"
:
{
"type"
:
"string"
}
,
"tags_last_modified"
:
{
"type"
:
"string"
}
"books_last_id"
:
{
"type"
:
"integer"
,
"optional"
:
True
}
#
"books_last_id": {"type": "integer", "optional": True}
},
},
}
}
...
@@ -97,8 +97,8 @@ class SyncToken:
...
@@ -97,8 +97,8 @@ class SyncToken:
books_last_modified
=
datetime
.
min
,
books_last_modified
=
datetime
.
min
,
archive_last_modified
=
datetime
.
min
,
archive_last_modified
=
datetime
.
min
,
reading_state_last_modified
=
datetime
.
min
,
reading_state_last_modified
=
datetime
.
min
,
tags_last_modified
=
datetime
.
min
,
tags_last_modified
=
datetime
.
min
books_last_id
=-
1
#
books_last_id=-1
):
# nosec
):
# nosec
self
.
raw_kobo_store_token
=
raw_kobo_store_token
self
.
raw_kobo_store_token
=
raw_kobo_store_token
self
.
books_last_created
=
books_last_created
self
.
books_last_created
=
books_last_created
...
@@ -106,7 +106,7 @@ class SyncToken:
...
@@ -106,7 +106,7 @@ class SyncToken:
self
.
archive_last_modified
=
archive_last_modified
self
.
archive_last_modified
=
archive_last_modified
self
.
reading_state_last_modified
=
reading_state_last_modified
self
.
reading_state_last_modified
=
reading_state_last_modified
self
.
tags_last_modified
=
tags_last_modified
self
.
tags_last_modified
=
tags_last_modified
self
.
books_last_id
=
books_last_id
#
self.books_last_id = books_last_id
@
staticmethod
@
staticmethod
def
from_headers
(
headers
):
def
from_headers
(
headers
):
...
@@ -141,12 +141,12 @@ class SyncToken:
...
@@ -141,12 +141,12 @@ class SyncToken:
archive_last_modified
=
get_datetime_from_json
(
data_json
,
"archive_last_modified"
)
archive_last_modified
=
get_datetime_from_json
(
data_json
,
"archive_last_modified"
)
reading_state_last_modified
=
get_datetime_from_json
(
data_json
,
"reading_state_last_modified"
)
reading_state_last_modified
=
get_datetime_from_json
(
data_json
,
"reading_state_last_modified"
)
tags_last_modified
=
get_datetime_from_json
(
data_json
,
"tags_last_modified"
)
tags_last_modified
=
get_datetime_from_json
(
data_json
,
"tags_last_modified"
)
books_last_id
=
data_json
[
"books_last_id"
]
#
books_last_id = data_json["books_last_id"]
except
TypeError
:
except
TypeError
:
log
.
error
(
"SyncToken timestamps don't parse to a datetime."
)
log
.
error
(
"SyncToken timestamps don't parse to a datetime."
)
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
)
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
)
except
KeyError
:
#
except KeyError:
books_last_id
=
-
1
#
books_last_id = -1
return
SyncToken
(
return
SyncToken
(
raw_kobo_store_token
=
raw_kobo_store_token
,
raw_kobo_store_token
=
raw_kobo_store_token
,
...
@@ -155,7 +155,7 @@ class SyncToken:
...
@@ -155,7 +155,7 @@ class SyncToken:
archive_last_modified
=
archive_last_modified
,
archive_last_modified
=
archive_last_modified
,
reading_state_last_modified
=
reading_state_last_modified
,
reading_state_last_modified
=
reading_state_last_modified
,
tags_last_modified
=
tags_last_modified
,
tags_last_modified
=
tags_last_modified
,
books_last_id
=
books_last_id
#
books_last_id=books_last_id
)
)
def
set_kobo_store_header
(
self
,
store_headers
):
def
set_kobo_store_header
(
self
,
store_headers
):
...
@@ -179,16 +179,16 @@ class SyncToken:
...
@@ -179,16 +179,16 @@ class SyncToken:
"archive_last_modified"
:
to_epoch_timestamp
(
self
.
archive_last_modified
),
"archive_last_modified"
:
to_epoch_timestamp
(
self
.
archive_last_modified
),
"reading_state_last_modified"
:
to_epoch_timestamp
(
self
.
reading_state_last_modified
),
"reading_state_last_modified"
:
to_epoch_timestamp
(
self
.
reading_state_last_modified
),
"tags_last_modified"
:
to_epoch_timestamp
(
self
.
tags_last_modified
),
"tags_last_modified"
:
to_epoch_timestamp
(
self
.
tags_last_modified
),
"books_last_id"
:
self
.
books_last_id
#
"books_last_id":self.books_last_id
},
},
}
}
return
b64encode_json
(
token
)
return
b64encode_json
(
token
)
def
__str__
(
self
):
def
__str__
(
self
):
return
"{},{},{},{},{},{}
,{}
"
.
format
(
self
.
raw_kobo_store_token
,
return
"{},{},{},{},{},{}"
.
format
(
self
.
raw_kobo_store_token
,
self
.
books_last_created
,
self
.
books_last_created
,
self
.
books_last_modified
,
self
.
books_last_modified
,
self
.
archive_last_modified
,
self
.
archive_last_modified
,
self
.
reading_state_last_modified
,
self
.
reading_state_last_modified
,
self
.
tags_last_modified
,
self
.
tags_last_modified
)
self
.
books_last_id
)
#
self.books_last_id)
This diff is collapsed.
Click to expand it.
cps/ub.py
View file @
5edde53f
...
@@ -419,6 +419,12 @@ class ArchivedBook(Base):
...
@@ -419,6 +419,12 @@ class ArchivedBook(Base):
last_modified
=
Column
(
DateTime
,
default
=
datetime
.
datetime
.
utcnow
)
last_modified
=
Column
(
DateTime
,
default
=
datetime
.
datetime
.
utcnow
)
class
KoboSyncedBooks
(
Base
):
__tablename__
=
'kobo_synced_books'
id
=
Column
(
Integer
,
primary_key
=
True
,
autoincrement
=
True
)
user_id
=
Column
(
Integer
,
ForeignKey
(
'user.id'
))
book_id
=
Column
(
Integer
)
# The Kobo ReadingState API keeps track of 4 timestamped entities:
# The Kobo ReadingState API keeps track of 4 timestamped entities:
# ReadingState, StatusInfo, Statistics, CurrentBookmark
# ReadingState, StatusInfo, Statistics, CurrentBookmark
# Which we map to the following 4 tables:
# Which we map to the following 4 tables:
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment