This commit has no recorded session.
diff
commit 410da163fe78f56a61f10628241fb69621ba6e42Author: Colin Powell <colin@unbl.ink>Date: Thu Jun 11 18:38:51 2026 -0400 [books] Fix koreader imports, maybe forever@@ -1,9 +1,8 @@ import logging import re import sqlite3-from datetime import datetime, timedelta+from datetime import datetime, timedelta, timezone from enum import Enum-from zoneinfo import ZoneInfo import requests from books.constants import BOOKS_TITLES_TO_IGNORE@@ -174,7 +173,7 @@ def build_book_map(rows) -> dict: return book_id_map -def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict:+def build_page_data(page_rows: list, book_map: dict) -> dict: """Given rows of page data from KoReader, parse each row and build scrobbles for our user, loading the page data into the page_data field on the scrobble instance.@@ -275,10 +274,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl continue timestamp = user.profile.get_timestamp_with_tz(- datetime.fromtimestamp(int(first_page.get("start_ts")))+ datetime.fromtimestamp(+ int(first_page.get("start_ts"))+ ) ) stop_timestamp = user.profile.get_timestamp_with_tz(- datetime.fromtimestamp(int(last_page.get("end_ts")))+ datetime.fromtimestamp(+ int(last_page.get("end_ts"))+ ) ) scrobble = Scrobble.objects.filter(@@ -326,6 +329,80 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl last_page_number = page_number prev_page_stats = stats++ # Handle leftover pages that never triggered a session gap+ if scrobble_page_data:+ scrobble_page_data = dict(+ sorted(+ scrobble_page_data.items(),+ key=lambda x: x[1]["start_ts"],+ )+ )+ try:+ first_page = scrobble_page_data.get(+ list(scrobble_page_data.keys())[0]+ )+ last_page = scrobble_page_data.get(+ list(scrobble_page_data.keys())[-1]+ )+ except IndexError:+ logger.error(+ "Could not process book, no page data found",+ extra={"scrobble_page_data": scrobble_page_data},+ )+ continue++ playback_position_seconds = sum(+ p["duration"] for p in scrobble_page_data.values()+ )++ timestamp = user.profile.get_timestamp_with_tz(+ datetime.fromtimestamp(+ int(first_page.get("start_ts"))+ )+ )+ stop_timestamp = user.profile.get_timestamp_with_tz(+ datetime.fromtimestamp(+ int(last_page.get("end_ts"))+ )+ )++ scrobble = Scrobble.objects.filter(+ timestamp=timestamp,+ book_id=book_id,+ user_id=user.id,+ ).first()++ if not scrobble:+ logger.info(+ f"Queueing scrobble for {book_id}, page {list(scrobble_page_data.keys())[0]}"+ )+ log_data = {+ "koreader_hash": book_dict.get("hash"),+ "page_data": scrobble_page_data,+ "pages_read": len(scrobble_page_data.keys()),+ }+ if hasattr(timestamp.tzinfo, "tzname"):+ tz = timestamp.tzinfo.tzname+ if hasattr(timestamp.tzinfo, "name"):+ tz = timestamp.tzinfo.name+ scrobbles_to_create.append(+ Scrobble(+ book_id=book_id,+ user_id=user.id,+ source="KOReader",+ media_type=Scrobble.MediaType.BOOK,+ timestamp=timestamp,+ log=log_data,+ stop_timestamp=stop_timestamp,+ playback_position_seconds=playback_position_seconds,+ in_progress=False,+ played_to_completion=True,+ long_play_complete=False,+ timezone=tz,+ )+ )+ if pages_not_found: logger.info(f"Pages not found for books: {set(pages_not_found)}") return scrobbles_to_create@@ -355,9 +432,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: new_scrobbles = [] user = User.objects.filter(id=user_id).first()- tz = ZoneInfo("UTC")- if user:- tz = user.profile.tzinfo is_os_file = "https://" not in file_path if is_os_file:@@ -373,7 +447,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: book_map = build_page_data( cur.execute("SELECT * from page_stat_data ORDER BY id_book, start_time"), book_map,- tz, ) new_scrobbles = build_scrobbles_from_book_map(book_map, user) else:@@ -390,7 +463,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: _sqlite_bytes(file_path), max_buffer_size=1_048_576 ): if table_name == "page_stat_data":- book_map = build_page_data(rows, book_map, tz)+ book_map = build_page_data(rows, book_map) new_scrobbles = build_scrobbles_from_book_map(book_map, user) logger.info(f"Creating {len(new_scrobbles)} new scrobbles")@@ -4,6 +4,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Optional from uuid import uuid4+from zoneinfo import ZoneInfo import requests from books.constants import MediaSourceTag, READCOMICSONLINE_URL@@ -472,8 +473,11 @@ class Book(LongPlayScrobblableMixin): if scrobble.logdata.page_data: for page, data in scrobble.logdata.page_data.items(): if convert_timestamps:- data["start_ts"] = datetime.fromtimestamp(data["start_ts"])- data["end_ts"] = datetime.fromtimestamp(data["end_ts"])+ tz = None+ if scrobble.timezone:+ tz = ZoneInfo(scrobble.timezone)+ data["start_ts"] = datetime.fromtimestamp(data["start_ts"], tz=tz)+ data["end_ts"] = datetime.fromtimestamp(data["end_ts"], tz=tz) pages[page] = data sorted_pages = OrderedDict( sorted(pages.items(), key=lambda x: x[1]["start_ts"])@@ -44,8 +44,12 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map) scrobbles = build_scrobbles_from_book_map(book_map, demo_user)- # Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read )- expected_scrobbles = 6 * len(book_map.keys())+ # The test data generator adds the session-gap 3600s AFTER the trigger page+ # (not before), so the first session includes 21 pages (1-21), and each+ # subsequent session has 20 until the last. The last trigger page (120) was+ # previously orphaned by the loop structure; the post-loop fix now creates a+ # scrobble for it.+ expected_scrobbles = 7 * len(book_map.keys()) assert len(scrobbles) == expected_scrobbles assert len(scrobbles[0].logdata.page_data.keys()) == 21 assert len(scrobbles[1].logdata.page_data.keys()) == 20@@ -53,6 +57,7 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re assert len(scrobbles[3].logdata.page_data.keys()) == 20 assert len(scrobbles[4].logdata.page_data.keys()) == 20 assert len(scrobbles[5].logdata.page_data.keys()) == 18+ assert len(scrobbles[6].logdata.page_data.keys()) == 1 def test_get_author_str_from_row():@@ -140,6 +140,11 @@ class UserProfile(TimeStampedModel): return history def get_timestamp_with_tz(self, timestamp):+ from django.conf import settings++ server_tz = ZoneInfo(settings.TIME_ZONE)+ ref_dt = timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=server_tz)+ timezone = self.tzinfo if self.timezone_change_log: change_list = self.historic_timezone_changes@@ -150,13 +155,13 @@ class UserProfile(TimeStampedModel): end = None if end:- if start <= timestamp.replace(tzinfo=end.timezone) <= end:+ if start <= ref_dt <= end: timezone = start.timezone else:- if start <= timestamp.replace(tzinfo=start.timezone):+ if start <= ref_dt: timezone = start.timezone - return timestamp.replace(tzinfo=timezone)+ return ref_dt.astimezone(timezone) def adjust_timezone_of_scrobbles(self, commit=False): current_dt = None