This commit has no recorded session.
diff
commit 5f6e75b14e1bf31b34826eb8827f8a9f9b41ac28Author: Colin Powell <colin@unbl.ink>Date: Mon Jun 15 12:19:54 2026 -0400 [books] Fix comic book metadata importing@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/15] :vrobbler:project:personal:+* Backlog [1/17] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb@@ -530,6 +530,35 @@ easily. And our exposure to PII is really low at this point in the project, so we can probably use backtrace=True and diagnose=True to help us root cause bugs faster. +** TODO [#B] Add a /trends/ page that shows trends based on scrobble data :feature:trends:scrobbles:++*** Description++This project is a bit invovled. But we should add a top level URL `trends` that shows+various trends as defined either in a static settings file, or dynamically via a database table.++Examples of trends:++- How often does the user:+ + watch sports while doing a task?+ + do a task while watching a video?+ * how often do I do + +- trail_scrobble__average_heartrate per trail +- ...++** DONE [#A] Clean up metadata comicbook enrichment :bug:comics:books:metadata:+:PROPERTIES:+:ID: cd875450-7117-78ca-8be4-9c8b73037dba+:END:++*** Description++Still getting wonky results with some comicbooks. Would be nice to be able to+tag a Book as a comicbook, and also gather volume information. I also noticed+that some books that are found in OL never get their comicvine_id populated. We+should make sure we always have comicvine_ids if available.+ * Version 51.3 [1/1] ** DONE [#A] Improve speed of index and chart pages :bug:scrobbles:perf: :PROPERTIES:@@ -160,7 +160,10 @@ class Command(BaseCommand): ) def _enrich_book(self, book, sleep_secs):- from books.sources.comicvine import lookup_comic_from_comicvine+ from books.sources.comicvine import (+ lookup_comic_from_comicvine,+ lookup_issue_by_comicvine_id,+ ) from books.sources.google import lookup_book_from_google from books.sources.openlibrary import lookup_book_from_openlibrary as lookup_book_from_ol @@ -168,13 +171,13 @@ class Command(BaseCommand): author_name = book.author.name if book.author else None book_dict = {} - is_comic = bool(book.readcomics_url) or (- book.issue_number is not None or book.volume_number is not None- )- if is_comic and READCOMICSONLINE_URL in (book.readcomics_url or ""):+ cv_data = None+ if book.comicvine_id:+ cv_data = lookup_issue_by_comicvine_id(str(book.comicvine_id))+ if not cv_data: cv_data = lookup_comic_from_comicvine(title)- if cv_data:- book_dict.update(cv_data)+ if cv_data:+ book_dict.update(cv_data) ol_data = lookup_book_from_ol(title, author=author_name) time.sleep(sleep_secs)@@ -261,6 +264,14 @@ class Command(BaseCommand): book.volume_number = data["volume_number"] update_fields.append("volume_number") + if data.get("volume") and not book.volume:+ book.volume = data["volume"]+ update_fields.append("volume")++ if data.get("volume_comicvine_id") and not book.volume_comicvine_id:+ book.volume_comicvine_id = data["volume_comicvine_id"]+ update_fields.append("volume_comicvine_id")+ if update_fields: book.save(update_fields=update_fields) self.stdout.write(f" [ENRICHED] {book} — {', '.join(update_fields)}")@@ -279,4 +290,12 @@ class Command(BaseCommand): book.genre.add(*new_genres) self.stdout.write(f" [GENRES] {book} — added {len(new_genres)} genres") + tags = data.pop("tags", [])+ if tags:+ existing_tags = set(book.tags.names())+ new_tags = [t for t in tags if t not in existing_tags]+ if new_tags:+ book.tags.add(*new_tags)+ self.stdout.write(f" [TAGS] {book} — added {', '.join(new_tags)}")+ return changed if any(changed.values()) else None@@ -0,0 +1,23 @@+# Generated by Django 4.2.29 on 2026-06-15 16:09++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("books", "0036_alter_book_genre_alter_paper_genre"),+ ]++ operations = [+ migrations.AddField(+ model_name="book",+ name="volume",+ field=models.CharField(blank=True, max_length=255, null=True),+ ),+ migrations.AddField(+ model_name="book",+ name="volume_comicvine_id",+ field=models.CharField(blank=True, max_length=255, null=True),+ ),+ ]@@ -20,6 +20,7 @@ from books.openlibrary import ( from books.sources.comicvine import ( ComicVineClient, lookup_comic_from_comicvine,+ lookup_issue_by_comicvine_id, ) from books.sources.google import lookup_book_from_google from books.sources.openlibrary import (@@ -150,6 +151,8 @@ class Book(LongPlayScrobblableMixin): first_sentence = models.TextField(**BNULL) # ComicVine comicvine_id = models.CharField(max_length=255, **BNULL)+ volume = models.CharField(max_length=255, **BNULL)+ volume_comicvine_id = models.CharField(max_length=255, **BNULL) readcomics_url = models.CharField(max_length=255, **BNULL) next_readcomics_url = models.CharField(max_length=255, **BNULL) issue_number = models.IntegerField(**BNULL)@@ -236,13 +239,17 @@ class Book(LongPlayScrobblableMixin): if not book_dict: return book + tags = book_dict.pop("tags", [])+ genres = book_dict.pop("genres", book_dict.pop("generes", []))+ for k, v in book_dict.items(): setattr(book, k, v) book.save() - genres = book_dict.get("genres", []) if genres: book.genre.add(*genres)+ if tags:+ book.tags.add(*tags) return book @classmethod@@ -276,8 +283,10 @@ class Book(LongPlayScrobblableMixin): book_dict = None source_tag = None+ tried_comicvine = False if READCOMICSONLINE_URL in url: book_dict = lookup_comic_from_comicvine(title)+ tried_comicvine = True if book_dict: source_tag = MediaSourceTag.COMICVINE book_dict["readcomics_url"] = get_comic_issue_url(url)@@ -302,6 +311,16 @@ class Book(LongPlayScrobblableMixin): if ol_data and ol_data.get("cover_url"): book_dict["cover_url"] = ol_data["cover_url"] + # Always try ComicVine as a fallback — it may recognize books that+ # OL/Google don't flag as comics+ if not tried_comicvine:+ cv_data = lookup_comic_from_comicvine(title)+ if cv_data:+ for k, v in cv_data.items():+ if v:+ book_dict.setdefault(k, v)+ source_tag = MediaSourceTag.COMICVINE+ if not book_dict: logger.warning( "No book found in any source, using data as is",@@ -312,6 +331,7 @@ class Book(LongPlayScrobblableMixin): authors = book_dict.pop("authors", []) cover_url = book_dict.pop("cover_url", "") genres = book_dict.pop("genres", book_dict.pop("generes", []))+ tags = book_dict.pop("tags", []) if authors: for author_str in authors:@@ -331,6 +351,8 @@ class Book(LongPlayScrobblableMixin): book.save_image_from_url(cover_url) if genres: book.genre.add(*genres)+ if tags:+ book.tags.add(*tags) book.authors.add(*author_list) if source_tag: book.tags.add(source_tag.value)@@ -368,8 +390,14 @@ class Book(LongPlayScrobblableMixin): data = lookup_comic_from_locg(str(self.title)) if not data and COMICVINE_API_KEY:- logger.warn(f"Checking ComicVine for {self.title}")- data = lookup_comic_from_comicvine(str(self.title))+ if self.comicvine_id:+ logger.warn(+ f"Checking ComicVine by ID for {self.title}"+ )+ data = lookup_issue_by_comicvine_id(str(self.comicvine_id))+ if not data:+ logger.warn(f"Checking ComicVine for {self.title}")+ data = lookup_comic_from_comicvine(str(self.title)) if not data: logger.warn(f"Book not found in any sources: {self.title}")@@ -407,10 +435,10 @@ class Book(LongPlayScrobblableMixin): ) data.pop("pages") - # Pop this, so we can look it up later+ # Pop these so they don't get passed to update() cover_url = data.pop("cover_url", "")- subject_key_list = data.pop("subject_key_list", [])+ tags = data.pop("tags", []) # Fun trick for updating all fields at once Book.objects.filter(pk=self.id).update(**data)@@ -418,6 +446,8 @@ class Book(LongPlayScrobblableMixin): if subject_key_list: self.genre.add(*subject_key_list)+ if tags:+ self.tags.add(*tags) if cover_url: r = requests.get(cover_url)@@ -17,8 +17,10 @@ class ComicVineClient(object): account on https://comicvine.gamespot.com/ in order to obtain an API key. """ - # All API requests made by this client will be made to this URL.+ # All API requests made by this client will be made to these URLs. API_URL = "https://comicvine.gamespot.com/api/search/"+ ISSUE_API_URL = "https://comicvine.gamespot.com/api/issue/4000-{issue_id}/"+ VOLUME_API_URL = "https://comicvine.gamespot.com/api/volume/4050-{volume_id}/" # A valid User-Agent header must be set in order for our API requests to # be accepted, otherwise our request will be rejected with a@@ -197,6 +199,74 @@ class ComicVineClient(object): raise exception(message) + def get_issue(self, issue_id: str) -> dict:+ """+ Fetch a single issue by its ComicVine ID directly from the issue detail+ endpoint, which returns richer data than the search endpoint.++ :param issue_id: The ComicVine numeric ID for the issue (e.g. "538480")+ :type issue_id: str++ :return: The full JSON response for the issue, or empty dict on failure.+ :rtype: dict+ """+ params = {+ "api_key": self.api_key,+ "format": "json",+ }+ url = self.ISSUE_API_URL.format(issue_id=issue_id)+ response = requests.get(url, headers=self.HEADERS, params=params)++ if not response.ok:+ self._handle_http_error(response)++ json_data = response.json()++ if json_data.get("status_code") != 1:+ error_msg = json_data.get("error", "Unknown ComicVine API error")+ logger.error(+ "ComicVine API returned status_code %s: %s",+ json_data.get("status_code"),+ error_msg,+ )+ return {}++ return json_data.get("results", {})++ def get_volume(self, volume_id: str) -> dict:+ """+ Fetch a single volume by its ComicVine ID from the volume detail+ endpoint. Used to get publisher info and other volume-level metadata.++ :param volume_id: The ComicVine numeric ID for the volume (e.g. "91273")+ :type volume_id: str++ :return: The full JSON response for the volume, or empty dict on failure.+ :rtype: dict+ """+ params = {+ "api_key": self.api_key,+ "format": "json",+ }+ url = self.VOLUME_API_URL.format(volume_id=volume_id)+ response = requests.get(url, headers=self.HEADERS, params=params)++ if not response.ok:+ self._handle_http_error(response)++ json_data = response.json()++ if json_data.get("status_code") != 1:+ error_msg = json_data.get("error", "Unknown ComicVine API error")+ logger.error(+ "ComicVine API returned status_code %s: %s",+ json_data.get("status_code"),+ error_msg,+ )+ return {}++ return json_data.get("results", {})+ def lookup_comic_from_comicvine(title: str) -> dict: original_title = title@@ -238,26 +308,113 @@ def lookup_comic_from_comicvine(title: str) -> dict: if not found_result: found_result = results[0] - title = found_result.get("name")+ data_dict = _build_data_dict_from_issue(found_result, original_title)+ _enrich_with_volume_data(client, data_dict)+ return data_dict+++def lookup_issue_by_comicvine_id(comicvine_id: str) -> dict:+ """+ Look up an issue directly by its ComicVine ID using the issue detail+ endpoint. Returns richer data than the search-based lookup.++ :param comicvine_id: The ComicVine numeric ID for the issue (e.g. "538480")+ :type comicvine_id: str - if found_result.get("volume"):- title = found_result.get("volume").get("name")+ :return: A dict of extracted book metadata, or empty dict on failure.+ :rtype: dict+ """+ if not comicvine_id:+ return {}++ api_key = getattr(settings, "COMICVINE_API_KEY", "")+ if not api_key:+ logger.warning("No ComicVine API key configured, not looking anything up")+ return {}++ client = ComicVineClient(api_key=api_key)+ issue_data = client.get_issue(comicvine_id)+ if not issue_data:+ logger.warning("No issue found on ComicVine for ID %s", comicvine_id)+ return {}++ data_dict = _build_data_dict_from_issue(issue_data, issue_data.get("name", ""))+ _enrich_with_volume_data(client, data_dict)+ return data_dict+++def _build_data_dict_from_issue(issue_data: dict, original_title: str = "") -> dict:+ """+ Build a book metadata dict from a ComicVine issue resource (either from+ search results or issue detail endpoint). Both return the same shape of+ issue data.++ :param issue_data: The issue resource dict from ComicVine.+ :param original_title: The original search term, if any.++ :return: A dict of extracted book metadata.+ :rtype: dict+ """+ title = issue_data.get("name")+ if issue_data.get("volume"):+ title = issue_data.get("volume").get("name") cover_url = None- if found_result.get("image"):- cover_url = found_result["image"].get("original_url")+ if issue_data.get("image"):+ cover_url = issue_data["image"].get("original_url")++ volume_name = None+ volume_cv_id = None+ publisher_name = None+ volume_data = issue_data.get("volume")+ if volume_data:+ volume_name = volume_data.get("name")+ volume_cv_id = volume_data.get("id")+ publisher_data = volume_data.get("publisher")+ if publisher_data:+ publisher_name = publisher_data.get("name") data_dict = { "title": title, "original_title": original_title,- "issue_number": found_result.get("issue_number"),- "volume_number": found_result.get("volume_number"),+ "issue_number": issue_data.get("issue_number"),+ "volume_number": issue_data.get("volume_number"),+ "volume": volume_name,+ "volume_comicvine_id": volume_cv_id,+ "publisher": publisher_name, "cover_url": cover_url,- "comicvine_id": found_result.get("id"),- "comicvine_data": found_result,- "summary": found_result.get("description"),- "publish_date": found_result.get("cover_date"),- "first_publish_year": (found_result.get("cover_date") or "")[:4],+ "comicvine_id": issue_data.get("id"),+ "summary": issue_data.get("description"),+ "publish_date": issue_data.get("cover_date"),+ "first_publish_year": (issue_data.get("cover_date") or "")[:4],+ "tags": ["comicbook"], } return data_dict+++def _enrich_with_volume_data(client: ComicVineClient, data_dict: dict) -> None:+ """+ Follow-up a successful issue lookup by fetching the volume detail and+ filling in publisher and other volume-level metadata that the issue+ endpoint doesn't provide.++ :param client: An initialised ComicVineClient instance.+ :param data_dict: The data dict from an issue lookup (mutated in place).+ """+ volume_cv_id = data_dict.get("volume_comicvine_id")+ if not volume_cv_id:+ return++ volume_data = client.get_volume(str(volume_cv_id))+ if not volume_data:+ return++ publisher_data = volume_data.get("publisher")+ if publisher_data:+ publisher_name = publisher_data.get("name")+ if publisher_name and not data_dict.get("publisher"):+ data_dict["publisher"] = publisher_name++ if not data_dict.get("volume"):+ data_dict["volume"] = volume_data.get("name")