This commit has no recorded session.
diff
commit d5dd63be0d4f19214730cdedc6f311e0215cedd1Author: Colin Powell <colin@unbl.ink>Date: Sat Jul 4 01:34:42 2026 -0400 [boardgames] Add expansion fetching@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/26] :vrobbler:project:personal:+* Backlog [2/26] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES:@@ -617,7 +617,10 @@ boardgame_variant_ids assocaited with their log data, and a variant is created for one specific board game. -** TODO [#A] Lookup all Expansions for a game when creating it :boardgames:+** DONE [#A] Lookup all Expansions for a game when creating it :boardgames:+:PROPERTIES:+:ID: 8a84b06d-555c-4701-9058-ff364c89c198+:END: *** Description @@ -0,0 +1,78 @@+import logging++from django.core.management.base import BaseCommand++from boardgames.models import BoardGame+from boardgames.sources.bgg import lookup_boardgame_from_bgg+from boardgames.utils import fetch_and_link_expansions++logger = logging.getLogger(__name__)+++class Command(BaseCommand):+ help = "Fetch and link expansions for existing board games from BGG"++ def add_arguments(self, parser):+ parser.add_argument(+ "--commit",+ action="store_true",+ help="Persist changes to the database",+ )+ parser.add_argument(+ "--bggeek-id",+ type=str,+ help="Only process a single game by BGG ID",+ )++ def handle(self, *args, **options):+ commit = options.get("commit", False)+ bggeek_id = options.get("bggeek_id")++ games = BoardGame.objects.exclude(bggeek_id__isnull=True).exclude(+ bggeek_id=""+ )+ if bggeek_id:+ games = games.filter(bggeek_id=bggeek_id)++ total = games.count()+ self.stdout.write(f"Found {total} board games with BGG IDs")++ if total == 0:+ return++ updated = 0+ for game in games.iterator(chunk_size=100):+ try:+ data = lookup_boardgame_from_bgg(lookup_id=str(game.bggeek_id))+ except Exception as e:+ self.stdout.write(+ self.style.WARNING(+ f"Failed to fetch BGG data for {game.title} ({game.bggeek_id}): {e}"+ )+ )+ continue++ expansions = data.get("expansions", [])+ if not expansions:+ continue++ if commit:+ fetch_and_link_expansions(game, expansions)+ updated += 1+ self.stdout.write(+ f" Linked {len(expansions)} expansions to {game.title}"+ )+ else:+ self.stdout.write(+ f" Would link {len(expansions)} expansions to {game.title}"+ )+ updated += 1++ if commit:+ self.stdout.write(+ self.style.SUCCESS(f"Updated {updated} games (changes committed)")+ )+ else:+ self.stdout.write(+ f"Would update {updated} games (pass --commit to persist)"+ )@@ -71,7 +71,7 @@ class BoardGameLogData(BaseLogData, LongPlayLogData): difficulty: Optional[int] = None solo: Optional[bool] = None two_handed: Optional[bool] = None- expansion_ids: Optional[int] = None+ expansion_ids: Optional[list[int]] = None moves: Optional[list] = None rated: Optional[str] = None speed: Optional[str] = None@@ -112,6 +112,13 @@ class BoardGameLogData(BaseLogData, LongPlayLogData): required=False, widget=forms.SelectMultiple(attrs={"size": 5}), ),+ "expansion_ids": forms.ModelMultipleChoiceField(+ queryset=BoardGame.objects.filter(+ expansion_for_boardgame__isnull=False+ ),+ required=False,+ widget=forms.SelectMultiple(attrs={"size": 5}),+ ), } fields.update(custom_fields) return fields@@ -326,18 +333,28 @@ class BoardGame(ScrobblableMixin): if not data: data = lookup_boardgame_from_bgg(str(self.bggeek_id)) - cover_url = data.pop("cover_url")- year = data.pop("year_published")- publisher_name = data.pop("publisher_name")+ expansions = data.pop("expansions", [])+ cover_url = data.pop("cover_url", "")+ year_published = data.pop("year_published", None)+ if year_published is None:+ year_published = data.pop("published_year", None)+ publisher_name = data.pop("publisher_name", "") - if year:- data["published_year"] = int(year)+ if year_published:+ data["published_year"] = int(year_published) if not data["min_players"]: data.pop("min_players") if not data["min_players"]: data.pop("max_players") + # Pop extra BGG metadata that isn't a model field+ data.pop("mechanics", None)+ data.pop("categories", None)+ data.pop("designers", None)+ data.pop("publishers", None)+ data.pop("publisher", None)+ # Fun trick for updating all fields at once BoardGame.objects.filter(pk=self.id).update(**data) self.refresh_from_db()@@ -353,6 +370,10 @@ class BoardGame(ScrobblableMixin): if cover_url and not self.cover: self.save_image_from_url(cover_url) + from boardgames.utils import fetch_and_link_expansions++ fetch_and_link_expansions(self, expansions)+ def save_image_from_url(self, url): headers = {"User-Agent": "Vrobbler 0.11.12"} r = requests.get(url, headers=headers)@@ -374,20 +395,6 @@ class BoardGame(ScrobblableMixin): ) return game --class BoardGameVariant(TimeStampedModel):- name = models.CharField(max_length=255)- board_game = models.ForeignKey(- BoardGame,- on_delete=models.CASCADE,- related_name="variants",- )- description = models.TextField(**BNULL)- uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)-- def __str__(self) -> str:- return f"{self.name} ({self.board_game.title})"- if data.get("bggId"): bgg_data = lookup_boardgame_from_bgg(lookup_id=data.get("bggId")) elif data.get("name"):@@ -398,6 +405,7 @@ class BoardGameVariant(TimeStampedModel): else: bgg_data = lookup_boardgame_from_bgg(title=lookup_id) + expansions = bgg_data.pop("expansions", []) mechanics = bgg_data.pop("mechanics", []) designers = bgg_data.pop("designers", []) categories = bgg_data.pop("categories", [])@@ -430,4 +438,22 @@ class BoardGameVariant(TimeStampedModel): publisher, _ = BoardGamePublisher.objects.get_or_create(name=name) game.publishers.add(publisher) + from boardgames.utils import fetch_and_link_expansions++ fetch_and_link_expansions(game, expansions)+ return game+++class BoardGameVariant(TimeStampedModel):+ name = models.CharField(max_length=255)+ board_game = models.ForeignKey(+ BoardGame,+ on_delete=models.CASCADE,+ related_name="variants",+ )+ description = models.TextField(**BNULL)+ uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)++ def __str__(self) -> str:+ return f"{self.name} ({self.board_game.title})"@@ -36,5 +36,8 @@ def lookup_boardgame_from_bgg( game_dict["publishers"] = game.publishers if game.publishers: game_dict["publisher"] = game.publishers[0]+ game_dict["expansions"] = [+ {"id": exp.id, "name": exp.name} for exp in game.expansions+ ] return game_dict@@ -1,3 +1,5 @@+from unittest.mock import patch+ import pytest from django.contrib.auth import get_user_model @@ -63,3 +65,161 @@ def test_board_game_variant_cascade_delete(): ) game.delete() assert BoardGameVariant.objects.count() == 0+++def _mock_bgg_game(bggeek_id, title, expansions=None):+ """Build a fake BGG game object shape used by lookup_boardgame_from_bgg."""+ class FakeGame:+ id = bggeek_id+ name = title+ description = f"Description of {title}"+ yearpublished = 2020+ image = "https://example.com/cover.jpg"+ minplayers = 1+ maxplayers = 4+ minage = 8+ rating_average = 7.5+ bgg_rank = 100+ playingtime = 60+ mechanics = []+ categories = []+ designers = []+ publishers = []++ @property+ def expansions(self):+ if expansions is None:+ return []+ return expansions++ return FakeGame()+++def _mock_bgg_client(return_game):+ """Return a callable that creates a fake BGGClient instance."""+ class FakeBGGClient:+ def __init__(self, access_token=None):+ pass++ def game(self, game_id=None, name=None):+ return return_game++ return FakeBGGClient()+++@pytest.mark.django_db+@patch("boardgames.models.requests.get")+@patch("boardgames.sources.bgg.BGGClient")+def test_find_or_create_links_expansions(mock_bgg, mock_get):+ exp1 = type("Thing", (), {"id": 200, "name": "Expansion 1"})()+ exp2 = type("Thing", (), {"id": 201, "name": "Expansion 2"})()+ base_game = _mock_bgg_game("100", "Base Game", expansions=[exp1, exp2])+ exp1_game = _mock_bgg_game("200", "Expansion 1")+ exp2_game = _mock_bgg_game("201", "Expansion 2")+ mock_bgg.side_effect = [+ _mock_bgg_client(base_game),+ _mock_bgg_client(exp1_game),+ _mock_bgg_client(exp2_game),+ ]+ mock_get.return_value.status_code = 200+ mock_get.return_value.content = b"fake_image_data"++ game = BoardGame.find_or_create("100")+ assert game.title == "Base Game"++ expansions = BoardGame.objects.filter(expansion_for_boardgame=game)+ assert expansions.count() == 2+ assert {e.title for e in expansions} == {"Expansion 1", "Expansion 2"}+++@pytest.mark.django_db+@patch("boardgames.sources.bgg.BGGClient")+def test_find_or_create_skips_expansions_for_existing_game(mock_bgg):+ game = BoardGame.objects.create(title="Base Game", bggeek_id="100")+ mock_bgg.assert_not_called()++ result = BoardGame.find_or_create("100")+ assert result == game+++@pytest.mark.django_db+@patch("boardgames.models.requests.get")+@patch("boardgames.sources.bgg.BGGClient")+def test_fetch_and_link_expansions(mock_bgg, mock_get):+ game = BoardGame.objects.create(title="Base Game", bggeek_id="100")+ exp_game = _mock_bgg_game("200", "Expansion 1")+ mock_bgg.side_effect = [_mock_bgg_client(exp_game)]+ mock_get.return_value.status_code = 200+ mock_get.return_value.content = b"fake_image_data"++ from boardgames.utils import fetch_and_link_expansions++ fetch_and_link_expansions(game, [{"id": 200, "name": "Expansion 1"}])++ expansion = BoardGame.objects.get(bggeek_id="200")+ assert expansion.expansion_for_boardgame == game+++@pytest.mark.django_db+@patch("boardgames.models.requests.get")+@patch("boardgames.sources.bgg.BGGClient")+def test_fix_metadata_links_expansions(mock_bgg, mock_get):+ game = BoardGame.objects.create(title="Base Game", bggeek_id="100")+ exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()+ base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])+ exp_game = _mock_bgg_game("200", "Expansion 1")+ # first call = fix_metadata -> lookup_boardgame_from_bgg(lookup_id='100')+ # second call = find_or_create inside fetch_and_link_expansions for expansion+ mock_bgg.side_effect = [+ _mock_bgg_client(base_with_exp),+ _mock_bgg_client(exp_game),+ ]+ mock_get.return_value.status_code = 200+ mock_get.return_value.content = b"fake_image_data"++ game.fix_metadata(force_update=True)++ expansion = BoardGame.objects.get(bggeek_id="200")+ assert expansion.expansion_for_boardgame == game+++@pytest.mark.django_db+@patch("boardgames.sources.bgg.BGGClient")+def test_management_command_fetch_expansions_dry_run(mock_bgg, capsys):+ from django.core.management import call_command++ game = BoardGame.objects.create(title="Base Game", bggeek_id="100")+ exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()+ base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])+ mock_bgg.side_effect = [_mock_bgg_client(base_with_exp)]++ call_command("fetch_expansions")+ captured = capsys.readouterr()+ assert "Would link 1 expansions" in captured.out++ assert BoardGame.objects.filter(bggeek_id="200").count() == 0+++@pytest.mark.django_db+@patch("boardgames.models.requests.get")+@patch("boardgames.sources.bgg.BGGClient")+def test_management_command_fetch_expansions_commit(mock_bgg, mock_get, capsys):+ from django.core.management import call_command++ game = BoardGame.objects.create(title="Base Game", bggeek_id="100")+ exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()+ exp_game = _mock_bgg_game("200", "Expansion 1")+ base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])+ mock_bgg.side_effect = [+ _mock_bgg_client(base_with_exp),+ _mock_bgg_client(exp_game),+ ]+ mock_get.return_value.status_code = 200+ mock_get.return_value.content = b"fake_image_data"++ call_command("fetch_expansions", commit=True)+ captured = capsys.readouterr()+ assert "Updated 1 games" in captured.out++ expansion = BoardGame.objects.get(bggeek_id="200")+ assert expansion.expansion_for_boardgame == game@@ -1,4 +1,5 @@ import logging+from typing import Any from boardgames.models import BoardGame, BoardGameVariant @@ -35,3 +36,27 @@ def board_names_to_variants( ) variant_ids.append(variant.id) return variant_ids+++def fetch_and_link_expansions(+ board_game: BoardGame, expansions_data: list[dict[str, Any]]+) -> None:+ """Given a board game and a list of expansion dicts (with 'id' and 'name'),+ find or create each expansion BoardGame and link it via expansion_for_boardgame.+ """+ for exp_data in expansions_data:+ exp_id = exp_data.get("id")+ if not exp_id:+ continue+ expansion = BoardGame.find_or_create(str(exp_id))+ if expansion and expansion.id != board_game.id:+ expansion.expansion_for_boardgame = board_game+ expansion.save(update_fields=["expansion_for_boardgame"])+ logger.info(+ "Linked expansion to board game",+ extra={+ "board_game_id": board_game.id,+ "expansion_id": expansion.id,+ "expansion_name": expansion.title,+ },+ )@@ -1235,6 +1235,15 @@ class ScrobbleDetailView(DetailView): def get_form_class(self): return self.object.media_obj.logdata_cls().form() + def _update_expansion_ids_queryset(self, form):+ from boardgames.models import BoardGame++ if isinstance(self.object.media_obj, BoardGame) and "expansion_ids" in form.fields:+ expansions = BoardGame.objects.filter(+ expansion_for_boardgame=self.object.media_obj+ )+ form.fields["expansion_ids"].queryset = expansions+ def get_form(self): FormClass = self.get_form_class() @@ -1245,12 +1254,15 @@ class ScrobbleDetailView(DetailView): else: log["notes"] = self.object.logdata.notes_as_str(separator="\n") - return FormClass(initial=log)+ form = FormClass(initial=log)+ self._update_expansion_ids_queryset(form)+ return form def post(self, request, *args, **kwargs): self.object = self.get_object() FormClass = self.get_form_class() form = FormClass(request.POST)+ self._update_expansion_ids_queryset(form) if form.is_valid(): data = form.cleaned_data.copy()@@ -1263,6 +1275,9 @@ class ScrobbleDetailView(DetailView): if data.get("with_people_ids") is not None: data["with_people_ids"] = [p.id for p in data["with_people_ids"]] + if data.get("expansion_ids") is not None:+ data["expansion_ids"] = [e.id for e in data["expansion_ids"]]+ if data.get("mood_reason_ids") is not None: data["mood_reason_ids"] = [r.id for r in data["mood_reason_ids"]]