This commit has no recorded session.
diff
commit c7f5d7d38432d593ad7e1a8fa842d715ab08ee26Author: Colin Powell <colin@unbl.ink>Date: Fri Jun 12 13:34:41 2026 -0400 [charts] Add index to speed things up@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/14] :vrobbler:project:personal:+* Backlog [1/16] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb@@ -518,6 +518,31 @@ favorited media objects. :PROPERTIES: :ID: 79f867c3-1288-4143-b6bf-2a452983ee9f :END:+** TODO [#C] Implement loguru into project :feature:loguru:logging:+:PROPERTIES:+:ID: efcd0c0a-db81-4518-9c23-5505d59e8ef5+:END:++*** Description++Would be great to formalize how we log so we can search for errors and such more+easily. And our exposure to PII is really low at this point in the project,+so we can probably use backtrace=True and diagnose=True to help us root cause+bugs faster.++** DONE [#A] Improve speed of index and chart pages :bug:scrobbles:perf:+:PROPERTIES:+:ID: 031a23f8-7c02-4926-9884-6654ceca16c2+:END:++*** Description++Over the last few releases, the home page and charts pages have gotten really+slow.++We should look into what's causing the slowness and maybe do more agressive+query optimization or caching.+ * Version 51.2 [2/2] ** DONE [#A] Fix bug where last page of book gets separate scrobble :bug:books:importers:koreader: :PROPERTIES:@@ -0,0 +1,146 @@+# Generated by Django 4.2.29 on 2026-06-12 16:09++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("charts", "0001_initial"),+ ]++ operations = [+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "album", "rank"],+ name="charts_char_user_id_1adcde_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "track", "rank"],+ name="charts_char_user_id_d18aab_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "video", "rank"],+ name="charts_char_user_id_de9f0a_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "board_game", "rank"],+ name="charts_char_user_id_d5d58f_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "book", "rank"],+ name="charts_char_user_id_e877cf_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "food", "rank"],+ name="charts_char_user_id_a0ad71_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "podcast", "rank"],+ name="charts_char_user_id_846b80_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "trail", "rank"],+ name="charts_char_user_id_54feba_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "album", "rank"],+ name="charts_char_user_id_a3dc49_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "track", "rank"],+ name="charts_char_user_id_4b01ab_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "video", "rank"],+ name="charts_char_user_id_2ac9d2_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "board_game", "rank"],+ name="charts_char_user_id_ba968a_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "book", "rank"],+ name="charts_char_user_id_e66751_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "food", "rank"],+ name="charts_char_user_id_d23f06_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "podcast", "rank"],+ name="charts_char_user_id_be8122_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "week", "trail", "rank"],+ name="charts_char_user_id_b94ea9_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "day", "artist", "rank"],+ name="charts_char_user_id_406e0e_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "day", "album", "rank"],+ name="charts_char_user_id_322b0d_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="chartrecord",+ index=models.Index(+ fields=["user", "year", "month", "day", "tv_series", "rank"],+ name="charts_char_user_id_aa44b7_idx",+ ),+ ),+ ]@@ -60,10 +60,29 @@ class ChartRecord(TimeStampedModel): models.Index(fields=["user", "year", "geo_location", "rank"]), models.Index(fields=["user", "year", "food", "rank"]), models.Index(fields=["user", "year", "book", "rank"]),- models.Index(fields=["user", "year", "week", "artist", "rank"]),- models.Index(fields=["user", "year", "week", "tv_series", "rank"]), models.Index(fields=["user", "year", "month", "artist", "rank"]),+ models.Index(fields=["user", "year", "month", "album", "rank"]),+ models.Index(fields=["user", "year", "month", "track", "rank"]), models.Index(fields=["user", "year", "month", "tv_series", "rank"]),+ models.Index(fields=["user", "year", "month", "video", "rank"]),+ models.Index(fields=["user", "year", "month", "board_game", "rank"]),+ models.Index(fields=["user", "year", "month", "book", "rank"]),+ models.Index(fields=["user", "year", "month", "food", "rank"]),+ models.Index(fields=["user", "year", "month", "podcast", "rank"]),+ models.Index(fields=["user", "year", "month", "trail", "rank"]),+ models.Index(fields=["user", "year", "week", "artist", "rank"]),+ models.Index(fields=["user", "year", "week", "album", "rank"]),+ models.Index(fields=["user", "year", "week", "track", "rank"]),+ models.Index(fields=["user", "year", "week", "tv_series", "rank"]),+ models.Index(fields=["user", "year", "week", "video", "rank"]),+ models.Index(fields=["user", "year", "week", "board_game", "rank"]),+ models.Index(fields=["user", "year", "week", "book", "rank"]),+ models.Index(fields=["user", "year", "week", "food", "rank"]),+ models.Index(fields=["user", "year", "week", "podcast", "rank"]),+ models.Index(fields=["user", "year", "week", "trail", "rank"]),+ models.Index(fields=["user", "year", "month", "day", "artist", "rank"]),+ models.Index(fields=["user", "year", "month", "day", "album", "rank"]),+ models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]), ] @property@@ -438,12 +438,14 @@ class ChartRecordView(TemplateView): return context def get_available_years(self, user):- return list(- ChartRecord.objects.filter(user=user)- .values_list("year", flat=True)- .distinct()- .order_by("-year")- )+ if not hasattr(self, "_available_years"):+ self._available_years = list(+ ChartRecord.objects.filter(user=user)+ .values_list("year", flat=True)+ .distinct()+ .order_by("-year")+ )+ return self._available_years def get_period_type(self): date_param = self.request.GET.get("date")@@ -0,0 +1,19 @@+# Generated by Django 4.2.29 on 2026-06-12 16:09++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("scrobbles", "0096_convert_book_page_data_to_dict"),+ ]++ operations = [+ migrations.AddIndex(+ model_name="scrobble",+ index=models.Index(+ fields=["user", "-timestamp"], name="scrobbles_s_user_id_d367a7_idx"+ ),+ ),+ ]@@ -612,6 +612,30 @@ class EBirdCSVImport(BaseFileImportMixin): self.mark_finished() +TYPE_FK_PREFETCHES: dict[str, tuple[str, ...]] = {+ "Video": ("video",),+ "Track": ("track", "track__artist_fk"),+ "PodcastEpisode": ("podcast_episode", "podcast_episode__podcast"),+ "SportEvent": ("sport_event",),+ "Book": ("book",),+ "Paper": ("paper",),+ "VideoGame": ("video_game",),+ "BoardGame": ("board_game",),+ "GeoLocation": ("geo_location",),+ "Trail": ("trail",),+ "Beer": ("beer",),+ "Puzzle": ("puzzle",),+ "Food": ("food",),+ "Task": ("task",),+ "WebPage": ("web_page",),+ "LifeEvent": ("life_event",),+ "Mood": ("mood",),+ "BrickSet": ("brick_set",),+ "Channel": ("channel",),+ "BirdingLocation": ("birding_location",),+}++ class ScrobbleQuerySet(models.QuerySet): def with_related(self): return self.select_related("user").prefetch_related(@@ -638,6 +662,13 @@ class ScrobbleQuerySet(models.QuerySet): "birding_location", ) + def with_related_for_types(self, media_types: list[str]):+ prefetches = []+ for t in media_types:+ if t in TYPE_FK_PREFETCHES:+ prefetches.extend(TYPE_FK_PREFETCHES[t])+ return self.select_related("user").prefetch_related(*prefetches)+ class ShareViewLog(TimeStampedModel): scrobble = models.ForeignKey(@@ -778,6 +809,7 @@ class Scrobble(TimeStampedModel): "is_paused", ] ),+ models.Index(fields=["user", "-timestamp"]), ] @classmethod@@ -810,11 +842,14 @@ class Scrobble(TimeStampedModel): @classmethod def as_dict_by_type(cls, scrobble_qs: models.QuerySet) -> dict: scrobbles_by_type = defaultdict(list)- scrobbles = (- scrobble_qs.with_related()- if hasattr(scrobble_qs, "with_related")- else scrobble_qs- )++ if hasattr(scrobble_qs, "with_related"):+ media_types_present = list(+ scrobble_qs.values_list("media_type", flat=True).distinct()+ )+ scrobbles = scrobble_qs.with_related_for_types(media_types_present)+ else:+ scrobbles = scrobble_qs for scrobble in scrobbles: scrobbles_by_type[scrobble.media_type].append(scrobble)@@ -829,7 +864,7 @@ class Scrobble(TimeStampedModel): # Remove any locations without titles if "GeoLocation" in scrobbles_by_type.keys():- for loc_scrobble in scrobbles_by_type["GeoLocation"]:+ for loc_scrobble in list(scrobbles_by_type["GeoLocation"]): if not loc_scrobble.media_obj.title: scrobbles_by_type["GeoLocation"].remove(loc_scrobble) scrobbles_by_type["GeoLocation_count"] -= 1