This commit has no recorded session.
diff
commit 033239260f31eecf8cfffcf99541ad88f938e695Author: Colin Powell <colin@unbl.ink>Date: Fri Jun 19 13:37:30 2026 -0400 [perf] Add caching and lock protections@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/22] :vrobbler:project:personal:+* Backlog [0/23] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb@@ -604,6 +604,11 @@ independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** STRT [#A] Tighten up the speed of startup and first request :perf:+:PROPERTIES:+:ID: 9ee8834c-6be2-d04b-df6d-56375504083f+:END:+ * Version 55.3 [3/3] ** DONE [#C] =alt_names= feature for artists (commented out / dead code) :music:dead-code: :PROPERTIES:@@ -4,9 +4,15 @@ from unittest.mock import MagicMock, patch import pytest +from vrobbler import context_processors from vrobbler.context_processors import version_info +@pytest.fixture(autouse=True)+def reset_git_cache():+ context_processors._GIT_COMMIT = None++ @pytest.fixture def mock_request(): return MagicMock()@@ -0,0 +1,109 @@+# Generated by Django 4.2.29 on 2026-06-19 16:25++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("charts", "0002_chartrecord_charts_char_user_id_1adcde_idx_and_more"),+ ]++ operations = [+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("artist__isnull", False)),+ fields=("user", "year", "month", "week", "day", "artist"),+ name="unique_chart_artist_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("album__isnull", False)),+ fields=("user", "year", "month", "week", "day", "album"),+ name="unique_chart_album_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("track__isnull", False)),+ fields=("user", "year", "month", "week", "day", "track"),+ name="unique_chart_track_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("tv_series__isnull", False)),+ fields=("user", "year", "month", "week", "day", "tv_series"),+ name="unique_chart_tv_series_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("video__isnull", False)),+ fields=("user", "year", "month", "week", "day", "video"),+ name="unique_chart_video_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("podcast__isnull", False)),+ fields=("user", "year", "month", "week", "day", "podcast"),+ name="unique_chart_podcast_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("podcast_episode__isnull", False)),+ fields=("user", "year", "month", "week", "day", "podcast_episode"),+ name="unique_chart_podcast_episode_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("board_game__isnull", False)),+ fields=("user", "year", "month", "week", "day", "board_game"),+ name="unique_chart_board_game_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("trail__isnull", False)),+ fields=("user", "year", "month", "week", "day", "trail"),+ name="unique_chart_trail_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("geo_location__isnull", False)),+ fields=("user", "year", "month", "week", "day", "geo_location"),+ name="unique_chart_geo_location_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("food__isnull", False)),+ fields=("user", "year", "month", "week", "day", "food"),+ name="unique_chart_food_period",+ ),+ ),+ migrations.AddConstraint(+ model_name="chartrecord",+ constraint=models.UniqueConstraint(+ condition=models.Q(("book__isnull", False)),+ fields=("user", "year", "month", "week", "day", "book"),+ name="unique_chart_book_period",+ ),+ ),+ ]@@ -2,6 +2,7 @@ import calendar from django.contrib.auth import get_user_model from django.db import models+from django.db.models import Q from django.urls import reverse from django_extensions.db.models import TimeStampedModel @@ -84,6 +85,68 @@ class ChartRecord(TimeStampedModel): models.Index(fields=["user", "year", "month", "day", "album", "rank"]), models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]), ]+ constraints = [+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "artist"],+ condition=Q(artist__isnull=False),+ name="unique_chart_artist_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "album"],+ condition=Q(album__isnull=False),+ name="unique_chart_album_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "track"],+ condition=Q(track__isnull=False),+ name="unique_chart_track_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "tv_series"],+ condition=Q(tv_series__isnull=False),+ name="unique_chart_tv_series_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "video"],+ condition=Q(video__isnull=False),+ name="unique_chart_video_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "podcast"],+ condition=Q(podcast__isnull=False),+ name="unique_chart_podcast_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "podcast_episode"],+ condition=Q(podcast_episode__isnull=False),+ name="unique_chart_podcast_episode_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "board_game"],+ condition=Q(board_game__isnull=False),+ name="unique_chart_board_game_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "trail"],+ condition=Q(trail__isnull=False),+ name="unique_chart_trail_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "geo_location"],+ condition=Q(geo_location__isnull=False),+ name="unique_chart_geo_location_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "food"],+ condition=Q(food__isnull=False),+ name="unique_chart_food_period",+ ),+ models.UniqueConstraint(+ fields=["user", "year", "month", "week", "day", "book"],+ condition=Q(book__isnull=False),+ name="unique_chart_book_period",+ ),+ ] @property def media_obj(self):@@ -6,6 +6,7 @@ from typing import Optional import pytz from django.apps import apps from django.conf import settings+from django.db import transaction from django.db.models import Count, Q from django.utils import timezone @@ -186,60 +187,64 @@ def build_charts( ranks = {count: rank for rank, count in enumerate(unique_counts, start=1)} media_field = f"{media_type}_id"- records_to_create = []- records_to_update = [] - existing = ChartRecord.objects.filter(- period_filter, user=user, **{media_field + "__isnull": False}- )- existing_by_media_id = {getattr(r, media_field): r for r in existing}- found_media_ids = set()-- for result in results:- media_id = result[config["values"]]- if media_id is None:- continue-- found_media_ids.add(media_id)-- chart_record_data = {- "user_id": user.id,- "year": year,- "month": month,- "week": week,- "day": day,- "rank": ranks[result["scrobble_count"]],- "count": result["scrobble_count"],- }- chart_record_data[media_field] = media_id-- if media_id in existing_by_media_id:- existing_record = existing_by_media_id[media_id]- existing_record.rank = chart_record_data["rank"]- existing_record.count = chart_record_data["count"]- records_to_update.append(existing_record)- else:- records_to_create.append(ChartRecord(**chart_record_data))-- ids_to_delete = [- r.id for r in existing if getattr(r, media_field) not in found_media_ids- ]- if ids_to_delete:- ChartRecord.objects.filter(id__in=ids_to_delete).delete()+ with transaction.atomic():+ records_to_create = []+ records_to_update = [] - if records_to_update:- ChartRecord.objects.bulk_update(- records_to_update, ["rank", "count"], batch_size=500+ existing = ChartRecord.objects.select_for_update().filter(+ period_filter, user=user, **{media_field + "__isnull": False}+ )+ existing_by_media_id = {getattr(r, media_field): r for r in existing}+ found_media_ids = set()++ for result in results:+ media_id = result[config["values"]]+ if media_id is None:+ continue++ found_media_ids.add(media_id)++ chart_record_data = {+ "user_id": user.id,+ "year": year,+ "month": month,+ "week": week,+ "day": day,+ "rank": ranks[result["scrobble_count"]],+ "count": result["scrobble_count"],+ }+ chart_record_data[media_field] = media_id++ if media_id in existing_by_media_id:+ existing_record = existing_by_media_id[media_id]+ existing_record.rank = chart_record_data["rank"]+ existing_record.count = chart_record_data["count"]+ records_to_update.append(existing_record)+ else:+ records_to_create.append(ChartRecord(**chart_record_data))++ ids_to_delete = [+ r.id+ for r in existing+ if getattr(r, media_field) not in found_media_ids+ ]+ if ids_to_delete:+ ChartRecord.objects.filter(id__in=ids_to_delete).delete()++ if records_to_update:+ ChartRecord.objects.bulk_update(+ records_to_update, ["rank", "count"], batch_size=500+ )++ if records_to_create:+ ChartRecord.objects.bulk_create(records_to_create, batch_size=500)++ logger.info(+ f"Built {len(records_to_create)} new, {len(records_to_update)} updated "+ f"chart records for {media_type}, period "+ f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}" )-- if records_to_create:- ChartRecord.objects.bulk_create(records_to_create, batch_size=500)-- logger.info(- f"Built {len(records_to_create)} new, {len(records_to_update)} updated "- f"chart records for {media_type}, period "- f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}"- ) def build_yesterdays_charts(user, media_types: Optional[list] = None) -> None:@@ -1,8 +1,22 @@+from django.core.cache import cache+ from music.models import Artist, Album +CACHE_TTL = 300+ def music_lists(request):+ artist_list = cache.get("music_lists_artist_list")+ if artist_list is None:+ artist_list = list(Artist.objects.all().only("id", "name"))+ cache.set("music_lists_artist_list", artist_list, CACHE_TTL)++ album_list = cache.get("music_lists_album_list")+ if album_list is None:+ album_list = list(Album.objects.all().only("id", "name"))+ cache.set("music_lists_album_list", album_list, CACHE_TTL)+ return {- "artist_list": Artist.objects.all(),- "album_list": Album.objects.all(),+ "artist_list": artist_list,+ "album_list": album_list, }@@ -1,4 +1,5 @@ import pytz+from django.core.cache import cache from django.utils import timezone from scrobbles.constants import EXCLUDE_FROM_NOW_PLAYING@@ -19,6 +20,8 @@ MONTH_COLORS = [ "#db7a7a", # Dec ] +CACHE_TTL = 60+ def month_color(request): from datetime import date@@ -27,15 +30,25 @@ def month_color(request): def now_playing(request): user = request.user- now = timezone.now() if not user.is_authenticated: return {}- return {- "now_playing_list": Scrobble.objects.filter(- in_progress=True,- is_paused=False,- user=user,- ).exclude(- media_type__in=EXCLUDE_FROM_NOW_PLAYING,++ cache_key = f"now_playing_list_{user.id}"+ now_playing_list = cache.get(cache_key)+ if now_playing_list is None:+ now_playing_list = list(+ Scrobble.objects.filter(+ in_progress=True,+ is_paused=False,+ user=user,+ )+ .exclude(+ media_type__in=EXCLUDE_FROM_NOW_PLAYING,+ )+ .select_related("track", "video", "podcast_episode") )+ cache.set(cache_key, now_playing_list, CACHE_TTL)++ return {+ "now_playing_list": now_playing_list, }@@ -1,5 +1,6 @@ import logging +from django.core.cache import cache from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.utils import timezone@@ -52,6 +53,11 @@ def _update_charts_for_timestamp(user, ts): if ts is None: return + lock_key = f"chart_update_{user.id}"+ if not cache.add(lock_key, "locked", timeout=30):+ logger.info(f"Chart update already queued for user {user.id}, skipping")+ return+ if timezone.is_naive(ts): ts = timezone.make_aware(ts) @@ -12,6 +12,7 @@ from charts.utils import ( from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model+from django.core.cache import cache from django.db import models from django.utils import timezone @@ -241,6 +242,11 @@ def update_charts_for_timestamp(user_id, year, month, day, week): logger.error(f"User with id {user_id} not found") return + lock_key = f"chart_update_running_{user_id}"+ if not cache.add(lock_key, "locked", timeout=300):+ logger.info(f"Chart update already running for user {user_id}, skipping")+ return+ try: build_daily_charts(user, year, month, day, CHARTABLE_MEDIA_TYPES) build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES)@@ -250,6 +256,8 @@ def update_charts_for_timestamp(user_id, year, month, day, week): logger.info(f"[charts] Updated charts for {user} on {date_str}") except Exception as e: logger.error(f"[charts] Failed to update charts: {e}")+ finally:+ cache.delete(lock_key) @shared_task@@ -1,8 +1,24 @@+from django.core.cache import cache+ from videos.models import Video, Series +CACHE_TTL = 300+ def video_lists(request):+ movie_list = cache.get("video_lists_movie_list")+ if movie_list is None:+ movie_list = list(+ Video.objects.filter(video_type=Video.VideoType.MOVIE).only("id", "title")+ )+ cache.set("video_lists_movie_list", movie_list, CACHE_TTL)++ series_list = cache.get("video_lists_series_list")+ if series_list is None:+ series_list = list(Series.objects.all().only("id", "name"))+ cache.set("video_lists_series_list", series_list, CACHE_TTL)+ return {- "movie_list": Video.objects.filter(video_type=Video.VideoType.MOVIE),- "series_list": Series.objects.all(),+ "movie_list": movie_list,+ "series_list": series_list, }@@ -4,46 +4,55 @@ from importlib.metadata import version as get_version from pathlib import Path +_GIT_COMMIT = None++ def version_info(request):+ global _GIT_COMMIT+ try: app_version = get_version("vrobbler") except Exception: app_version = "unknown" commit = os.environ.get("VROBBLER_COMMIT")- if not commit:- # Try to import from _commit.py module first+ if commit:+ return {"app_version": app_version, "git_commit": commit}++ if _GIT_COMMIT is not None:+ return {"app_version": app_version, "git_commit": _GIT_COMMIT}++ try:+ from vrobbler._commit import commit as _commit+ except ImportError:+ pass+ else:+ if _commit and _commit != "unknown":+ _GIT_COMMIT = _commit+ return {"app_version": app_version, "git_commit": _GIT_COMMIT}++ commit_file = Path("/var/lib/vrobbler/commit.txt")+ if commit_file.exists(): try:- from vrobbler._commit import commit as _commit- except ImportError:+ commit = commit_file.read_text().strip()+ if commit:+ _GIT_COMMIT = commit+ return {"app_version": app_version, "git_commit": _GIT_COMMIT}+ except OSError: pass- else:- if _commit and _commit != "unknown":- return {"app_version": app_version, "git_commit": _commit}-- # Try to read from commit file (written during deploy)- commit_file = Path("/var/lib/vrobbler/commit.txt")- if commit_file.exists():- try:- commit = commit_file.read_text().strip()- if commit:- return {"app_version": app_version, "git_commit": commit}- except OSError:- pass-- # Fall back to git command- PROJECT_ROOT = Path(__file__).resolve().parent.parent- try:- commit = (- subprocess.check_output(- ["git", "rev-parse", "--short", "HEAD"],- cwd=PROJECT_ROOT,- stderr=subprocess.DEVNULL,- )- .decode("utf-8")- .strip()++ PROJECT_ROOT = Path(__file__).resolve().parent.parent+ try:+ _GIT_COMMIT = (+ subprocess.check_output(+ ["git", "rev-parse", "--short", "HEAD"],+ cwd=PROJECT_ROOT,+ stderr=subprocess.DEVNULL, )- except (subprocess.SubprocessError, FileNotFoundError):- commit = "unknown"+ .decode("utf-8")+ .strip()+ )+ except (subprocess.SubprocessError, FileNotFoundError):+ _GIT_COMMIT = "unknown" - return {"app_version": app_version, "git_commit": commit}+ return {"app_version": app_version, "git_commit": _GIT_COMMIT}@@ -361,11 +361,11 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ #-from storages.backends import s3boto3- USE_S3_STORAGE = os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY if USE_S3_STORAGE:+ from storages.backends import s3boto3+ AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "") AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "") AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID")