This commit has no recorded session.
diff
commit cb23d5a5be20e418403f1b0444611cfc2138d8dbAuthor: Colin Powell <colin@unbl.ink>Date: Tue Jun 30 16:03:29 2026 -0400 [metadata] Fix cleanup scripts to check for dead images@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/23] :vrobbler:project:personal:+* Backlog [1/24] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES:@@ -605,6 +605,10 @@ independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** DONE [#B] Cleanup commands should check for broken images :metadata:cleanup:+:PROPERTIES:+:ID: bacce321-73c7-ae1f-bfa7-c3ee517b5441+:END: * Version 58.5 [1/1] ** DONE [#A] The maloja style charts are messed up :templates:charts: :PROPERTIES:@@ -18,8 +18,17 @@ MISSING_ALL = [ "publish_year", ] +def _cover_missing_or_broken(book) -> bool:+ if not bool(book.cover):+ return True+ try:+ return not book.cover.storage.exists(book.cover.name)+ except Exception:+ return True++ MISSING_GROUPS = {- "cover": lambda b: not bool(b.cover),+ "cover": _cover_missing_or_broken, "summary": lambda b: not b.summary, "isbn": lambda b: not b.isbn_13 and not b.isbn_10, "pages": lambda b: b.pages is None,@@ -0,0 +1,202 @@+import logging++from django.core.management.base import BaseCommand+from django.db import models, transaction++logger = logging.getLogger(__name__)+++class Command(BaseCommand):+ help = "Enrich artist and album metadata (covers, thumbnails) from MusicBrainz and TheAudioDB"++ def add_arguments(self, parser):+ parser.add_argument(+ "--force",+ action="store_true",+ help="Overwrite existing cover image and metadata",+ )+ parser.add_argument(+ "--dry-run",+ action="store_true",+ help="Show what would be done without making changes",+ )+ parser.add_argument(+ "--artists",+ action="store_true",+ help="Only process artists",+ )+ parser.add_argument(+ "--albums",+ action="store_true",+ help="Only process albums",+ )+ parser.add_argument(+ "--needs-metadata",+ action="store_true",+ help="Only process items missing metadata or with broken images",+ )++ def _has_broken_image(self, obj, field_name: str) -> bool:+ field = getattr(obj, field_name, None)+ if not field or not field.name:+ return False+ try:+ return not field.storage.exists(field.name)+ except Exception:+ return True++ def handle(self, *args, **options):+ from music.models import Album, Artist++ force = options["force"]+ dry_run = options["dry_run"]+ only_artists = options["artists"]+ only_albums = options["albums"]+ needs_metadata = options["needs_metadata"]++ if not only_artists and not only_albums:+ only_artists = only_albums = True++ updated_total = 0+ errors_total = 0++ if only_artists:+ updated_total += self._process_artists(force, dry_run, needs_metadata)+ errors_total = 0 # reset per section++ if only_albums:+ updated_total += self._process_albums(force, dry_run, needs_metadata)++ self.stdout.write(+ self.style.SUCCESS(f"\nDone! {updated_total} items processed")+ )++ def _get_artists(self, needs_metadata: bool):+ from music.models import Artist++ qs = Artist.objects.all()+ if needs_metadata:+ qs = qs.filter(+ models.Q(theaudiodb_id__isnull=True)+ | models.Q(theaudiodb_id="")+ | models.Q(thumbnail__isnull=True)+ | models.Q(thumbnail="")+ )++ broken = []+ if needs_metadata:+ broken_qs = Artist.objects.exclude(+ models.Q(thumbnail__isnull=True) | models.Q(thumbnail=""),+ )+ for artist in broken_qs.iterator():+ if self._has_broken_image(artist, "thumbnail"):+ broken.append(artist)++ return list(qs) + broken++ def _get_albums(self, needs_metadata: bool):+ from music.models import Album++ qs = Album.objects.all()+ if needs_metadata:+ qs = qs.filter(+ models.Q(cover_image__isnull=True)+ | models.Q(cover_image="")+ | models.Q(cover_image="default-image-replace-me")+ )++ broken = []+ if needs_metadata:+ broken_qs = Album.objects.exclude(+ models.Q(cover_image__isnull=True) | models.Q(cover_image=""),+ )+ for album in broken_qs.iterator():+ if self._has_broken_image(album, "cover_image"):+ broken.append(album)++ return list(qs) + broken++ def _process_artists(self, force, dry_run, needs_metadata):+ from music.models import Artist++ artists = self._get_artists(needs_metadata) if needs_metadata else list(Artist.objects.all())+ total = len(artists)+ self.stdout.write(f"Processing {total} artists")++ if dry_run:+ for artist in artists:+ has_tadb = bool(artist.theaudiodb_id)+ has_thumb = bool(artist.thumbnail)+ thumb_broken = self._has_broken_image(artist, "thumbnail")+ status = f"theaudiodb_id={'✓' if has_tadb else '✗'}"+ if thumb_broken:+ status += ", thumbnail=BROKEN"+ elif has_thumb:+ status += ", thumbnail=✓"+ else:+ status += ", thumbnail=✗"+ self.stdout.write(f" [DRY RUN] Would fix {artist.name} ({status})")+ return 0++ updated = 0+ errors = 0+ for artist in artists:+ try:+ with transaction.atomic():+ artist.fix_metadata(+ force_update=force or self._has_broken_image(artist, "thumbnail")+ )+ updated += 1+ self.stdout.write(f" [ARTIST {updated}/{total}] {artist.name}")+ except Exception as e:+ errors += 1+ self.stdout.write(+ self.style.ERROR(f" Error updating artist {artist.name}: {e}")+ )++ self.stdout.write(+ self.style.SUCCESS(f"\nArtists done! {updated} updated, {errors} errors")+ )+ return updated++ def _process_albums(self, force, dry_run, needs_metadata):+ from music.models import Album++ albums = self._get_albums(needs_metadata) if needs_metadata else list(Album.objects.all())+ total = len(albums)+ self.stdout.write(f"Processing {total} albums")++ if dry_run:+ for album in albums:+ has_cover = bool(album.cover_image)+ cover_broken = self._has_broken_image(album, "cover_image")+ if cover_broken:+ status = "cover=BROKEN"+ elif has_cover:+ status = "cover=✓"+ else:+ status = "cover=✗"+ self.stdout.write(f" [DRY RUN] Would fix {album.name} ({status})")+ return 0++ updated = 0+ errors = 0+ for album in albums:+ try:+ with transaction.atomic():+ if self._has_broken_image(album, "cover_image") or force:+ album.fetch_artwork(force=True)+ else:+ album.fix_metadata()+ updated += 1+ self.stdout.write(f" [ALBUM {updated}/{total}] {album.name}")+ except Exception as e:+ errors += 1+ self.stdout.write(+ self.style.ERROR(f" Error updating album {album.name}: {e}")+ )++ self.stdout.write(+ self.style.SUCCESS(f"\nAlbums done! {updated} updated, {errors} errors")+ )+ return updated@@ -17,8 +17,8 @@ MISSING_ALL = [ ] MISSING_GROUPS = {- "cover": lambda g: not bool(g.cover),- "screenshot": lambda g: not bool(g.screenshot),+ "cover": lambda g: _image_missing_or_broken(g, "cover"),+ "screenshot": lambda g: _image_missing_or_broken(g, "screenshot"), "summary": lambda g: not g.summary, "rating": lambda g: g.rating is None, "release_date": lambda g: g.release_date is None,@@ -28,6 +28,16 @@ MISSING_GROUPS = { } +def _image_missing_or_broken(game, field_name) -> bool:+ field = getattr(game, field_name)+ if not bool(field):+ return True+ try:+ return not field.storage.exists(field.name)+ except Exception:+ return True++ def _game_matches(game, flags): if not flags: return False@@ -103,6 +113,7 @@ class Command(BaseCommand): if all_missing: flags = MISSING_ALL+ fix_broken_images = True if not flags and not game_id and not force and not fix_broken_images: self.stdout.write(@@ -30,6 +30,19 @@ class Command(BaseCommand): action="store_true", help="Only process channels with a twitch_id", )+ parser.add_argument(+ "--needs-metadata",+ action="store_true",+ help="Only process channels missing youtube_id, twitch_id, cover image, or with broken cover image",+ )++ def _has_broken_image(self, channel) -> bool:+ if not channel.cover_image or not channel.cover_image.name:+ return False+ try:+ return not channel.cover_image.storage.exists(channel.cover_image.name)+ except Exception:+ return True def handle(self, *args, **options): from videos.models import Channel@@ -38,6 +51,7 @@ class Command(BaseCommand): dry_run = options["dry_run"] youtube_only = options["youtube_only"] twitch_only = options["twitch_only"]+ needs_metadata = options["needs_metadata"] qs = Channel.objects.all() @@ -45,29 +59,61 @@ class Command(BaseCommand): qs = qs.exclude(youtube_id__isnull=True).exclude(youtube_id="") elif twitch_only: qs = qs.exclude(twitch_id__isnull=True).exclude(twitch_id="")+ elif needs_metadata:+ no_id = models.Q(youtube_id__isnull=True) & models.Q(twitch_id__isnull=True)+ no_id |= models.Q(youtube_id="") & models.Q(twitch_id="")+ no_id |= models.Q(youtube_id__isnull=True) & models.Q(twitch_id="")+ no_id |= models.Q(youtube_id="") & models.Q(twitch_id__isnull=True)+ qs = qs.filter(+ no_id+ | models.Q(cover_image__isnull=True)+ | models.Q(cover_image="")+ ) else: qs = qs.filter( models.Q(youtube_id__isnull=False) | models.Q(twitch_id__isnull=False) ).exclude(youtube_id="", twitch_id="") total = qs.count()- self.stdout.write(f"Processing {total} channels")+ self.stdout.write(f"Processing {total} channels from DB filter")++ broken_channels = []+ if needs_metadata:+ broken_qs = Channel.objects.filter(+ cover_image__isnull=False,+ ).exclude(+ cover_image="",+ )+ if youtube_only:+ broken_qs = broken_qs.exclude(youtube_id__isnull=True).exclude(youtube_id="")+ elif twitch_only:+ broken_qs = broken_qs.exclude(twitch_id__isnull=True).exclude(twitch_id="")+ for channel in broken_qs.iterator():+ if self._has_broken_image(channel):+ broken_channels.append(channel)++ all_channels = list(qs) + broken_channels+ total = len(all_channels)+ self.stdout.write(f"Total channels to process: {total}") if dry_run:- for channel in qs.iterator():+ for channel in all_channels: source = "youtube" if channel.youtube_id else "twitch" identifier = channel.youtube_id or channel.twitch_id+ status = f"({source}: {identifier})"+ if self._has_broken_image(channel):+ status += " [image BROKEN]" self.stdout.write(- f" [DRY RUN] Would fix {channel.name} ({source}: {identifier})"+ f" [DRY RUN] Would fix {channel.name} {status}" ) return updated = 0 errors = 0- for channel in qs.iterator():+ for channel in all_channels: try: with transaction.atomic():- channel.fix_metadata(force=force)+ channel.fix_metadata(force=force or self._has_broken_image(channel)) updated += 1 source = "youtube" if channel.youtube_id else "twitch" self.stdout.write(f" [{updated}/{total}] {channel.name} ({source})")@@ -1,4 +1,5 @@ import logging+import os from django.core.management.base import BaseCommand from django.db import models, transaction@@ -28,9 +29,18 @@ class Command(BaseCommand): parser.add_argument( "--needs-metadata", action="store_true",- help="Only process series missing imdb_id or cover image",+ help="Only process series missing imdb_id or with broken cover image", ) + def _has_broken_image(self, series) -> bool:+ """Check if a series has a cover_image set but the file is missing."""+ if not series.cover_image:+ return False+ try:+ return not os.path.exists(series.cover_image.path)+ except Exception:+ return True+ def handle(self, *args, **options): from videos.models import Series @@ -51,25 +61,50 @@ class Command(BaseCommand): ) total = qs.count()- self.stdout.write(f"Processing {total} series")+ self.stdout.write(f"Processing {total} series from DB filter")++ # Also find series with broken cover images+ broken_image_series = []+ if needs_metadata:+ broken_qs = Series.objects.filter(+ cover_image__isnull=False,+ ).exclude(+ models.Q(imdb_id__isnull=True)+ | models.Q(imdb_id="")+ | models.Q(cover_image__isnull=True)+ | models.Q(cover_image=""),+ )+ if imdb_id:+ broken_qs = broken_qs.filter(imdb_id=imdb_id)+ for series in broken_qs.iterator():+ if self._has_broken_image(series):+ broken_image_series.append(series)++ all_series = list(qs) + broken_image_series+ total = len(all_series)+ self.stdout.write(f"Total series to process: {total}") if dry_run:- for series in qs.iterator():+ for series in all_series: has_imdb = bool(series.imdb_id) has_image = bool(series.cover_image)- self.stdout.write(- f" [DRY RUN] Would fix {series.name}"- f" (imdb_id={'✓' if has_imdb else '✗'}"- f", image={'✓' if has_image else '✗'})"- )+ image_broken = self._has_broken_image(series)+ status = f"imdb_id={'✓' if has_imdb else '✗'}"+ if image_broken:+ status += ", image=BROKEN"+ elif has_image:+ status += ", image=✓"+ else:+ status += ", image=✗"+ self.stdout.write(f" [DRY RUN] Would fix {series.name} ({status})") return updated = 0 errors = 0- for series in qs.iterator():+ for series in all_series: try: with transaction.atomic():- series.fix_metadata(force_update=force)+ series.fix_metadata(force_update=force or self._has_broken_image(series)) updated += 1 self.stdout.write(f" [{updated}/{total}] {series.name}") except Exception as e: