This commit has no recorded session.
diff
commit 5a2c41155c5e61dbf67abcb0683ab69301750867Author: Colin Powell <colin@unbl.ink>Date: Tue Jun 16 09:52:56 2026 -0400 [webpages] Add historical extract stashing@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/21] :vrobbler:project:personal:+* Backlog [2/22] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb@@ -594,6 +594,26 @@ named constants for maintainability. - ~vrobbler/apps/webpages/models.py~ (line 290) -- ="url"= - ~vrobbler/apps/scrobbles/importers/tsv.py~ (line 55) -- ="S"= completion status +** DONE [#B] Webpage scrobbles should diff existing webpages content :webpages:metadata:+:PROPERTIES:+:ID: 25576197-258f-48d6-bfe9-e4172a0a1898+:END:++*** Description++Webpages change content between scrobbles. The current model stores the webpage content once, the+first time it's scrobbled. When a page has been seen before, we should move the existing content+to a new model HistoricalWebPage with the following fields:++webpage_id -> FK to WebPage+date -> date from existing WebPage content+domain -> same as existing WebPage content+extract -> copy of existing WebPage content++Once the HistoricalWebPage instance is successfully created, the new extract data+should be saved into the WebPage instance.++ ** DONE [#B] Make ArchiveBox push asynchronous :archivebox:async: :PROPERTIES: :ID: 17c116a7-5952-db37-e56c-2987c2fc456b@@ -1,6 +1,6 @@ from django.contrib import admin -from webpages.models import Domain, WebPage+from webpages.models import Domain, HistoricalWebPage, WebPage from scrobbles.admin import ScrobbleInline @@ -20,6 +20,12 @@ class DomainAdmin(admin.ModelAdmin): inlines = [WebPageInline] +class HistoricalWebPageInline(admin.TabularInline):+ model = HistoricalWebPage+ extra = 0+ readonly_fields = ("date", "domain", "extract", "created")++ @admin.register(WebPage) class WebPageAdmin(admin.ModelAdmin): date_hierarchy = "created"@@ -33,4 +39,20 @@ class WebPageAdmin(admin.ModelAdmin): search_fields = ("title",) inlines = [ ScrobbleInline,+ HistoricalWebPageInline, ]+++@admin.register(HistoricalWebPage)+class HistoricalWebPageAdmin(admin.ModelAdmin):+ date_hierarchy = "created"+ list_display = (+ "uuid",+ "webpage",+ "date",+ "domain",+ "created",+ )+ raw_id_fields = ("webpage", "domain")+ ordering = ("-created",)+ search_fields = ("webpage__title",)@@ -0,0 +1,71 @@+# Generated by Django 4.2.29 on 2026-06-16 13:39++from django.db import migrations, models+import django.db.models.deletion+import django_extensions.db.fields+import uuid+++class Migration(migrations.Migration):++ dependencies = [+ ("webpages", "0009_alter_webpage_genre"),+ ]++ operations = [+ migrations.CreateModel(+ name="HistoricalWebPage",+ fields=[+ (+ "id",+ models.BigAutoField(+ auto_created=True,+ primary_key=True,+ serialize=False,+ verbose_name="ID",+ ),+ ),+ (+ "created",+ django_extensions.db.fields.CreationDateTimeField(+ auto_now_add=True, verbose_name="created"+ ),+ ),+ (+ "modified",+ django_extensions.db.fields.ModificationDateTimeField(+ auto_now=True, verbose_name="modified"+ ),+ ),+ (+ "uuid",+ models.UUIDField(+ blank=True, default=uuid.uuid4, editable=False, null=True+ ),+ ),+ ("date", models.DateField(blank=True, null=True)),+ ("extract", models.TextField(blank=True, null=True)),+ (+ "domain",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.DO_NOTHING,+ to="webpages.domain",+ ),+ ),+ (+ "webpage",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="historical_webpages",+ to="webpages.webpage",+ ),+ ),+ ],+ options={+ "get_latest_by": "modified",+ "abstract": False,+ },+ ),+ ]@@ -303,4 +303,41 @@ class WebPage(ScrobblableMixin): if not webpage: webpage = cls(url=data_dict.get("url")) webpage.fetch_data_from_web(save=True)+ else:+ webpage._archive_and_refetch() return webpage++ def _archive_and_refetch(self):+ """Archive current content to HistoricalWebPage and re-fetch from web."""+ if self.extract or self.date or self.domain:+ HistoricalWebPage.objects.create(+ webpage=self,+ date=self.date,+ domain=self.domain,+ extract=self.extract,+ )++ self.extract = None+ self.date = None+ self.domain = None+ self.title = None+ self.base_run_time_seconds = None+ self.image = None+ self.fetch_data_from_web(save=True, force=True)+++class HistoricalWebPage(TimeStampedModel):+ uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)+ webpage = models.ForeignKey(+ WebPage, on_delete=models.CASCADE, related_name="historical_webpages"+ )+ date = models.DateField(**BNULL)+ domain = models.ForeignKey(Domain, on_delete=models.DO_NOTHING, **BNULL)+ extract = models.TextField(**BNULL)++ def __str__(self) -> str:+ if self.webpage.title:+ return "{} ({}) - {}".format(+ self.webpage.title, self.webpage.domain, self.created+ )+ return "{} - {}".format(self.webpage.url, self.created)