This commit has no recorded session.
diff
commit 6fde9ec8d2f12630109946d7bfbbf2113d6677a1Author: Colin Powell <colin@unbl.ink>Date: Thu Jun 18 10:43:21 2026 -0400 [webpages] Add autotagging to webpages@@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/20] :vrobbler:project:personal:+* Backlog [1/21] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb@@ -590,6 +590,18 @@ We should rename `email_scrobble_board_game` to reflect the fact that it's just a helper method to create board game scrobbles given a json blob. It's independent of the email flow it was originally creatdd for +** DONE [#B] Add autotagging to webpages based on domain, title :webpages:metadata:+:PROPERTIES:+:ID: f658435b-f7a0-42e6-b9f6-226678a77a55+:END:++*** Description++For easier filtering, like we do with tasks, we should auto tag WebPage instances+based on the domain name split part by periods (so news.ycombinator.com tags: news, ycombinator, com)++And also based on the nouns in the title.+ * Version 54.5 [1/1] ** DONE Fix bug in generating mood trends :trends: :PROPERTIES:@@ -1,4 +1,5 @@ import hashlib+import html import logging import requests import re@@ -795,6 +796,7 @@ def tokenize_title_to_tags(title: str) -> list[str]: if not title: return [] + title = html.unescape(title) cleaned = re.sub(r"[\(\)\[\]\{\}]", "", title) cleaned = re.sub(r"[^\w\s]", "", cleaned) @@ -3,3 +3,6 @@ from django.apps import AppConfig class WebpagesConfig(AppConfig): name = "webpages"++ def ready(self):+ import webpages.signals # noqa@@ -0,0 +1,68 @@+import re++from django.core.management.base import BaseCommand++from scrobbles.utils import tokenize_title_to_tags+from webpages.models import WebPage+++def _clean(s: str) -> str:+ return re.sub(r"[^\x20-\x7e]", "", s)+++class Command(BaseCommand):+ help = "Backfill auto tags on webpages from domain and title"++ def add_arguments(self, parser):+ parser.add_argument(+ "--commit",+ action="store_true",+ help="Actually add tags",+ )++ def handle(self, *args, **options):+ commit = options["commit"]++ webpages = WebPage.objects.all()+ total = webpages.count()+ updated_count = 0+ skipped_count = 0++ for i, webpage in enumerate(webpages.iterator(), start=1):+ new_tags = set()++ if webpage.domain:+ parts = webpage.domain.root.split(".")+ for part in parts:+ part = part.strip().lower()+ if part and part != "www":+ new_tags.add(part)++ if webpage.title:+ title_tags = tokenize_title_to_tags(webpage.title)+ new_tags.update(title_tags)++ existing_tags = {+ t.name for t in webpage.tags.all()+ }+ tags_to_add = new_tags - existing_tags++ if tags_to_add:+ updated_count += 1+ if commit:+ for tag in tags_to_add:+ webpage.tags.add(tag)+ self.stdout.write(+ f"[{i}/{total}] Added tags to {_clean(str(webpage))}: "+ f"{sorted(tags_to_add)}"+ )+ else:+ self.stdout.write(+ f"[{i}/{total}] [DRY RUN] Would add tags to "+ f"{_clean(str(webpage))}: {sorted(tags_to_add)}"+ )+ else:+ skipped_count += 1++ self.stdout.write(f"\nDone. {updated_count} webpages to update, "+ f"{skipped_count} already up to date.")@@ -0,0 +1,29 @@+import logging++from django.db.models.signals import post_save+from django.dispatch import receiver++from scrobbles.utils import tokenize_title_to_tags+from webpages.models import WebPage++logger = logging.getLogger(__name__)+++@receiver(post_save, sender=WebPage)+def add_auto_tags_to_webpage(sender, instance, **kwargs):+ existing_tags = {t.name for t in instance.tags.all()}++ if instance.domain:+ domain_parts = instance.domain.root.split(".")+ for part in domain_parts:+ part = part.strip().lower()+ if part and part != "www" and part not in existing_tags:+ instance.tags.add(part)+ existing_tags.add(part)++ if instance.title:+ title_tags = tokenize_title_to_tags(instance.title)+ for tag in title_tags:+ if tag not in existing_tags:+ instance.tags.add(tag)+ existing_tags.add(tag)@@ -16,8 +16,23 @@ from webpages.models import WebPage class WebPageListView(ScrobbleableListView): model = WebPage + def get_queryset(self):+ queryset = super().get_queryset()+ tag_name = self.request.GET.get("tag")+ if tag_name:+ from django.contrib.contenttypes.models import ContentType+ from taggit.models import TaggedItem+ ct = ContentType.objects.get_for_model(WebPage)+ webpages = TaggedItem.objects.filter(+ content_type=ct,+ tag__name__iexact=tag_name,+ ).values_list("object_id", flat=True)+ queryset = queryset.filter(pk__in=webpages)+ return queryset+ def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs)+ context_data["active_tag"] = self.request.GET.get("tag", "") user = self.request.user now = timezone.now() start_day_of_week = now - datetime.timedelta(days=now.weekday())@@ -204,6 +204,13 @@ <p>Source: <a href="{{object.url}}">{{object.domain}}</a></p> {% if object.date %}<p>Published: <em>{{object.date}}</em></p>{% endif %} <p>Time to read: {{object.estimated_time_to_read_in_minutes}} minutes</p>+ {% if object.tags.all %}+ <p>Tags:+ {% for tag in object.tags.all %}+ <a href="{% url 'webpages:webpage_list' %}?tag={{ tag.name|urlencode }}">{{ tag.name }}</a>{% if not forloop.last %}, {% endif %}+ {% endfor %}+ </p>+ {% endif %} </div> {% if object.extract %} <div class="col">