accumulated fixes and improvements from neodb

.github/workflows/test.yml

@@ -25,7 +25,7 @@ jobs:
     timeout-minutes: 8
     strategy:
       matrix:
-        python-version: ["3.13", "3.14"]
+        python-version: ["3.14"]
         postgres-version: [14, 18]
     steps:
       - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -1,3 +1,6 @@
+# Hook envs need 3.14 so they can parse PEP 758 (unparenthesized except groups)
+default_language_version:
+  python: python3.14
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0

.python-version

@@ -1 +1 @@
-3.13
+3.14

activities/management/commands/pruneposts.py

@@ -37,7 +37,7 @@ def handle(self, number: int, *args, **options):
                 Q(interactions__identity__local=True)
                 | Q(visibility=Post.Visibilities.mentioned)
             )
-            .order_by("?")[:number]
+            .order_by("created")[:number]
         )
         post_ids_and_uris = dict(posts.values_list("object_uri", "id"))
         print(f"  found {len(post_ids_and_uris)}")

activities/migrations/0026_alter_timelineevent_type.py

@@ -24,6 +24,7 @@ class Migration(migrations.Migration):
                     ("quoted", "Quoted"),
                     ("announcement", "Announcement"),
                     ("identity_created", "Identity Created"),
+                    ("poll", "Poll"),
                 ],
                 max_length=100,
             ),

activities/migrations/0030_timelineevent_undismissed_idx.py

@@ -0,0 +1,21 @@
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("activities", "0029_post_url_db_index"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="timelineevent",
+            index=models.Index(
+                fields=["identity", "-id"],
+                condition=models.Q(dismissed=False),
+                name="te_identity_idneg_undismissed",
+            ),
+        ),
+    ]

activities/migrations/0031_quoteauthorization.py

@@ -0,0 +1,46 @@
+import core.snowflake
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("activities", "0030_timelineevent_undismissed_idx"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="QuoteAuthorization",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=core.snowflake.Snowflake.generate_post,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("interacting_object_uri", models.CharField(max_length=2048)),
+                (
+                    "request_uri",
+                    models.CharField(blank=True, max_length=2048, null=True),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "target_post",
+                    models.ForeignKey(
+                        on_delete=models.deletion.CASCADE,
+                        related_name="quote_authorizations",
+                        to="activities.post",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["target_post", "interacting_object_uri"],
+                        name="activities__target__448802_idx",
+                    ),
+                ],
+            },
+        ),
+    ]

activities/migrations/0032_fanout_subject_document_alter_fanout_type.py

@@ -0,0 +1,38 @@
+# Generated by Django 5.2.16 on 2026-07-22 01:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("activities", "0031_quoteauthorization"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="fanout",
+            name="subject_document",
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="fanout",
+            name="type",
+            field=models.CharField(
+                choices=[
+                    ("post", "Post"),
+                    ("post_edited", "Post Edited"),
+                    ("post_deleted", "Post Deleted"),
+                    ("interaction", "Interaction"),
+                    ("undo_interaction", "Undo Interaction"),
+                    ("identity_edited", "Identity Edited"),
+                    ("identity_deleted", "Identity Deleted"),
+                    ("identity_created", "Identity Created"),
+                    ("identity_moved", "Identity Moved"),
+                    ("tag_featured", "Tag Featured"),
+                    ("tag_unfeatured", "Tag Unfeatured"),
+                    ("forward", "Forward"),
+                ],
+                max_length=100,
+            ),
+        ),
+    ]

activities/models/__init__.py

@@ -6,4 +6,5 @@
 from .post_attachment import PostAttachment, PostAttachmentStates  # noqa
 from .post_interaction import PostInteraction, PostInteractionStates  # noqa
 from .preview_card import PreviewCard, PreviewCardStates  # noqa
+from .quote_authorization import QuoteAuthorization  # noqa
 from .timeline_event import TimelineEvent  # noqa

activities/models/emoji.py

@@ -287,6 +287,9 @@ def by_ap_tag(cls, domain: Domain, data: dict, create: bool = False):
             if not create:
                 raise KeyError(f"No emoji with ID {data['id']}", data)
 
+        if domain is None:
+            raise ValueError("No domain for emoji", data)
+
         # Name could be a direct property, or in a language'd value
         if "name" in data:
             name = data["name"]

activities/models/fan_out.py

@@ -5,6 +5,7 @@
 from core.ld import canonicalise
 from stator.models import State, StateField, StateGraph, StatorModel
 from users.models import Block, FollowStates, Identity
+from users.models.system_actor import SystemActor
 
 
 class FanOutStates(StateGraph):
@@ -143,31 +144,31 @@ def handle_new(cls, instance: "FanOut"):
             # Handle sending remote posts create
             case (FanOut.Types.post, False):
                 post = instance.subject_post
-                # Sign it and send it
+                # Sign it (HTTP and, for public posts, LD) and send it
                 try:
                     post.author.signed_request(
                         method="post",
                         uri=(
                             instance.identity.shared_inbox_uri
                             or instance.identity.inbox_uri
                         ),
-                        body=canonicalise(post.to_create_ap()),
+                        body=post.to_fan_out_ap(instance.type),
                     )
                 except httpx.RequestError:
                     return
 
             # Handle sending remote posts update
             case (FanOut.Types.post_edited, False):
                 post = instance.subject_post
-                # Sign it and send it
+                # Sign it (HTTP and, for public posts, LD) and send it
                 try:
                     post.author.signed_request(
                         method="post",
                         uri=(
                             instance.identity.shared_inbox_uri
                             or instance.identity.inbox_uri
                         ),
-                        body=canonicalise(post.to_update_ap()),
+                        body=post.to_fan_out_ap(instance.type),
                     )
                 except httpx.RequestError:
                     return
@@ -190,7 +191,7 @@ def handle_new(cls, instance: "FanOut"):
                             instance.identity.shared_inbox_uri
                             or instance.identity.inbox_uri
                         ),
-                        body=canonicalise(post.to_delete_ap()),
+                        body=post.to_fan_out_ap(instance.type),
                     )
                 except ValueError:
                     pass  # ignore 401 when identity deletion is processed by remote earlier
@@ -415,6 +416,31 @@ def handle_new(cls, instance: "FanOut"):
                 except httpx.RequestError:
                     return
 
+            # Forward a third-party LD-signed activity concerning a local
+            # thread (AP 7.1.2). The document is re-sent exactly as
+            # received - its LD signature authenticates the true author -
+            # while the system actor provides the delivery HTTP signature.
+            case (FanOut.Types.forward, False):
+                if not instance.subject_document:
+                    return cls.skipped
+                try:
+                    SystemActor().signed_request(
+                        method="post",
+                        uri=(
+                            instance.identity.shared_inbox_uri
+                            or instance.identity.inbox_uri
+                        ),
+                        body=instance.subject_document,
+                    )
+                except ValueError:
+                    pass  # remote refused the forward; it's best-effort
+                except httpx.RequestError:
+                    return
+
+            # Forwards are only ever created for remote followers
+            case (FanOut.Types.forward, True):
+                return cls.skipped
+
             case _:
                 raise ValueError(
                     f"Cannot fan out with type {instance.type} local={instance.identity.local}"
@@ -440,6 +466,7 @@ class Types(models.TextChoices):
         identity_moved = "identity_moved"
         tag_featured = "tag_featured"
         tag_unfeatured = "tag_unfeatured"
+        forward = "forward"
 
     state = StateField(FanOutStates)
 
@@ -486,5 +513,10 @@ class Types(models.TextChoices):
         related_name="fan_outs",
     )
 
+    # For forwards: the original LD-signed document to re-deliver verbatim.
+    # Deliberately not tied to subject_post so forwarded deletes survive
+    # the deletion of our local copy.
+    subject_document = models.JSONField(blank=True, null=True)
+
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)

activities/models/hashtag.py

@@ -1,13 +1,13 @@
 import re
 import time
-from datetime import date, timedelta
+from datetime import date, datetime, timedelta
 
 import urlman
+from core.models import Config
 from django.conf import settings
 from django.db import connection, models, transaction
+from django.db.models.functions import TruncDate
 from django.utils import timezone
-
-from core.models import Config
 from stator.models import State, StateField, StateGraph, StatorModel
 
 
@@ -25,33 +25,81 @@ def handle_outdated(cls, instance: "Hashtag"):
         """
         from .post import Post
 
-        posts_query = Post.objects.local_public().tagged_with(instance)
-        total = posts_query.count()
-
         today = timezone.now().date()
-        total_today = posts_query.filter(created__date=today).count()
-        total_month = posts_query.filter(
-            created__year=today.year,
-            created__month=today.month,
-        ).count()
-        total_year = posts_query.filter(
-            created__year=today.year,
-        ).count()
-
-        history = []
-        tagged_posts = Post.objects.not_hidden().tagged_with(instance)
-        for i in range(8):
-            # Mastodon doesn't include today in history (let's do it anyway)
-            day = today - timedelta(days=i)
-            data = tagged_posts.filter(published__date=day).aggregate(
+        # Use timezone-aware datetime ranges instead of __date / __year /
+        # __month lookups so the FILTER predicates are plain timestamp
+        # comparisons instead of per-row AT TIME ZONE / EXTRACT calls
+        # (single aggregate was hitting ~800ms on popular hashtags because
+        # each candidate row had to be cast).
+        today_start = timezone.make_aware(datetime(today.year, today.month, today.day))
+        tomorrow_start = today_start + timedelta(days=1)
+        month_start = today_start.replace(day=1)
+        if today.month == 12:
+            next_month_start = timezone.make_aware(datetime(today.year + 1, 1, 1))
+        else:
+            next_month_start = timezone.make_aware(
+                datetime(today.year, today.month + 1, 1)
+            )
+        year_start = timezone.make_aware(datetime(today.year, 1, 1))
+        next_year_start = timezone.make_aware(datetime(today.year + 1, 1, 1))
+        # Collapse total / today / month / year into a single conditional
+        # aggregate so we hit the DB once instead of four times per hashtag
+        # transition (fired thousands of times daily by the stator loop).
+        totals = (
+            Post.objects.local_public()
+            .tagged_with(instance)
+            .aggregate(
+                total=models.Count("id"),
+                total_today=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=today_start, created__lt=tomorrow_start
+                    ),
+                ),
+                total_month=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=month_start, created__lt=next_month_start
+                    ),
+                ),
+                total_year=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=year_start, created__lt=next_year_start
+                    ),
+                ),
+            )
+        )
+        total = totals["total"]
+        total_today = totals["total_today"]
+        total_month = totals["total_month"]
+        total_year = totals["total_year"]
+
+        # Mastodon doesn't include today in history (let's do it anyway).
+        # One GROUP BY per-day aggregate over the 8-day window replaces the
+        # previous 8 separate filter().aggregate() calls.
+        history_start = today - timedelta(days=7)
+        history_rows = (
+            Post.objects.not_hidden()
+            .tagged_with(instance)
+            .annotate(_day=TruncDate("published"))
+            .filter(_day__gte=history_start, _day__lte=today)
+            .values("_day")
+            .annotate(
                 total=models.Count("id"),
                 num_authors=models.Count("author", distinct=True),
             )
+        )
+        by_day = {row["_day"]: row for row in history_rows}
+        history = []
+        for i in range(8):
+            day = today - timedelta(days=i)
+            data = by_day.get(day)
             history.append(
                 {
                     "day": str(int(time.mktime(day.timetuple()))),
-                    "uses": str(data["total"]),
-                    "accounts": str(data["num_authors"]),
+                    "uses": str(data["total"]) if data else "0",
+                    "accounts": str(data["num_authors"]) if data else "0",
                 }
             )