Add migration for the new published meta field

core/db.py

@@ -2,12 +2,14 @@ from enum import Enum
 from enum import unique
 from typing import Any
 from typing import Dict
+from typing import Iterable
 from typing import Optional
 
 from config import DB
 
 _Q = Dict[str, Any]
-_Doc = Optional[Dict[str, Any]]
+_D = Dict[str, Any]
+_Doc = Optional[_D]
 
 
 @unique
@@ -20,6 +22,10 @@ def find_one_activity(q: _Q) -> _Doc:
     return DB[CollectionName.ACTIVITIES.value].find_one(q)
 
 
+def find_activities(q: _Q) -> Iterable[_D]:
+    return DB[CollectionName.ACTIVITIES.value].find(q)
+
+
 def update_one_activity(q: _Q, update: _Q) -> None:
     DB[CollectionName.ACTIVITIES.value].update_one(q, update)
 

core/migrations.py

@@ -7,7 +7,11 @@ from little_boxes import activitypub as ap
 
 from config import ID
 from core import activitypub
-from utils.migrations import DB
+from core.db import DB
+from core.db import find_activities
+from core.db import update_one_activity
+from core.meta import MetaKey
+from core.meta import _meta
 from utils.migrations import Migration
 from utils.migrations import logger
 from utils.migrations import perform  # noqa:  just here for export
@@ -153,3 +157,27 @@ class _2_FollowMigration(Migration):
                     )
             except Exception:
                 logger.exception("failed to process actor {data!r}")
+
+
+class _20190808_MetaPublishedMigration(Migration):
+    """Add the `meta.published` field to old activities."""
+
+    def migrate(self) -> None:
+        for data in find_activities({"meta.published": {"$exists": False}}):
+            try:
+                raw = data["activity"]
+                # If the activity has its own `published` field, we'll use it
+                if "published" in raw:
+                    published = raw["published"]
+                else:
+                    # Otherwise, we take the date we received the activity as the published time
+                    published = ap.format_datetime(data["_id"].generation_time)
+
+                # Set the field in the DB
+                update_one_activity(
+                    {"_id": data["_id"]},
+                    {"$set": {_meta(MetaKey.PUBLISHED): published}},
+                )
+
+            except Exception:
+                logger.exception("failed to process activity {data!r}")

core/remote.py

@@ -13,6 +13,9 @@ def _update(url: str, replace: _Q) -> None:
     update_one_remote({"server": server(url)}, replace, upsert=True)
 
 
+# TODO(tsileo): track receive (and the user agent to help debug issues)
+
+
 def track_successful_send(url: str) -> None:
     now_ = now()
     _update(