Index hashtags and mentions

This commit is contained in:
Thomas Sileo 2019-09-01 20:58:51 +02:00
parent 36bc93cfda
commit 52bc600832
5 changed files with 66 additions and 16 deletions

29
app.py
View file

@ -55,10 +55,13 @@ from core.db import find_one_activity
from core.meta import Box from core.meta import Box
from core.meta import MetaKey from core.meta import MetaKey
from core.meta import _meta from core.meta import _meta
from core.meta import by_hashtag
from core.meta import by_remote_id from core.meta import by_remote_id
from core.meta import by_type from core.meta import by_type
from core.meta import by_visibility
from core.meta import in_outbox from core.meta import in_outbox
from core.meta import is_public from core.meta import is_public
from core.meta import not_deleted
from core.meta import not_undo from core.meta import not_undo
from core.shared import _build_thread from core.shared import _build_thread
from core.shared import _get_ip from core.shared import _get_ip
@ -875,9 +878,10 @@ def following():
def tags(tag): def tags(tag):
if not DB.activities.count( if not DB.activities.count(
{ {
"box": Box.OUTBOX.value, **in_outbox(),
"activity.object.tag.type": "Hashtag", **by_hashtag(tag),
"activity.object.tag.name": "#" + tag, **by_visibility(ap.Visibility.PUBLIC),
**not_deleted(),
} }
): ):
abort(404) abort(404)
@ -888,23 +892,20 @@ def tags(tag):
tag=tag, tag=tag,
outbox_data=DB.activities.find( outbox_data=DB.activities.find(
{ {
"box": Box.OUTBOX.value, **in_outbox(),
"type": ActivityType.CREATE.value, **by_hashtag(tag),
"meta.deleted": False, **by_visibility(ap.Visibility.PUBLIC),
"activity.object.tag.type": "Hashtag", **not_deleted(),
"activity.object.tag.name": "#" + tag,
} }
), ),
) )
) )
_log_sig() _log_sig()
q = { q = {
"box": Box.OUTBOX.value, **in_outbox(),
"meta.deleted": False, **by_hashtag(tag),
"meta.undo": False, **by_visibility(ap.Visibility.PUBLIC),
"type": ActivityType.CREATE.value, **not_deleted(),
"activity.object.tag.type": "Hashtag",
"activity.object.tag.name": "#" + tag,
} }
return activitypubify( return activitypubify(
**activitypub.build_ordered_collection( **activitypub.build_ordered_collection(

View file

@ -128,9 +128,20 @@ def save(box: Box, activity: ap.BaseActivity) -> None:
actor_id = activity.get_actor().id actor_id = activity.get_actor().id
# Set some "type"-related neta # Set some "type"-related neta
extra = {} extra: Dict[str, Any] = {}
if box == Box.OUTBOX and activity.has_type(ap.Follow): if box == Box.OUTBOX and activity.has_type(ap.ActivityType.FOLLOW):
extra[MetaKey.FOLLOW_STATUS.value] = FollowStatus.WAITING.value extra[MetaKey.FOLLOW_STATUS.value] = FollowStatus.WAITING.value
elif activity.has_type(ap.ActivityType.CREATE):
mentions = []
obj = activity.get_object()
for m in obj.get_mentions():
mentions.append(m.href)
hashtags = []
for h in obj.get_hashtags():
hashtags.append(h.name[1:]) # Strip the #
extra.update(
{MetaKey.MENTIONS.value: mentions, MetaKey.HASHTAGS.value: hashtags}
)
DB.activities.insert_one( DB.activities.insert_one(
{ {

View file

@ -26,6 +26,8 @@ def create_indexes():
DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("remote_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.mentions", pymongo.ASCENDING)])
DB.activities.create_index([("meta.hashtags", pymongo.ASCENDING)])
DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)])
DB.activities.create_index( DB.activities.create_index(
[ [

View file

@ -46,6 +46,9 @@ class MetaKey(Enum):
OBJECT_ACTOR_HASH = "object_actor_hash" OBJECT_ACTOR_HASH = "object_actor_hash"
PUBLIC = "public" PUBLIC = "public"
HASHTAGS = "hashtags"
MENTIONS = "mentions"
FOLLOW_STATUS = "follow_status" FOLLOW_STATUS = "follow_status"
THREAD_ROOT_PARENT = "thread_root_parent" THREAD_ROOT_PARENT = "thread_root_parent"
@ -121,6 +124,14 @@ def is_public() -> _SubQuery:
return flag(MetaKey.PUBLIC, True) return flag(MetaKey.PUBLIC, True)
def by_visibility(vis: ap.Visibility) -> _SubQuery:
return flag(MetaKey.VISIBILITY, vis.name)
def by_hashtag(ht: str) -> _SubQuery:
return flag(MetaKey.HASHTAGS, ht)
def inc(mk: MetaKey, val: int) -> _SubQuery: def inc(mk: MetaKey, val: int) -> _SubQuery:
return {"$inc": flag(mk, val)} return {"$inc": flag(mk, val)}

View file

@ -19,6 +19,7 @@ from core.meta import by_remote_id
from core.meta import by_type from core.meta import by_type
from core.meta import in_inbox from core.meta import in_inbox
from core.meta import in_outbox from core.meta import in_outbox
from core.meta import not_deleted
from core.meta import not_undo from core.meta import not_undo
from core.meta import upsert from core.meta import upsert
from utils.migrations import Migration from utils.migrations import Migration
@ -293,3 +294,27 @@ class _20190901_FollowFollowBackMigrationFix(Migration):
except Exception: except Exception:
logger.exception(f"failed to process activity {data!r}") logger.exception(f"failed to process activity {data!r}")
class _20190901_MetaHashtagsAndMentions(Migration):
def migrate(self) -> None:
for data in find_activities(
{**by_type(ap.ActivityType.CREATE), **not_deleted()}
):
try:
activity = ap.parse_activity(data["activity"])
mentions = []
obj = activity.get_object()
for m in obj.get_mentions():
mentions.append(m.href)
hashtags = []
for h in obj.get_hashtags():
hashtags.append(h.name[1:]) # Strip the #
update_one_activity(
by_remote_id(data["remote_id"]),
upsert({MetaKey.MENTIONS: mentions, MetaKey.HASHTAGS: hashtags}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")