diff --git a/core/activitypub.py b/core/activitypub.py
index beeeffe..2f6f693 100644
--- a/core/activitypub.py
+++ b/core/activitypub.py
@@ -31,6 +31,7 @@ from config import ME
 from config import USER_AGENT
 from core.db import find_one_activity
 from core.db import update_many_activities
+from core.db import update_one_activity
 from core.meta import Box
 from core.meta import MetaKey
 from core.meta import by_object_id
@@ -682,7 +683,7 @@ def handle_question_reply(create: ap.Create, question: ap.Question) -> None:
         by_remote_id(create.id),
         {
             "$set": {
-                "meta.answer_to": question.id,
+                "meta.poll_answer_to": question.id,
                 "meta.poll_answer_choice": choice,
                 "meta.stream": False,
                 "meta.poll_answer": True,
@@ -727,6 +728,11 @@ def handle_replies(create: ap.Create) -> None:
                 }
             },
         )
+        # Mark our reply as a poll answers, to "hide" it from the UI
+        update_one_activity(
+            by_remote_id(create.id),
+            upsert({MetaKey.POLL_ANSWER: True, MetaKey.POLL_ANSWER_TO: reply.id}),
+        )
         return None
 
     # It's a regular reply, try to increment the reply counter
diff --git a/core/meta.py b/core/meta.py
index 6a6ccef..422437c 100644
--- a/core/meta.py
+++ b/core/meta.py
@@ -24,6 +24,7 @@ class MetaKey(Enum):
     NOTIFICATION_UNREAD = "notification_unread"
     NOTIFICATION_FOLLOWS_BACK = "notification_follows_back"
     POLL_ANSWER = "poll_answer"
+    POLL_ANSWER_TO = "poll_answer_to"
     STREAM = "stream"
     ACTOR_ID = "actor_id"
     ACTOR = "actor"