microblog.pub/tasks.py

import json
import logging
import os
import random

import requests
from celery import Celery
from requests.exceptions import HTTPError

from little_boxes import activitypub as ap
from config import DB
from config import HEADERS
from config import KEY
from config import USER_AGENT
from little_boxes.httpsig import HTTPSigAuth
from little_boxes.linked_data_sig import generate_signature
from utils.opengraph import fetch_og_metadata
from utils.media import Kind
from config import MEDIA_CACHE

log = logging.getLogger(__name__)
app = Celery(
    "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//")
)
SigAuth = HTTPSigAuth(KEY)


@app.task(bind=True, max_retries=12)
def process_new_activity(self, iri: str) -> None:
    try:
        activity = ap.fetch_remote_activity(iri)
        log.info(f"activity={activity!r}")

        tag_stream = False
        if activity.has_type(ap.ActivityType.ANNOUCE):
            tag_stream = True
        elif activity.has_type(ap.ActivityType.CREATE):
            note = activity.get_object()
            if not note.inReplyTo:
                tag_stream = True

        log.info(f"{iri} tag_stream={tag_stream}")
        DB.update_one({"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}})

        log.info(f"new activity {iri} processed")
    except Exception as err:
        log.exception(f"failed to process new activity {iri}")
        self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))


@app.task(bind=True, max_retries=12)
def cache_attachments(self, iri: str) -> None:
    try:
        activity = ap.fetch_remote_activity(iri)
        log.info(f"activity={activity!r}")
        # Generates thumbnails for the actor's icon and the attachments if any

        actor = activity.get_actor()

        # Update the cached actor
        DB.actors.update_one(
            {"remote_id": iri},
            {"$set": {"remote_id": iri, "data": actor.to_dict(embed=True)}},
            upsert=True,
        )

        if actor.icon:
            MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON)

        if activity.has_type(ap.ActivityType.CREATE):
            for attachment in activity.get_object()._data.get("attachment", []):
                MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT)

        log.info(f"attachmwents cached for {iri}")

    except Exception as err:
        log.exception(f"failed to process new activity {iri}")
        self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))


@app.task(bind=True, max_retries=12)
def post_to_inbox(self, payload: str, to: str) -> None:
    try:
        log.info("payload=%s", payload)
        log.info("generating sig")
        signed_payload = json.loads(payload)
        generate_signature(signed_payload, KEY)
        log.info("to=%s", to)
        resp = requests.post(
            to,
            data=json.dumps(signed_payload),
            auth=SigAuth,
            headers={
                "Content-Type": HEADERS[1],
                "Accept": HEADERS[1],
                "User-Agent": USER_AGENT,
            },
        )
        log.info("resp=%s", resp)
        log.info("resp_body=%s", resp.text)
        resp.raise_for_status()
    except HTTPError as err:
        log.exception("request failed")
        if 400 >= err.response.status_code >= 499:
            log.info("client error, no retry")
            return
        self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))


@app.task(bind=True, max_retries=12)
def fetch_og(self, col, remote_id):
    try:
        log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col)
        if col == "INBOX":
            log.info(
                "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)
            )
        elif col == "OUTBOX":
            log.info(
                "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)
            )
    except Exception as err:
        self.log.exception("failed")
        self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))