1a7a02a221
* check for missing actor in stream item * reply on my own status * fix img exceed div * check for "actor" key in item.meta before render * force UTF-8 text encoding * check if a is a dict or string * check d object must be dict to get url or id * json/atom/rss feed * handle missing root_id * fix newer-link hover * fix older-link position * quick hack to support peertube video * enable IndieAuth endpoint * fix 500 when remote follow
62 lines
1.5 KiB
Python
62 lines
1.5 KiB
Python
import logging
|
|
import opengraph
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from little_boxes import activitypub as ap
|
|
from little_boxes.errors import NotAnActivityError
|
|
from little_boxes.urlutils import check_url
|
|
from little_boxes.urlutils import is_url_valid
|
|
|
|
from .lookup import lookup
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def links_from_note(note):
|
|
tags_href = set()
|
|
for t in note.get("tag", []):
|
|
h = t.get("href")
|
|
if h:
|
|
tags_href.add(h)
|
|
|
|
links = set()
|
|
soup = BeautifulSoup(note["content"])
|
|
for link in soup.find_all("a"):
|
|
h = link.get("href")
|
|
if h.startswith("http") and h not in tags_href and is_url_valid(h):
|
|
links.add(h)
|
|
|
|
return links
|
|
|
|
|
|
def fetch_og_metadata(user_agent, links):
|
|
res = []
|
|
for l in links:
|
|
check_url(l)
|
|
|
|
# Remove any AP actor from the list
|
|
try:
|
|
p = lookup(l)
|
|
if p.has_type(ap.ACTOR_TYPES):
|
|
continue
|
|
except NotAnActivityError:
|
|
pass
|
|
|
|
r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15)
|
|
r.raise_for_status()
|
|
if not r.headers.get("content-type").startswith("text/html"):
|
|
logger.debug(f"skipping {l}")
|
|
continue
|
|
|
|
r.encoding = 'UTF-8'
|
|
html = r.text
|
|
try:
|
|
data = dict(opengraph.OpenGraph(html=html))
|
|
except Exception:
|
|
logger.exception(f"failed to parse {l}")
|
|
continue
|
|
if data.get("url"):
|
|
res.append(data)
|
|
|
|
return res
|