askiiart/microblog.pub
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Switch to Little boxes, fixes #1 (#8)

Browse source
This commit is contained in:
Thomas Sileo 2018-06-19 00:10:19 +02:00 committed by GitHub
parent 070e39bdfe
commit 8d5f4a8e98
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 1529 additions and 2640 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1423
activitypub.py
View file

File diff suppressed because it is too large Load diff

1387
app.py
View file

File diff suppressed because it is too large Load diff

117
config.py
View file

@ -1,15 +1,17 @@
import subprocess
import os import os
import yaml import subprocess
from pymongo import MongoClient
import requests
from itsdangerous import JSONWebSignatureSerializer
from datetime import datetime from datetime import datetime
from utils import strtobool import requests
from utils.key import Key, KEY_DIR, get_secret_key import yaml
from utils.actor_service import ActorService from itsdangerous import JSONWebSignatureSerializer
from utils.object_service import ObjectService from pymongo import MongoClient
from little_boxes import strtobool
from utils.key import KEY_DIR
from utils.key import get_key
from utils.key import get_secret_key
def noop(): def noop():
pass pass
@ -17,103 +19,94 @@ def noop():
CUSTOM_CACHE_HOOKS = False CUSTOM_CACHE_HOOKS = False
try: try:
from cache_hooks import purge as custom_cache_purge_hook from cache_hooks import purge as custom_cache_purge_hook
except ModuleNotFoundError: except ModuleNotFoundError:
custom_cache_purge_hook = noop custom_cache_purge_hook = noop
VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') VERSION = (
subprocess.check_output(["git", "describe", "--always"]).split()[0].decode("utf-8")
)
DEBUG_MODE = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) DEBUG_MODE = strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false"))
CTX_AS = 'https://www.w3.org/ns/activitystreams' CTX_AS = "https://www.w3.org/ns/activitystreams"
CTX_SECURITY = 'https://w3id.org/security/v1' CTX_SECURITY = "https://w3id.org/security/v1"
AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public"
HEADERS = [ HEADERS = [
'application/activity+json', "application/activity+json",
'application/ld+json;profile=https://www.w3.org/ns/activitystreams', "application/ld+json;profile=https://www.w3.org/ns/activitystreams",
'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
'application/ld+json', "application/ld+json",
] ]
with open(os.path.join(KEY_DIR, 'me.yml')) as f: with open(os.path.join(KEY_DIR, "me.yml")) as f:
conf = yaml.load(f) conf = yaml.load(f)
USERNAME = conf['username'] USERNAME = conf["username"]
NAME = conf['name'] NAME = conf["name"]
DOMAIN = conf['domain'] DOMAIN = conf["domain"]
SCHEME = 'https' if conf.get('https', True) else 'http' SCHEME = "https" if conf.get("https", True) else "http"
BASE_URL = SCHEME + '://' + DOMAIN BASE_URL = SCHEME + "://" + DOMAIN
ID = BASE_URL ID = BASE_URL
SUMMARY = conf['summary'] SUMMARY = conf["summary"]
ICON_URL = conf['icon_url'] ICON_URL = conf["icon_url"]
PASS = conf['pass'] PASS = conf["pass"]
PUBLIC_INSTANCES = conf.get('public_instances', []) PUBLIC_INSTANCES = conf.get("public_instances", [])
# TODO(tsileo): choose dark/light style # TODO(tsileo): choose dark/light style
THEME_COLOR = conf.get('theme_color') THEME_COLOR = conf.get("theme_color")
USER_AGENT = ( USER_AGENT = (
f'{requests.utils.default_user_agent()} ' f"{requests.utils.default_user_agent()} " f"(microblog.pub/{VERSION}; +{BASE_URL})"
f'(microblog.pub/{VERSION}; +{BASE_URL})'
) )
# TODO(tsileo): use 'mongo:27017;
# mongo_client = MongoClient(host=['mongo:27017'])
mongo_client = MongoClient( mongo_client = MongoClient(
host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], host=[os.getenv("MICROBLOGPUB_MONGODB_HOST", "localhost:27017")]
) )
DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_"))
DB = mongo_client[DB_NAME] DB = mongo_client[DB_NAME]
def _drop_db(): def _drop_db():
if not DEBUG_MODE: if not DEBUG_MODE:
return return
mongo_client.drop_database(DB_NAME) mongo_client.drop_database(DB_NAME)
KEY = Key(USERNAME, DOMAIN, create=True)
KEY = get_key(ID, USERNAME, DOMAIN)
JWT_SECRET = get_secret_key('jwt') JWT_SECRET = get_secret_key("jwt")
JWT = JSONWebSignatureSerializer(JWT_SECRET) JWT = JSONWebSignatureSerializer(JWT_SECRET)
def _admin_jwt_token() -> str: def _admin_jwt_token() -> str:
return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore return JWT.dumps( # type: ignore
{"me": "ADMIN", "ts": datetime.now().timestamp()}
).decode( # type: ignore
"utf-8"
)
ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token)
ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token)
ME = { ME = {
"@context": [ "@context": [CTX_AS, CTX_SECURITY],
CTX_AS,
CTX_SECURITY,
],
"type": "Person", "type": "Person",
"id": ID, "id": ID,
"following": ID+"/following", "following": ID + "/following",
"followers": ID+"/followers", "followers": ID + "/followers",
"liked": ID+"/liked", "liked": ID + "/liked",
"inbox": ID+"/inbox", "inbox": ID + "/inbox",
"outbox": ID+"/outbox", "outbox": ID + "/outbox",
"preferredUsername": USERNAME, "preferredUsername": USERNAME,
"name": NAME, "name": NAME,
"summary": SUMMARY, "summary": SUMMARY,
"endpoints": {}, "endpoints": {},
"url": ID, "url": ID,
"icon": { "icon": {"mediaType": "image/png", "type": "Image", "url": ICON_URL},
"mediaType": "image/png", "publicKey": KEY.to_dict(),
"type": "Image",
"url": ICON_URL,
},
"publicKey": {
"id": ID+"#main-key",
"owner": ID,
"publicKeyPem": KEY.pubkey_pem,
},
} }
print(ME)
ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances)
OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances)

2
dev-requirements.txt
View file

@ -1,6 +1,8 @@
git+https://github.com/tsileo/little-boxes.git
pytest pytest
requests requests
html2text html2text
pyyaml pyyaml
flake8 flake8
mypy mypy
black

6
requirements.txt
View file

@ -2,21 +2,19 @@ libsass
gunicorn gunicorn
piexif piexif
requests requests
markdown
python-u2flib-server python-u2flib-server
Flask Flask
Flask-WTF Flask-WTF
Celery Celery
pymongo pymongo
pyld
timeago timeago
bleach bleach
pycryptodome
html2text html2text
feedgen feedgen
itsdangerous itsdangerous
bcrypt bcrypt
mf2py mf2py
passlib passlib
pyyaml
git+https://github.com/erikriver/opengraph.git git+https://github.com/erikriver/opengraph.git
git+https://github.com/tsileo/little-boxes.git
pyyaml

63
tasks.py
View file

@ -1,47 +1,52 @@
import os
import json import json
import logging import logging
import os
import random import random
import requests import requests
from celery import Celery from celery import Celery
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from config import HEADERS
from config import ID
from config import DB from config import DB
from config import HEADERS
from config import KEY from config import KEY
from config import USER_AGENT from config import USER_AGENT
from utils.httpsig import HTTPSigAuth from little_boxes.httpsig import HTTPSigAuth
from little_boxes.linked_data_sig import generate_signature
from utils.opengraph import fetch_og_metadata from utils.opengraph import fetch_og_metadata
from utils.linked_data_sig import generate_signature
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) app = Celery(
SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//")
)
SigAuth = HTTPSigAuth(KEY)
@app.task(bind=True, max_retries=12) @app.task(bind=True, max_retries=12)
def post_to_inbox(self, payload: str, to: str) -> None: def post_to_inbox(self, payload: str, to: str) -> None:
try: try:
log.info('payload=%s', payload) log.info("payload=%s", payload)
log.info('generating sig') log.info("generating sig")
signed_payload = json.loads(payload) signed_payload = json.loads(payload)
generate_signature(signed_payload, KEY.privkey) generate_signature(signed_payload, KEY)
log.info('to=%s', to) log.info("to=%s", to)
resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ resp = requests.post(
'Content-Type': HEADERS[1], to,
'Accept': HEADERS[1], data=json.dumps(signed_payload),
'User-Agent': USER_AGENT, auth=SigAuth,
}) headers={
log.info('resp=%s', resp) "Content-Type": HEADERS[1],
log.info('resp_body=%s', resp.text) "Accept": HEADERS[1],
"User-Agent": USER_AGENT,
},
)
log.info("resp=%s", resp)
log.info("resp_body=%s", resp.text)
resp.raise_for_status() resp.raise_for_status()
except HTTPError as err: except HTTPError as err:
log.exception('request failed') log.exception("request failed")
if 400 >= err.response.status_code >= 499: if 400 >= err.response.status_code >= 499:
log.info('client error, no retry') log.info("client error, no retry")
return return
self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))
@ -49,11 +54,15 @@ def post_to_inbox(self, payload: str, to: str) -> None:
@app.task(bind=True, max_retries=12) @app.task(bind=True, max_retries=12)
def fetch_og(self, col, remote_id): def fetch_og(self, col, remote_id):
try: try:
log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col)
if col == 'INBOX': if col == "INBOX":
log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) log.info(
elif col == 'OUTBOX': "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)
log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) )
elif col == "OUTBOX":
log.info(
"%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)
)
except Exception as err: except Exception as err:
self.log.exception('failed') self.log.exception("failed")
self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))

401
tests/federation_test.py
View file

@ -1,12 +1,12 @@
import time
import os import os
import time
from typing import List
from typing import Tuple
import requests import requests
from html2text import html2text from html2text import html2text
from utils import activitypub_utils
from typing import Tuple from little_boxes.collection import parse_collection
from typing import List
def resp2plaintext(resp): def resp2plaintext(resp):
@ -22,33 +22,38 @@ class Instance(object):
self.docker_url = docker_url or host_url self.docker_url = docker_url or host_url
self._create_delay = 10 self._create_delay = 10
with open( with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key') os.path.join(
os.path.dirname(os.path.abspath(__file__)),
f"fixtures/{name}/config/admin_api_key.key",
)
) as f: ) as f:
api_key = f.read() api_key = f.read()
self._auth_headers = {'Authorization': f'Bearer {api_key}'} self._auth_headers = {"Authorization": f"Bearer {api_key}"}
def _do_req(self, url, headers): def _do_req(self, url):
"""Used to parse collection.""" """Used to parse collection."""
url = url.replace(self.docker_url, self.host_url) url = url.replace(self.docker_url, self.host_url)
resp = requests.get(url, headers=headers) resp = requests.get(url, headers={'Accept': 'application/activity+json'})
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def _parse_collection(self, payload=None, url=None): def _parse_collection(self, payload=None, url=None):
"""Parses a collection (go through all the pages).""" """Parses a collection (go through all the pages)."""
return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) return parse_collection(
url=url, payload=payload, fetcher=self._do_req,
)
def ping(self): def ping(self):
"""Ensures the homepage is reachable.""" """Ensures the homepage is reachable."""
resp = requests.get(f'{self.host_url}/') resp = requests.get(f"{self.host_url}/")
resp.raise_for_status() resp.raise_for_status()
assert resp.status_code == 200 assert resp.status_code == 200
def debug(self): def debug(self):
"""Returns the debug infos (number of items in the inbox/outbox.""" """Returns the debug infos (number of items in the inbox/outbox."""
resp = requests.get( resp = requests.get(
f'{self.host_url}/api/debug', f"{self.host_url}/api/debug",
headers={**self._auth_headers, 'Accept': 'application/json'}, headers={**self._auth_headers, "Accept": "application/json"},
) )
resp.raise_for_status() resp.raise_for_status()
@ -57,8 +62,8 @@ class Instance(object):
def drop_db(self): def drop_db(self):
"""Drops the MongoDB DB.""" """Drops the MongoDB DB."""
resp = requests.delete( resp = requests.delete(
f'{self.host_url}/api/debug', f"{self.host_url}/api/debug",
headers={**self._auth_headers, 'Accept': 'application/json'}, headers={**self._auth_headers, "Accept": "application/json"},
) )
resp.raise_for_status() resp.raise_for_status()
@ -68,100 +73,92 @@ class Instance(object):
"""Blocks an actor.""" """Blocks an actor."""
# Instance1 follows instance2 # Instance1 follows instance2
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/block', f"{self.host_url}/api/block",
params={'actor': actor_url}, params={"actor": actor_url},
headers=self._auth_headers, headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay/2) time.sleep(self._create_delay / 2)
return resp.json().get('activity') return resp.json().get("activity")
def follow(self, instance: 'Instance') -> str: def follow(self, instance: "Instance") -> str:
"""Follows another instance.""" """Follows another instance."""
# Instance1 follows instance2 # Instance1 follows instance2
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/follow', f"{self.host_url}/api/follow",
json={'actor': instance.docker_url}, json={"actor": instance.docker_url},
headers=self._auth_headers, headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def new_note(self, content, reply=None) -> str: def new_note(self, content, reply=None) -> str:
"""Creates a new note.""" """Creates a new note."""
params = {'content': content} params = {"content": content}
if reply: if reply:
params['reply'] = reply params["reply"] = reply
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/new_note', f"{self.host_url}/api/new_note", json=params, headers=self._auth_headers
json=params,
headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def boost(self, oid: str) -> str: def boost(self, oid: str) -> str:
"""Creates an Announce activity.""" """Creates an Announce activity."""
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/boost', f"{self.host_url}/api/boost", json={"id": oid}, headers=self._auth_headers
json={'id': oid},
headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def like(self, oid: str) -> str: def like(self, oid: str) -> str:
"""Creates a Like activity.""" """Creates a Like activity."""
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/like', f"{self.host_url}/api/like", json={"id": oid}, headers=self._auth_headers
json={'id': oid},
headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def delete(self, oid: str) -> str: def delete(self, oid: str) -> str:
"""Creates a Delete activity.""" """Creates a Delete activity."""
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/note/delete', f"{self.host_url}/api/note/delete",
json={'id': oid}, json={"id": oid},
headers=self._auth_headers, headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def undo(self, oid: str) -> str: def undo(self, oid: str) -> str:
"""Creates a Undo activity.""" """Creates a Undo activity."""
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/undo', f"{self.host_url}/api/undo", json={"id": oid}, headers=self._auth_headers
json={'id': oid},
headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get("activity")
def followers(self) -> List[str]: def followers(self) -> List[str]:
"""Parses the followers collection.""" """Parses the followers collection."""
resp = requests.get( resp = requests.get(
f'{self.host_url}/followers', f"{self.host_url}/followers",
headers={'Accept': 'application/activity+json'}, headers={"Accept": "application/activity+json"},
) )
resp.raise_for_status() resp.raise_for_status()
@ -172,8 +169,8 @@ class Instance(object):
def following(self): def following(self):
"""Parses the following collection.""" """Parses the following collection."""
resp = requests.get( resp = requests.get(
f'{self.host_url}/following', f"{self.host_url}/following",
headers={'Accept': 'application/activity+json'}, headers={"Accept": "application/activity+json"},
) )
resp.raise_for_status() resp.raise_for_status()
@ -184,8 +181,8 @@ class Instance(object):
def outbox(self): def outbox(self):
"""Returns the instance outbox.""" """Returns the instance outbox."""
resp = requests.get( resp = requests.get(
f'{self.host_url}/following', f"{self.host_url}/following",
headers={'Accept': 'application/activity+json'}, headers={"Accept": "application/activity+json"},
) )
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
@ -194,7 +191,7 @@ class Instance(object):
"""Fetches a specific item from the instance outbox.""" """Fetches a specific item from the instance outbox."""
resp = requests.get( resp = requests.get(
aid.replace(self.docker_url, self.host_url), aid.replace(self.docker_url, self.host_url),
headers={'Accept': 'application/activity+json'}, headers={"Accept": "application/activity+json"},
) )
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
@ -202,8 +199,8 @@ class Instance(object):
def stream_jsonfeed(self): def stream_jsonfeed(self):
"""Returns the "stream"'s JSON feed.""" """Returns the "stream"'s JSON feed."""
resp = requests.get( resp = requests.get(
f'{self.host_url}/api/stream', f"{self.host_url}/api/stream",
headers={**self._auth_headers, 'Accept': 'application/json'}, headers={**self._auth_headers, "Accept": "application/json"},
) )
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
@ -211,10 +208,14 @@ class Instance(object):
def _instances() -> Tuple[Instance, Instance]: def _instances() -> Tuple[Instance, Instance]:
"""Initializes the client for the two test instances.""" """Initializes the client for the two test instances."""
instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') instance1 = Instance(
"instance1", "http://localhost:5006", "http://instance1_web_1:5005"
)
instance1.ping() instance1.ping()
instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') instance2 = Instance(
"instance2", "http://localhost:5007", "http://instance2_web_1:5005"
)
instance2.ping() instance2.ping()
# Return the DB # Return the DB
@ -230,12 +231,12 @@ def test_follow() -> None:
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 1 # An Accept activity should be there assert instance1_debug["inbox"] == 1 # An Accept activity should be there
assert instance1_debug['outbox'] == 1 # We've sent a Follow activity assert instance1_debug["outbox"] == 1 # We've sent a Follow activity
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 1 # An Follow activity should be there assert instance2_debug["inbox"] == 1 # An Follow activity should be there
assert instance2_debug['outbox'] == 1 # We've sent a Accept activity assert instance2_debug["outbox"] == 1 # We've sent a Accept activity
assert instance2.followers() == [instance1.docker_url] assert instance2.followers() == [instance1.docker_url]
assert instance1.following() == [instance2.docker_url] assert instance1.following() == [instance2.docker_url]
@ -247,12 +248,12 @@ def test_follow_unfollow():
# Instance1 follows instance2 # Instance1 follows instance2
follow_id = instance1.follow(instance2) follow_id = instance1.follow(instance2)
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 1 # An Accept activity should be there assert instance1_debug["inbox"] == 1 # An Accept activity should be there
assert instance1_debug['outbox'] == 1 # We've sent a Follow activity assert instance1_debug["outbox"] == 1 # We've sent a Follow activity
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 1 # An Follow activity should be there assert instance2_debug["inbox"] == 1 # An Follow activity should be there
assert instance2_debug['outbox'] == 1 # We've sent a Accept activity assert instance2_debug["outbox"] == 1 # We've sent a Accept activity
assert instance2.followers() == [instance1.docker_url] assert instance2.followers() == [instance1.docker_url]
assert instance1.following() == [instance2.docker_url] assert instance1.following() == [instance2.docker_url]
@ -263,12 +264,12 @@ def test_follow_unfollow():
assert instance1.following() == [] assert instance1.following() == []
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 1 # An Accept activity should be there assert instance1_debug["inbox"] == 1 # An Accept activity should be there
assert instance1_debug['outbox'] == 2 # We've sent a Follow and a Undo activity assert instance1_debug["outbox"] == 2 # We've sent a Follow and a Undo activity
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there assert instance2_debug["inbox"] == 2 # An Follow and Undo activity should be there
assert instance2_debug['outbox'] == 1 # We've sent a Accept activity assert instance2_debug["outbox"] == 1 # We've sent a Accept activity
def test_post_content(): def test_post_content():
@ -279,17 +280,19 @@ def test_post_content():
instance2.follow(instance1) instance2.follow(instance1)
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
def test_block_and_post_content(): def test_block_and_post_content():
@ -300,18 +303,22 @@ def test_block_and_post_content():
instance2.follow(instance1) instance2.follow(instance1)
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
instance2.block(instance1.docker_url) instance2.block(instance1.docker_url)
instance1.new_note('hello') instance1.new_note("hello")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 2 # An Follow, Accept activity should be there, Create should have been dropped assert (
assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow activity + the Block activity instance2_debug["inbox"] == 2
) # An Follow, Accept activity should be there, Create should have been dropped
assert (
instance2_debug["outbox"] == 3
) # We've sent a Accept and a Follow activity + the Block activity
# Ensure the post is not visible in instance2's stream # Ensure the post is not visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
def test_post_content_and_delete(): def test_post_content_and_delete():
@ -322,26 +329,30 @@ def test_post_content_and_delete():
instance2.follow(instance1) instance2.follow(instance1)
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
instance1.delete(f'{create_id}/activity') instance1.delete(f"{create_id}/activity")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there assert (
assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity instance2_debug["inbox"] == 4
) # An Follow, Accept and Create and Delete activity should be there
assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity
# Ensure the post has been delete from instance2's stream # Ensure the post has been delete from instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
def test_post_content_and_like(): def test_post_content_and_like():
@ -351,26 +362,26 @@ def test_post_content_and_like():
instance1.follow(instance2) instance1.follow(instance2)
instance2.follow(instance1) instance2.follow(instance1)
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
# Now, instance2 like the note # Now, instance2 like the note
like_id = instance2.like(f'{create_id}/activity') like_id = instance2.like(f"{create_id}/activity")
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # Follow, Accept and Like assert instance1_debug["inbox"] == 3 # Follow, Accept and Like
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'likes' in note assert "likes" in note
assert note['likes']['totalItems'] == 1 assert note["likes"]["totalItems"] == 1
likes = instance1._parse_collection(url=note['likes']['first']) likes = instance1._parse_collection(url=note["likes"]["first"])
assert len(likes) == 1 assert len(likes) == 1
assert likes[0]['id'] == like_id assert likes[0]["id"] == like_id
def test_post_content_and_like_unlike() -> None: def test_post_content_and_like_unlike() -> None:
@ -380,36 +391,36 @@ def test_post_content_and_like_unlike() -> None:
instance1.follow(instance2) instance1.follow(instance2)
instance2.follow(instance1) instance2.follow(instance1)
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
# Now, instance2 like the note # Now, instance2 like the note
like_id = instance2.like(f'{create_id}/activity') like_id = instance2.like(f"{create_id}/activity")
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # Follow, Accept and Like assert instance1_debug["inbox"] == 3 # Follow, Accept and Like
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'likes' in note assert "likes" in note
assert note['likes']['totalItems'] == 1 assert note["likes"]["totalItems"] == 1
likes = instance1._parse_collection(url=note['likes']['first']) likes = instance1._parse_collection(url=note["likes"]["first"])
assert len(likes) == 1 assert len(likes) == 1
assert likes[0]['id'] == like_id assert likes[0]["id"] == like_id
instance2.undo(like_id) instance2.undo(like_id)
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 4 # Follow, Accept and Like and Undo assert instance1_debug["inbox"] == 4 # Follow, Accept and Like and Undo
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'likes' in note assert "likes" in note
assert note['likes']['totalItems'] == 0 assert note["likes"]["totalItems"] == 0
def test_post_content_and_boost() -> None: def test_post_content_and_boost() -> None:
@ -419,26 +430,26 @@ def test_post_content_and_boost() -> None:
instance1.follow(instance2) instance1.follow(instance2)
instance2.follow(instance1) instance2.follow(instance1)
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
# Now, instance2 like the note # Now, instance2 like the note
boost_id = instance2.boost(f'{create_id}/activity') boost_id = instance2.boost(f"{create_id}/activity")
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'shares' in note assert "shares" in note
assert note['shares']['totalItems'] == 1 assert note["shares"]["totalItems"] == 1
shares = instance1._parse_collection(url=note['shares']['first']) shares = instance1._parse_collection(url=note["shares"]["first"])
assert len(shares) == 1 assert len(shares) == 1
assert shares[0]['id'] == boost_id assert shares[0]["id"] == boost_id
def test_post_content_and_boost_unboost() -> None: def test_post_content_and_boost_unboost() -> None:
@ -448,36 +459,36 @@ def test_post_content_and_boost_unboost() -> None:
instance1.follow(instance2) instance1.follow(instance2)
instance2.follow(instance1) instance2.follow(instance1)
create_id = instance1.new_note('hello') create_id = instance1.new_note("hello")
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 1 assert len(inbox_stream["items"]) == 1
assert inbox_stream['items'][0]['id'] == create_id assert inbox_stream["items"][0]["id"] == create_id
# Now, instance2 like the note # Now, instance2 like the note
boost_id = instance2.boost(f'{create_id}/activity') boost_id = instance2.boost(f"{create_id}/activity")
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'shares' in note assert "shares" in note
assert note['shares']['totalItems'] == 1 assert note["shares"]["totalItems"] == 1
shares = instance1._parse_collection(url=note['shares']['first']) shares = instance1._parse_collection(url=note["shares"]["first"])
assert len(shares) == 1 assert len(shares) == 1
assert shares[0]['id'] == boost_id assert shares[0]["id"] == boost_id
instance2.undo(boost_id) instance2.undo(boost_id)
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 4 # Follow, Accept and Announce and Undo assert instance1_debug["inbox"] == 4 # Follow, Accept and Announce and Undo
assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f"{create_id}/activity")
assert 'shares' in note assert "shares" in note
assert note['shares']['totalItems'] == 0 assert note["shares"]["totalItems"] == 0
def test_post_content_and_post_reply() -> None: def test_post_content_and_post_reply() -> None:
@ -488,40 +499,50 @@ def test_post_content_and_post_reply() -> None:
instance2.follow(instance1) instance2.follow(instance1)
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
instance1_create_id = instance1.new_note('hello') instance1_create_id = instance1.new_note("hello")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
instance2_inbox_stream = instance2.stream_jsonfeed() instance2_inbox_stream = instance2.stream_jsonfeed()
assert len(instance2_inbox_stream['items']) == 1 assert len(instance2_inbox_stream["items"]) == 1
assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id
instance2_create_id = instance2.new_note( instance2_create_id = instance2.new_note(
f'hey @instance1@{instance1.docker_url}', f"hey @instance1@{instance1.docker_url}",
reply=f'{instance1_create_id}/activity', reply=f"{instance1_create_id}/activity",
) )
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert (
instance2_debug["outbox"] == 3
) # We've sent a Accept and a Follow and a Create activity
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity instance1_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert (
instance1_debug["outbox"] == 3
) # We've sent a Accept and a Follow and a Create activity
instance1_inbox_stream = instance1.stream_jsonfeed() instance1_inbox_stream = instance1.stream_jsonfeed()
assert len(instance1_inbox_stream['items']) == 1 assert len(instance1_inbox_stream["items"]) == 1
assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id
instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity")
assert 'replies' in instance1_note assert "replies" in instance1_note
assert instance1_note['replies']['totalItems'] == 1 assert instance1_note["replies"]["totalItems"] == 1
replies = instance1._parse_collection(url=instance1_note['replies']['first']) replies = instance1._parse_collection(url=instance1_note["replies"]["first"])
assert len(replies) == 1 assert len(replies) == 1
assert replies[0]['id'] == f'{instance2_create_id}/activity' assert replies[0]["id"] == f"{instance2_create_id}/activity"
def test_post_content_and_post_reply_and_delete() -> None: def test_post_content_and_post_reply_and_delete() -> None:
@ -532,44 +553,58 @@ def test_post_content_and_post_reply_and_delete() -> None:
instance2.follow(instance1) instance2.follow(instance1)
inbox_stream = instance2.stream_jsonfeed() inbox_stream = instance2.stream_jsonfeed()
assert len(inbox_stream['items']) == 0 assert len(inbox_stream["items"]) == 0
instance1_create_id = instance1.new_note('hello') instance1_create_id = instance1.new_note("hello")
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity
# Ensure the post is visible in instance2's stream # Ensure the post is visible in instance2's stream
instance2_inbox_stream = instance2.stream_jsonfeed() instance2_inbox_stream = instance2.stream_jsonfeed()
assert len(instance2_inbox_stream['items']) == 1 assert len(instance2_inbox_stream["items"]) == 1
assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id
instance2_create_id = instance2.new_note( instance2_create_id = instance2.new_note(
f'hey @instance1@{instance1.docker_url}', f"hey @instance1@{instance1.docker_url}",
reply=f'{instance1_create_id}/activity', reply=f"{instance1_create_id}/activity",
) )
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity instance2_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert (
instance2_debug["outbox"] == 3
) # We've sent a Accept and a Follow and a Create activity
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert (
assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity instance1_debug["inbox"] == 3
) # An Follow, Accept and Create activity should be there
assert (
instance1_debug["outbox"] == 3
) # We've sent a Accept and a Follow and a Create activity
instance1_inbox_stream = instance1.stream_jsonfeed() instance1_inbox_stream = instance1.stream_jsonfeed()
assert len(instance1_inbox_stream['items']) == 1 assert len(instance1_inbox_stream["items"]) == 1
assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id
instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity")
assert 'replies' in instance1_note assert "replies" in instance1_note
assert instance1_note['replies']['totalItems'] == 1 assert instance1_note["replies"]["totalItems"] == 1
instance2.delete(f'{instance2_create_id}/activity') instance2.delete(f"{instance2_create_id}/activity")
instance1_debug = instance1.debug() instance1_debug = instance1.debug()
assert instance1_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there assert (
assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity instance1_debug["inbox"] == 4
) # An Follow, Accept and Create and Delete activity should be there
assert (
instance1_debug["outbox"] == 3
) # We've sent a Accept and a Follow and a Create activity
instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity")
assert 'replies' in instance1_note assert "replies" in instance1_note
assert instance1_note['replies']['totalItems'] == 0 assert instance1_note["replies"]["totalItems"] == 0

9
tests/integration_test.py
View file

@ -9,7 +9,10 @@ from html2text import html2text
def config(): def config():
"""Return the current config as a dict.""" """Return the current config as a dict."""
import yaml import yaml
with open(os.path.join(os.path.dirname(__file__), '..', 'config/me.yml'), 'rb') as f:
with open(
os.path.join(os.path.dirname(__file__), "..", "config/me.yml"), "rb"
) as f:
yield yaml.load(f) yield yaml.load(f)
@ -20,9 +23,9 @@ def resp2plaintext(resp):
def test_ping_homepage(config): def test_ping_homepage(config):
"""Ensure the homepage is accessible.""" """Ensure the homepage is accessible."""
resp = requests.get('http://localhost:5005') resp = requests.get("http://localhost:5005")
resp.raise_for_status() resp.raise_for_status()
assert resp.status_code == 200 assert resp.status_code == 200
body = resp2plaintext(resp) body = resp2plaintext(resp)
assert config['name'] in body assert config["name"] in body
assert f"@{config['username']}@{config['domain']}" in body assert f"@{config['username']}@{config['domain']}" in body

6
utils/__init__.py
View file

@ -4,9 +4,9 @@ logger = logging.getLogger(__name__)
def strtobool(s: str) -> bool: def strtobool(s: str) -> bool:
if s in ['y', 'yes', 'true', 'on', '1']: if s in ["y", "yes", "true", "on", "1"]:
return True return True
if s in ['n', 'no', 'false', 'off', '0']: if s in ["n", "no", "false", "off", "0"]:
return False return False
raise ValueError(f'cannot convert {s} to bool') raise ValueError(f"cannot convert {s} to bool")

65
utils/activitypub_utils.py
View file

@ -1,65 +0,0 @@
from typing import Optional, Dict, List, Any
import requests
from .errors import RecursionLimitExceededError
from .errors import UnexpectedActivityTypeError
def _do_req(url: str, headers: Dict[str, str]) -> Dict[str, Any]:
resp = requests.get(url, headers=headers)
resp.raise_for_status()
return resp.json()
def parse_collection(
payload: Optional[Dict[str, Any]] = None,
url: Optional[str] = None,
user_agent: Optional[str] = None,
level: int = 0,
do_req: Any = _do_req,
) -> List[str]:
"""Resolve/fetch a `Collection`/`OrderedCollection`."""
if level > 3:
raise RecursionLimitExceededError('recursion limit exceeded')
# Go through all the pages
headers = {'Accept': 'application/activity+json'}
if user_agent:
headers['User-Agent'] = user_agent
out: List[str] = []
if url:
payload = do_req(url, headers)
if not payload:
raise ValueError('must at least prove a payload or an URL')
if payload['type'] in ['Collection', 'OrderedCollection']:
if 'orderedItems' in payload:
return payload['orderedItems']
if 'items' in payload:
return payload['items']
if 'first' in payload:
if 'orderedItems' in payload['first']:
out.extend(payload['first']['orderedItems'])
if 'items' in payload['first']:
out.extend(payload['first']['items'])
n = payload['first'].get('next')
if n:
out.extend(parse_collection(url=n, user_agent=user_agent, level=level+1, do_req=do_req))
return out
while payload:
if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']:
if 'orderedItems' in payload:
out.extend(payload['orderedItems'])
if 'items' in payload:
out.extend(payload['items'])
n = payload.get('next')
if n is None:
break
payload = do_req(n, headers)
else:
raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type']))
return out

81
utils/actor_service.py
View file

@ -1,81 +0,0 @@
import logging
import requests
from urllib.parse import urlparse
from Crypto.PublicKey import RSA
from .urlutils import check_url
from .errors import ActivityNotFoundError
logger = logging.getLogger(__name__)
class NotAnActorError(Exception):
def __init__(self, activity):
self.activity = activity
class ActorService(object):
def __init__(self, user_agent, col, actor_id, actor_data, instances):
logger.debug(f'Initializing ActorService user_agent={user_agent}')
self._user_agent = user_agent
self._col = col
self._in_mem = {actor_id: actor_data}
self._instances = instances
self._known_instances = set()
def _fetch(self, actor_url):
logger.debug(f'fetching remote object {actor_url}')
check_url(actor_url)
resp = requests.get(actor_url, headers={
'Accept': 'application/activity+json',
'User-Agent': self._user_agent,
})
if resp.status_code == 404:
raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error')
resp.raise_for_status()
return resp.json()
def get(self, actor_url, reload_cache=False):
logger.info(f'get actor {actor_url} (reload_cache={reload_cache})')
if actor_url in self._in_mem:
return self._in_mem[actor_url]
instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl()
if instance not in self._known_instances:
self._known_instances.add(instance)
if not self._instances.find_one({'instance': instance}):
self._instances.insert({'instance': instance, 'first_object': actor_url})
if reload_cache:
actor = self._fetch(actor_url)
self._in_mem[actor_url] = actor
self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True)
return actor
cached_actor = self._col.find_one({'actor_id': actor_url})
if cached_actor:
return cached_actor['cached_response']
actor = self._fetch(actor_url)
if not 'type' in actor:
raise NotAnActorError(None)
if actor['type'] != 'Person':
raise NotAnActorError(actor)
self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True)
self._in_mem[actor_url] = actor
return actor
def get_public_key(self, actor_url, reload_cache=False):
profile = self.get(actor_url, reload_cache=reload_cache)
pub = profile['publicKey']
return pub['id'], RSA.importKey(pub['publicKeyPem'])
def get_inbox_url(self, actor_url, reload_cache=False):
profile = self.get(actor_url, reload_cache=reload_cache)
return profile.get('inbox')

58
utils/content_helper.py
View file

@ -1,58 +0,0 @@
import typing
import re
from bleach.linkifier import Linker
from markdown import markdown
from utils.webfinger import get_actor_url
from config import USERNAME, BASE_URL, ID
from config import ACTOR_SERVICE
from typing import List, Optional, Tuple, Dict, Any, Union, Type
def set_attrs(attrs, new=False):
attrs[(None, u'target')] = u'_blank'
attrs[(None, u'class')] = u'external'
attrs[(None, u'rel')] = u'noopener'
attrs[(None, u'title')] = attrs[(None, u'href')]
return attrs
LINKER = Linker(callbacks=[set_attrs])
HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)")
MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+")
def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]:
tags = []
for hashtag in re.findall(HASHTAG_REGEX, content):
tag = hashtag[1:]
link = f'<a href="{BASE_URL}/tags/{tag}" class="mention hashtag" rel="tag">#<span>{tag}</span></a>'
tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag'))
content = content.replace(hashtag, link)
return content, tags
def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]:
tags = []
for mention in re.findall(MENTION_REGEX, content):
_, username, domain = mention.split('@')
actor_url = get_actor_url(mention)
p = ACTOR_SERVICE.get(actor_url)
print(p)
tags.append(dict(type='Mention', href=p['id'], name=mention))
link = f'<span class="h-card"><a href="{p["url"]}" class="u-url mention">@<span>{username}</span></a></span>'
content = content.replace(mention, link)
return content, tags
def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]:
tags = []
content = LINKER.linkify(content)
content, hashtag_tags = hashtagify(content)
tags.extend(hashtag_tags)
content, mention_tags = mentionify(content)
tags.extend(mention_tags)
content = markdown(content)
return content, tags

37
utils/errors.py
View file

@ -1,37 +0,0 @@
class Error(Exception):
status_code = 400
def __init__(self, message, status_code=None, payload=None):
Exception.__init__(self)
self.message = message
if status_code is not None:
self.status_code = status_code
self.payload = payload
def to_dict(self):
rv = dict(self.payload or ())
rv['message'] = self.message
return rv
def __repr__(self):
return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})'
class NotFromOutboxError(Error):
pass
class ActivityNotFoundError(Error):
status_code = 404
class BadActivityError(Error):
pass
class RecursionLimitExceededError(BadActivityError):
pass
class UnexpectedActivityTypeError(BadActivityError):
pass

94
utils/httpsig.py
View file

@ -1,94 +0,0 @@
"""Implements HTTP signature for Flask requests.
Mastodon instances won't accept requests that are not signed using this scheme.
"""
from datetime import datetime
from urllib.parse import urlparse
from typing import Any, Dict, Optional
import base64
import hashlib
import logging
from flask import request
from requests.auth import AuthBase
from Crypto.Signature import PKCS1_v1_5
from Crypto.Hash import SHA256
logger = logging.getLogger(__name__)
def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str:
out = []
for signed_header in signed_headers.split(' '):
if signed_header == '(request-target)':
out.append('(request-target): '+method.lower()+' '+path)
elif signed_header == 'digest':
out.append('digest: '+body_digest)
else:
out.append(signed_header+': '+headers[signed_header])
return '\n'.join(out)
def _parse_sig_header(val: Optional[str]) -> Optional[Dict[str, str]]:
if not val:
return None
out = {}
for data in val.split(','):
k, v = data.split('=', 1)
out[k] = v[1:len(v)-1]
return out
def _verify_h(signed_string, signature, pubkey):
signer = PKCS1_v1_5.new(pubkey)
digest = SHA256.new()
digest.update(signed_string.encode('utf-8'))
return signer.verify(digest, signature)
def _body_digest() -> str:
h = hashlib.new('sha256')
h.update(request.data)
return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8')
def verify_request(actor_service) -> bool:
hsig = _parse_sig_header(request.headers.get('Signature'))
if not hsig:
logger.debug('no signature in header')
return False
logger.debug(f'hsig={hsig}')
signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest())
_, rk = actor_service.get_public_key(hsig['keyId'])
return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk)
class HTTPSigAuth(AuthBase):
def __init__(self, keyid, privkey):
self.keyid = keyid
self.privkey = privkey
def __call__(self, r):
logger.info(f'keyid={self.keyid}')
host = urlparse(r.url).netloc
bh = hashlib.new('sha256')
bh.update(r.body.encode('utf-8'))
bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8')
date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
r.headers.update({'Digest': bodydigest, 'Date': date})
r.headers.update({'Host': host})
sigheaders = '(request-target) user-agent host date digest content-type'
to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest)
signer = PKCS1_v1_5.new(self.privkey)
digest = SHA256.new()
digest.update(to_be_signed.encode('utf-8'))
sig = base64.b64encode(signer.sign(digest))
sig = sig.decode('utf-8')
headers = {
'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"'
}
logger.info(f'signed request headers={headers}')
r.headers.update(headers)
return r

54
utils/key.py
View file

@ -1,22 +1,22 @@
import os
import binascii import binascii
import os
from Crypto.PublicKey import RSA
from typing import Callable from typing import Callable
KEY_DIR = os.path.join( from little_boxes.key import Key
os.path.dirname(os.path.abspath(__file__)), '..', 'config'
) KEY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config")
def _new_key() -> str: def _new_key() -> str:
return binascii.hexlify(os.urandom(32)).decode('utf-8') return binascii.hexlify(os.urandom(32)).decode("utf-8")
def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str:
key_path = os.path.join(KEY_DIR, f'{name}.key') """Loads or generates a cryptographic key."""
key_path = os.path.join(KEY_DIR, f"{name}.key")
if not os.path.exists(key_path): if not os.path.exists(key_path):
k = new_key() k = new_key()
with open(key_path, 'w+') as f: with open(key_path, "w+") as f:
f.write(k) f.write(k)
return k return k
@ -24,23 +24,19 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str:
return f.read() return f.read()
class Key(object): def get_key(owner: str, user: str, domain: str) -> Key:
DEFAULT_KEY_SIZE = 2048 """"Loads or generates an RSA key."""
def __init__(self, user: str, domain: str, create: bool = True) -> None: k = Key(owner)
user = user.replace('.', '_') user = user.replace(".", "_")
domain = domain.replace('.', '_') domain = domain.replace(".", "_")
key_path = os.path.join(KEY_DIR, f'key_{user}_{domain}.pem') key_path = os.path.join(KEY_DIR, f"key_{user}_{domain}.pem")
if os.path.isfile(key_path): if os.path.isfile(key_path):
with open(key_path) as f: with open(key_path) as f:
self.privkey_pem = f.read() privkey_pem = f.read()
self.privkey = RSA.importKey(self.privkey_pem) k.load(privkey_pem)
self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') else:
else: k.new()
if not create: with open(key_path, "w") as f:
raise Exception('must init private key first') f.write(k.privkey_pem)
k = RSA.generate(self.DEFAULT_KEY_SIZE)
self.privkey_pem = k.exportKey('PEM').decode('utf-8') return k
self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8')
with open(key_path, 'w') as f:
f.write(self.privkey_pem)
self.privkey = k

70
utils/linked_data_sig.py
View file

@ -1,70 +0,0 @@
from pyld import jsonld
import hashlib
from datetime import datetime
from Crypto.Signature import PKCS1_v1_5
from Crypto.Hash import SHA256
import base64
from typing import Any, Dict
# cache the downloaded "schemas", otherwise the library is super slow
# (https://github.com/digitalbazaar/pyld/issues/70)
_CACHE: Dict[str, Any] = {}
LOADER = jsonld.requests_document_loader()
def _caching_document_loader(url: str) -> Any:
if url in _CACHE:
return _CACHE[url]
resp = LOADER(url)
_CACHE[url] = resp
return resp
jsonld.set_document_loader(_caching_document_loader)
def options_hash(doc):
doc = dict(doc['signature'])
for k in ['type', 'id', 'signatureValue']:
if k in doc:
del doc[k]
doc['@context'] = 'https://w3id.org/identity/v1'
normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'})
h = hashlib.new('sha256')
h.update(normalized.encode('utf-8'))
return h.hexdigest()
def doc_hash(doc):
doc = dict(doc)
if 'signature' in doc:
del doc['signature']
normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'})
h = hashlib.new('sha256')
h.update(normalized.encode('utf-8'))
return h.hexdigest()
def verify_signature(doc, pubkey):
to_be_signed = options_hash(doc) + doc_hash(doc)
signature = doc['signature']['signatureValue']
signer = PKCS1_v1_5.new(pubkey)
digest = SHA256.new()
digest.update(to_be_signed.encode('utf-8'))
return signer.verify(digest, base64.b64decode(signature))
def generate_signature(doc, privkey):
options = {
'type': 'RsaSignature2017',
'creator': doc['actor'] + '#main-key',
'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z',
}
doc['signature'] = options
to_be_signed = options_hash(doc) + doc_hash(doc)
signer = PKCS1_v1_5.new(privkey)
digest = SHA256.new()
digest.update(to_be_signed.encode('utf-8'))
sig = base64.b64encode(signer.sign(digest))
options['signatureValue'] = sig.decode('utf-8')

72
utils/object_service.py
View file

@ -1,67 +1,21 @@
import requests import logging
from urllib.parse import urlparse
from .urlutils import check_url from little_boxes.activitypub import get_backend
from .errors import ActivityNotFoundError
logger = logging.getLogger(__name__)
class ObjectService(object): class ObjectService(object):
def __init__(self, user_agent, col, inbox, outbox, instances): def __init__(self):
self._user_agent = user_agent logger.debug("Initializing ObjectService")
self._col = col self._cache = {}
self._inbox = inbox
self._outbox = outbox
self._instances = instances
self._known_instances = set()
def _fetch_remote(self, object_id): def get(self, iri, reload_cache=False):
print(f'fetch remote {object_id}') logger.info(f"get actor {iri} (reload_cache={reload_cache})")
check_url(object_id)
resp = requests.get(object_id, headers={
'Accept': 'application/activity+json',
'User-Agent': self._user_agent,
})
if resp.status_code == 404:
raise ActivityNotFoundError(f'{object_id} cannot be fetched, 404 error not found')
resp.raise_for_status() if not reload_cache and iri in self._cache:
return resp.json() return self._cache[iri]
def _fetch(self, object_id):
instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl()
if instance not in self._known_instances:
self._known_instances.add(instance)
if not self._instances.find_one({'instance': instance}):
self._instances.insert({'instance': instance, 'first_object': object_id})
obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]})
if obj:
if obj['remote_id'] == object_id:
return obj['activity']
return obj['activity']['object']
obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]})
if obj:
if obj['remote_id'] == object_id:
return obj['activity']
return obj['activity']['object']
return self._fetch_remote(object_id)
def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None):
if reload_cache:
obj = self._fetch(object_id)
self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True)
return obj
cached_object = self._col.find_one({'object_id': object_id})
if cached_object:
print(f'ObjectService: {cached_object}')
return cached_object['cached_object']
obj = self._fetch(object_id)
self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True)
# print(f'ObjectService: {obj}')
obj = get_backend().fetch_iri(iri)
self._cache[iri] = obj
return obj return obj

30
utils/opengraph.py
View file

@ -1,36 +1,34 @@
from urllib.parse import urlparse
import ipaddress
import opengraph import opengraph
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from .urlutils import is_url_valid, check_url from little_boxes.urlutils import check_url
from little_boxes.urlutils import is_url_valid
def links_from_note(note): def links_from_note(note):
tags_href= set() tags_href = set()
for t in note.get('tag', []): for t in note.get("tag", []):
h = t.get('href') h = t.get("href")
if h: if h:
# TODO(tsileo): fetch the URL for Actor profile, type=mention # TODO(tsileo): fetch the URL for Actor profile, type=mention
tags_href.add(h) tags_href.add(h)
links = set() links = set()
soup = BeautifulSoup(note['content']) soup = BeautifulSoup(note["content"])
for link in soup.find_all('a'): for link in soup.find_all("a"):
h = link.get('href') h = link.get("href")
if h.startswith('http') and h not in tags_href and is_url_valid(h): if h.startswith("http") and h not in tags_href and is_url_valid(h):
links.add(h) links.add(h)
return links return links
def fetch_og_metadata(user_agent, col, remote_id): def fetch_og_metadata(user_agent, col, remote_id):
doc = col.find_one({'remote_id': remote_id}) doc = col.find_one({"remote_id": remote_id})
if not doc: if not doc:
raise ValueError raise ValueError
note = doc['activity']['object'] note = doc["activity"]["object"]
print(note) print(note)
links = links_from_note(note) links = links_from_note(note)
if not links: if not links:
@ -39,9 +37,11 @@ def fetch_og_metadata(user_agent, col, remote_id):
htmls = [] htmls = []
for l in links: for l in links:
check_url(l) check_url(l)
r = requests.get(l, headers={'User-Agent': user_agent}) r = requests.get(l, headers={"User-Agent": user_agent})
r.raise_for_status() r.raise_for_status()
htmls.append(r.text) htmls.append(r.text)
links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls]
col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) col.update_one(
{"remote_id": remote_id}, {"$set": {"meta.og_metadata": links_og_metadata}}
)
return len(links) return len(links)

47
utils/urlutils.py
View file

@ -1,47 +0,0 @@
import logging
import os
import socket
import ipaddress
from urllib.parse import urlparse
from . import strtobool
from .errors import Error
logger = logging.getLogger(__name__)
class InvalidURLError(Error):
pass
def is_url_valid(url: str) -> bool:
parsed = urlparse(url)
if parsed.scheme not in ['http', 'https']:
return False
# XXX in debug mode, we want to allow requests to localhost to test the federation with local instances
debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false'))
if debug_mode:
return True
if parsed.hostname in ['localhost']:
return False
try:
ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0]
except socket.gaierror:
logger.exception(f'failed to lookup url {url}')
return False
if ipaddress.ip_address(ip_address).is_private:
logger.info(f'rejecting private URL {url}')
return False
return True
def check_url(url: str) -> None:
if not is_url_valid(url):
raise InvalidURLError(f'"{url}" is invalid')
return None

75
utils/webfinger.py
View file

@ -1,75 +0,0 @@
from urllib.parse import urlparse
from typing import Dict, Any
from typing import Optional
import logging
import requests
from .urlutils import check_url
logger = logging.getLogger(__name__)
def webfinger(resource: str) -> Optional[Dict[str, Any]]:
"""Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL.
"""
logger.info(f'performing webfinger resolution for {resource}')
protos = ['https', 'http']
if resource.startswith('http://'):
protos.reverse()
host = urlparse(resource).netloc
elif resource.startswith('https://'):
host = urlparse(resource).netloc
else:
if resource.startswith('acct:'):
resource = resource[5:]
if resource.startswith('@'):
resource = resource[1:]
_, host = resource.split('@', 1)
resource='acct:'+resource
# Security check on the url (like not calling localhost)
check_url(f'https://{host}')
for i, proto in enumerate(protos):
try:
url = f'{proto}://{host}/.well-known/webfinger'
resp = requests.get(
url,
{'resource': resource}
)
except requests.ConnectionError:
# If we tried https first and the domain is "http only"
if i == 0:
continue
break
if resp.status_code == 404:
return None
resp.raise_for_status()
return resp.json()
def get_remote_follow_template(resource: str) -> Optional[str]:
data = webfinger(resource)
if data is None:
return None
for link in data['links']:
if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe':
return link.get('template')
return None
def get_actor_url(resource: str) -> Optional[str]:
"""Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL.
Returns:
the Actor URL or None if the resolution failed.
"""
data = webfinger(resource)
if data is None:
return None
for link in data['links']:
if link.get('rel') == 'self' and link.get('type') == 'application/activity+json':
return link.get('href')
return None