User API cleanup

This commit is contained in:
Thomas Sileo 2018-06-01 20:29:44 +02:00
parent 45afd99098
commit f8ee19b4d1
11 changed files with 422 additions and 178 deletions

View file

@ -15,6 +15,7 @@ script:
- mypy --ignore-missing-imports . - mypy --ignore-missing-imports .
- flake8 activitypub.py - flake8 activitypub.py
- cp -r tests/fixtures/me.yml config/me.yml - cp -r tests/fixtures/me.yml config/me.yml
- docker build . -t microblogpub:latest
- docker-compose up -d - docker-compose up -d
- docker-compose ps - docker-compose ps
- WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d

View file

@ -4,7 +4,18 @@ css:
password: password:
python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))"
docker:
mypy . --ignore-missing-imports
docker build . -t microblogpub:latest
reload-fed:
docker-compose -p instance2 -f docker-compose-tests.yml stop
docker-compose -p instance1 -f docker-compose-tests.yml stop
WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d --force-recreate --build
WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d --force-recreate --build
update: update:
docker-compose stop docker-compose stop
git pull git pull
docker build . -t microblogpub:latest
docker-compose up -d --force-recreate --build docker-compose up -d --force-recreate --build

144
README.md
View file

@ -87,6 +87,20 @@ $ docker-compose -f docker-compose-dev.yml up -d
$ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads $ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads
``` ```
## ActivityPub API
### GET /
Returns the actor profile, with links to all the "standard" collections.
### GET /tags/:tag
Special collection that reference notes with the given tag.
### GET /stream
Special collection that returns the stream/inbox as displayed in the UI.
## User API ## User API
The user API is used by the admin UI (and requires a CSRF token when used with a regular user session), but it can also be accessed with an API key. The user API is used by the admin UI (and requires a CSRF token when used with a regular user session), but it can also be accessed with an API key.
@ -95,7 +109,7 @@ All the examples are using [HTTPie](https://httpie.org/).
### POST /api/note/delete{?id} ### POST /api/note/delete{?id}
Deletes the given note `id`. Deletes the given note `id` (the note must from the instance outbox).
Answers a **201** (Created) status code. Answers a **201** (Created) status code.
@ -104,7 +118,7 @@ You can pass the `id` via JSON, form data or query argument.
#### Example #### Example
```shell ```shell
$ http POST https://microblog.pub/api/note/delete Authorization:'Bearer <token>' id=http://microblob.pub/outbox/<node_id>/activity $ http POST https://microblog.pub/api/note/delete Authorization:'Bearer <token>' id=http://microblob.pub/outbox/<note_id>/activity
``` ```
#### Response #### Response
@ -115,6 +129,132 @@ $ http POST https://microblog.pub/api/note/delete Authorization:'Bearer <token>'
} }
``` ```
### POST /api/like{?id}
Likes the given activity.
Answers a **201** (Created) status code.
You can pass the `id` via JSON, form data or query argument.
#### Example
```shell
$ http POST https://microblog.pub/api/like Authorization:'Bearer <token>' id=http://activity-iri.tld
```
#### Response
```json
{
"activity": "https://microblog.pub/outbox/<like_id>"
}
```
### POST /api/boost{?id}
Boosts/Announces the given activity.
Answers a **201** (Created) status code.
You can pass the `id` via JSON, form data or query argument.
#### Example
```shell
$ http POST https://microblog.pub/api/boost Authorization:'Bearer <token>' id=http://activity-iri.tld
```
#### Response
```json
{
"activity": "https://microblog.pub/outbox/<announce_id>"
}
```
### POST /api/block{?actor}
Blocks the given actor, all activities from this actor will be dropped after that.
Answers a **201** (Created) status code.
You can pass the `id` via JSON, form data or query argument.
#### Example
```shell
$ http POST https://microblog.pub/api/block Authorization:'Bearer <token>' actor=http://actor-iri.tld/
```
#### Response
```json
{
"activity": "https://microblog.pub/outbox/<block_id>"
}
```
### POST /api/follow{?actor}
Follows the given actor.
Answers a **201** (Created) status code.
You can pass the `id` via JSON, form data or query argument.
#### Example
```shell
$ http POST https://microblog.pub/api/follow Authorization:'Bearer <token>' actor=http://actor-iri.tld/
```
#### Response
```json
{
"activity": "https://microblog.pub/outbox/<follow_id>"
}
```
### POST /api/new_note{?content,reply}
Creates a new note. `reply` is the IRI of the "replied" note if any.
Answers a **201** (Created) status code.
You can pass the `content` and `reply` via JSON, form data or query argument.
#### Example
```shell
$ http POST https://microblog.pub/api/new_note Authorization:'Bearer <token>' content=hello
```
#### Response
```json
{
"activity": "https://microblog.pub/outbox/<create_id>"
}
```
### GET /api/stream
#### Example
```shell
$ http GET https://microblog.pub/api/stream Authorization:'Bearer <token>'
```
#### Response
```json
```
## Contributions ## Contributions
PRs are welcome, please open an issue to start a discussion before your start any work. PRs are welcome, please open an issue to start a discussion before your start any work.

View file

@ -9,13 +9,12 @@ from bson.objectid import ObjectId
from html2text import html2text from html2text import html2text
from feedgen.feed import FeedGenerator from feedgen.feed import FeedGenerator
from utils.linked_data_sig import generate_signature
from utils.actor_service import NotAnActorError from utils.actor_service import NotAnActorError
from utils.errors import BadActivityError, UnexpectedActivityTypeError from utils.errors import BadActivityError, UnexpectedActivityTypeError
from utils import activitypub_utils from utils import activitypub_utils
from config import USERNAME, BASE_URL, ID from config import USERNAME, BASE_URL, ID
from config import CTX_AS, CTX_SECURITY, AS_PUBLIC from config import CTX_AS, CTX_SECURITY, AS_PUBLIC
from config import KEY, DB, ME, ACTOR_SERVICE from config import DB, ME, ACTOR_SERVICE
from config import OBJECT_SERVICE from config import OBJECT_SERVICE
from config import PUBLIC_INSTANCES from config import PUBLIC_INSTANCES
import tasks import tasks
@ -350,7 +349,6 @@ class BaseActivity(object):
except NotImplementedError: except NotImplementedError:
logger.debug('post to outbox hook not implemented') logger.debug('post to outbox hook not implemented')
generate_signature(activity, KEY.privkey)
payload = json.dumps(activity) payload = json.dumps(activity)
for recp in recipients: for recp in recipients:
logger.debug(f'posting to {recp}') logger.debug(f'posting to {recp}')
@ -571,7 +569,6 @@ class Like(BaseActivity):
# Update the meta counter if the object is published by the server # Update the meta counter if the object is published by the server
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_like': 1}, '$inc': {'meta.count_like': 1},
'$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)},
}) })
# XXX(tsileo): notification?? # XXX(tsileo): notification??
@ -580,7 +577,6 @@ class Like(BaseActivity):
# Update the meta counter if the object is published by the server # Update the meta counter if the object is published by the server
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_like': -1}, '$inc': {'meta.count_like': -1},
'$pull': {'meta.col_likes': {'id': self.id}},
}) })
def _undo_should_purge_cache(self) -> bool: def _undo_should_purge_cache(self) -> bool:
@ -592,7 +588,6 @@ class Like(BaseActivity):
# Unlikely, but an actor can like it's own post # Unlikely, but an actor can like it's own post
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_like': 1}, '$inc': {'meta.count_like': 1},
'$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)},
}) })
# Keep track of the like we just performed # Keep track of the like we just performed
@ -603,7 +598,6 @@ class Like(BaseActivity):
# Unlikely, but an actor can like it's own post # Unlikely, but an actor can like it's own post
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_like': -1}, '$inc': {'meta.count_like': -1},
'$pull': {'meta.col_likes': {'id': self.id}},
}) })
DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}})
@ -646,7 +640,6 @@ class Announce(BaseActivity):
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_boost': 1}, '$inc': {'meta.count_boost': 1},
'$addToSet': {'meta.col_shares': self.to_dict(embed=True, embed_object_id_only=True)},
}) })
def _undo_inbox(self) -> None: def _undo_inbox(self) -> None:
@ -654,7 +647,6 @@ class Announce(BaseActivity):
# Update the meta counter if the object is published by the server # Update the meta counter if the object is published by the server
DB.outbox.update_one({'activity.object.id': obj.id}, { DB.outbox.update_one({'activity.object.id': obj.id}, {
'$inc': {'meta.count_boost': -1}, '$inc': {'meta.count_boost': -1},
'$pull': {'meta.col_shares': {'id': self.id}},
}) })
def _undo_should_purge_cache(self) -> bool: def _undo_should_purge_cache(self) -> bool:
@ -1079,11 +1071,12 @@ def embed_collection(total_items, first_page_id):
return { return {
"type": ActivityType.ORDERED_COLLECTION.value, "type": ActivityType.ORDERED_COLLECTION.value,
"totalItems": total_items, "totalItems": total_items,
"first": first_page_id, "first": f'{first_page_id}?page=first',
"id": first_page_id,
} }
def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False):
col_name = col_name or col.name col_name = col_name or col.name
if q is None: if q is None:
q = {} q = {}
@ -1127,6 +1120,9 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50,
if len(data) == limit: if len(data) == limit:
resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor
if first_page:
return resp['first']
return resp return resp
# If there's a cursor, then we return an OrderedCollectionPage # If there's a cursor, then we return an OrderedCollectionPage
@ -1141,6 +1137,9 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50,
if len(data) == limit: if len(data) == limit:
resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor
# TODO(tsileo): implements prev with prev=<first item cursor> if first_page:
return resp['first']
# XXX(tsileo): implements prev with prev=<first item cursor>?
return resp return resp

174
app.py
View file

@ -62,6 +62,8 @@ from utils.webfinger import get_actor_url
from utils.errors import Error from utils.errors import Error
from utils.errors import UnexpectedActivityTypeError from utils.errors import UnexpectedActivityTypeError
from utils.errors import BadActivityError from utils.errors import BadActivityError
from utils.errors import NotFromOutboxError
from utils.errors import ActivityNotFoundError
from typing import Dict, Any from typing import Dict, Any
@ -509,7 +511,7 @@ def outbox():
DB.outbox, DB.outbox,
q=q, q=q,
cursor=request.args.get('cursor'), cursor=request.args.get('cursor'),
map_func=lambda doc: clean_activity(doc['activity']), map_func=lambda doc: activity_from_doc(doc),
)) ))
# Handle POST request # Handle POST request
@ -557,7 +559,7 @@ def outbox_activity_replies(item_id):
data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False})
if not data: if not data:
abort(404) abort(404)
obj = activitypub.parse_activity(data) obj = activitypub.parse_activity(data['activity'])
if obj.type_enum != ActivityType.CREATE: if obj.type_enum != ActivityType.CREATE:
abort(404) abort(404)
@ -571,8 +573,9 @@ def outbox_activity_replies(item_id):
DB.inbox, DB.inbox,
q=q, q=q,
cursor=request.args.get('cursor'), cursor=request.args.get('cursor'),
map_func=lambda doc: doc['activity'], map_func=lambda doc: doc['activity']['object'],
col_name=f'outbox/{item_id}/replies', col_name=f'outbox/{item_id}/replies',
first_page=request.args.get('page') == 'first',
)) ))
@ -583,7 +586,7 @@ def outbox_activity_likes(item_id):
data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False})
if not data: if not data:
abort(404) abort(404)
obj = activitypub.parse_activity(data) obj = activitypub.parse_activity(data['activity'])
if obj.type_enum != ActivityType.CREATE: if obj.type_enum != ActivityType.CREATE:
abort(404) abort(404)
@ -600,6 +603,7 @@ def outbox_activity_likes(item_id):
cursor=request.args.get('cursor'), cursor=request.args.get('cursor'),
map_func=lambda doc: doc['activity'], map_func=lambda doc: doc['activity'],
col_name=f'outbox/{item_id}/likes', col_name=f'outbox/{item_id}/likes',
first_page=request.args.get('page') == 'first',
)) ))
@ -610,7 +614,7 @@ def outbox_activity_shares(item_id):
data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False})
if not data: if not data:
abort(404) abort(404)
obj = activitypub.parse_activity(data) obj = activitypub.parse_activity(data['activity'])
if obj.type_enum != ActivityType.CREATE: if obj.type_enum != ActivityType.CREATE:
abort(404) abort(404)
@ -627,6 +631,7 @@ def outbox_activity_shares(item_id):
cursor=request.args.get('cursor'), cursor=request.args.get('cursor'),
map_func=lambda doc: doc['activity'], map_func=lambda doc: doc['activity'],
col_name=f'outbox/{item_id}/shares', col_name=f'outbox/{item_id}/shares',
first_page=request.args.get('page') == 'first',
)) ))
@ -744,16 +749,26 @@ def api_user_key():
return flask_jsonify(api_key=ADMIN_API_KEY) return flask_jsonify(api_key=ADMIN_API_KEY)
def _user_api_get_note(): def _user_api_arg(key: str) -> str:
"""Try to get the given key from the requests, try JSON body, form data and query arg."""
if request.is_json: if request.is_json:
oid = request.json.get('id') oid = request.json.get(key)
else: else:
oid = request.args.get('id') or request.form.get('id') oid = request.args.get(key) or request.form.get(key)
if not oid: if not oid:
raise ValueError('missing id') raise ValueError(f'missing {key}')
return activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) return oid
def _user_api_get_note(from_outbox: bool = False):
oid = _user_api_arg('id')
note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE)
if from_outbox and not note.id.startswith(ID):
raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server')
return note
def _user_api_response(**kwargs): def _user_api_response(**kwargs):
@ -769,64 +784,50 @@ def _user_api_response(**kwargs):
@api_required @api_required
def api_delete(): def api_delete():
"""API endpoint to delete a Note activity.""" """API endpoint to delete a Note activity."""
note = _user_api_get_note() note = _user_api_get_note(from_outbox=True)
delete = note.build_delete() delete = note.build_delete()
delete.post_to_outbox() delete.post_to_outbox()
return _user_api_response(activity=delete.id) return _user_api_response(activity=delete.id)
@app.route('/api/boost') @app.route('/api/boost', methods=['POST'])
@api_required @api_required
def api_boost(): def api_boost():
# FIXME(tsileo): ensure a Note and not a Create is given note = _user_api_get_note()
oid = request.args.get('id')
obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid))
announce = obj.build_announce()
announce.post_to_outbox()
if request.args.get('redirect'):
return redirect(request.args.get('redirect'))
return Response(
status=201,
headers={'Microblogpub-Created-Activity': announce.id},
)
@app.route('/api/like') announce = note.build_announce()
announce.post_to_outbox()
return _user_api_response(activity=announce.id)
@app.route('/api/like', methods=['POST'])
@api_required @api_required
def api_like(): def api_like():
# FIXME(tsileo): ensure a Note and not a Create is given note = _user_api_get_note()
oid = request.args.get('id')
obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) like = note.build_like()
if not obj:
raise ValueError(f'unkown {oid} object')
like = obj.build_like()
like.post_to_outbox() like.post_to_outbox()
if request.args.get('redirect'):
return redirect(request.args.get('redirect')) return _user_api_response(activity=like.id)
return Response(
status=201,
headers={'Microblogpub-Created-Activity': like.id},
)
@app.route('/api/undo', methods=['GET', 'POST']) @app.route('/api/undo', methods=['POST'])
@api_required @api_required
def api_undo(): def api_undo():
oid = request.args.get('id') oid = _user_api_arg('id')
doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]})
undo_id = None if not doc:
if doc: raise ActivityNotFoundError(f'cannot found {oid}')
obj = activitypub.parse_activity(doc.get('activity'))
# FIXME(tsileo): detect already undo-ed and make this API call idempotent obj = activitypub.parse_activity(doc.get('activity'))
undo = obj.build_undo() # FIXME(tsileo): detect already undo-ed and make this API call idempotent
undo.post_to_outbox() undo = obj.build_undo()
undo_id = undo.id undo.post_to_outbox()
if request.args.get('redirect'):
return redirect(request.args.get('redirect')) return _user_api_response(activity=undo.id)
return Response(
status=201,
headers={'Microblogpub-Created-Activity': undo_id},
)
@app.route('/stream') @app.route('/stream')
@ -980,22 +981,27 @@ def api_upload():
) )
@app.route('/api/new_note') @app.route('/api/new_note', methods=['POST'])
@api_required @api_required
def api_new_note(): def api_new_note():
source = request.args.get('content') source = _user_api_arg('content')
if not source: if not source:
raise ValueError('missing content') raise ValueError('missing content')
reply = None _reply, reply = None, None
if request.args.get('reply'): try:
reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) _reply = _user_api_arg('reply')
source = request.args.get('content') except ValueError:
pass
content, tags = parse_markdown(source) content, tags = parse_markdown(source)
to = request.args.get('to') to = request.args.get('to')
cc = [ID+'/followers'] cc = [ID+'/followers']
if reply:
if _reply:
reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply))
cc.append(reply.attributedTo) cc.append(reply.attributedTo)
for tag in tags: for tag in tags:
if tag['type'] == 'Mention': if tag['type'] == 'Mention':
cc.append(tag['href']) cc.append(tag['href'])
@ -1003,7 +1009,7 @@ def api_new_note():
note = activitypub.Note( note = activitypub.Note(
cc=cc, cc=cc,
to=[to if to else config.AS_PUBLIC], to=[to if to else config.AS_PUBLIC],
content=content, # TODO(tsileo): handle markdown content=content,
tag=tags, tag=tags,
source={'mediaType': 'text/markdown', 'content': source}, source={'mediaType': 'text/markdown', 'content': source},
inReplyTo=reply.id if reply else None inReplyTo=reply.id if reply else None
@ -1011,11 +1017,8 @@ def api_new_note():
create = note.build_create() create = note.build_create()
create.post_to_outbox() create.post_to_outbox()
return Response( return _user_api_response(activity=create.id)
status=201,
response='OK',
headers={'Microblogpub-Created-Activity': create.id},
)
@app.route('/api/stream') @app.route('/api/stream')
@api_required @api_required
@ -1026,41 +1029,38 @@ def api_stream():
) )
@app.route('/api/block') @app.route('/api/block', methods=['POST'])
@api_required @api_required
def api_block(): def api_block():
# FIXME(tsileo): ensure it's a Person ID actor = _user_api_arg('actor')
actor = request.args.get('actor')
if not actor: existing = DB.outbox.find_one({
raise ValueError('missing actor') 'type': ActivityType.BLOCK.value,
if DB.outbox.find_one({'type': ActivityType.BLOCK.value, 'activity.object': actor,
'activity.object': actor, 'meta.undo': False,
'meta.undo': False}): })
return Response(status=201) if existing:
return _user_api_response(activity=existing['activity']['id'])
block = activitypub.Block(object=actor) block = activitypub.Block(object=actor)
block.post_to_outbox() block.post_to_outbox()
return Response(
status=201, return _user_api_response(activity=block.id)
headers={'Microblogpub-Created-Activity': block.id},
)
@app.route('/api/follow') @app.route('/api/follow', methods=['POST'])
@api_required @api_required
def api_follow(): def api_follow():
actor = request.args.get('actor') actor = _user_api_arg('actor')
if not actor:
raise ValueError('missing actor') existing = DB.following.find_one({'remote_actor': actor})
if DB.following.find({'remote_actor': actor}).count() > 0: if existing:
return Response(status=201) return _user_api_response(activity=existing['activity']['id'])
follow = activitypub.Follow(object=actor) follow = activitypub.Follow(object=actor)
follow.post_to_outbox() follow.post_to_outbox()
return Response(
status=201, return _user_api_response(activity=follow.id)
headers={'Microblogpub-Created-Activity': follow.id},
)
@app.route('/followers') @app.route('/followers')

View file

@ -1,7 +1,7 @@
version: '3.5' version: '3.5'
services: services:
web: web:
build: . image: 'microblogpub:latest'
ports: ports:
- "${WEB_PORT}:5005" - "${WEB_PORT}:5005"
links: links:
@ -16,7 +16,7 @@ services:
- MICROBLOGPUB_DEBUG=1 - MICROBLOGPUB_DEBUG=1
celery: celery:
# image: "instance1_web" # image: "instance1_web"
build: . image: 'microblogpub:latest'
links: links:
- mongo - mongo
- rmq - rmq

View file

@ -1,7 +1,7 @@
version: '2' version: '2'
services: services:
web: web:
build: . image: 'microblogpub:latest'
ports: ports:
- "${WEB_PORT}:5005" - "${WEB_PORT}:5005"
links: links:
@ -14,7 +14,7 @@ services:
- MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq//
- MICROBLOGPUB_MONGODB_HOST=mongo:27017 - MICROBLOGPUB_MONGODB_HOST=mongo:27017
celery: celery:
build: . image: 'microblogpub:latest'
links: links:
- mongo - mongo
- rmq - rmq

View file

@ -1,4 +1,5 @@
import os import os
import json
import logging import logging
import random import random
@ -13,6 +14,7 @@ from config import KEY
from config import USER_AGENT from config import USER_AGENT
from utils.httpsig import HTTPSigAuth from utils.httpsig import HTTPSigAuth
from utils.opengraph import fetch_og_metadata from utils.opengraph import fetch_og_metadata
from utils.linked_data_sig import generate_signature
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -22,11 +24,14 @@ SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey)
@app.task(bind=True, max_retries=12) @app.task(bind=True, max_retries=12)
def post_to_inbox(self, payload, to): def post_to_inbox(self, payload: str, to: str) -> None:
try: try:
log.info('payload=%s', payload) log.info('payload=%s', payload)
log.info('generating sig')
signed_payload = json.loads(payload)
generate_signature(signed_payload, KEY.privkey)
log.info('to=%s', to) log.info('to=%s', to)
resp = requests.post(to, data=payload, auth=SigAuth, headers={ resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={
'Content-Type': HEADERS[1], 'Content-Type': HEADERS[1],
'Accept': HEADERS[1], 'Accept': HEADERS[1],
'User-Agent': USER_AGENT, 'User-Agent': USER_AGENT,

View file

@ -5,6 +5,9 @@ import requests
from html2text import html2text from html2text import html2text
from utils import activitypub_utils from utils import activitypub_utils
from typing import Tuple
from typing import List
def resp2plaintext(resp): def resp2plaintext(resp):
"""Convert the body of a requests reponse to plain text in order to make basic assertions.""" """Convert the body of a requests reponse to plain text in order to make basic assertions."""
@ -17,107 +20,149 @@ class Instance(object):
def __init__(self, name, host_url, docker_url=None): def __init__(self, name, host_url, docker_url=None):
self.host_url = host_url self.host_url = host_url
self.docker_url = docker_url or host_url self.docker_url = docker_url or host_url
self.session = requests.Session()
self._create_delay = 10 self._create_delay = 10
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key')) as f: with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key')
) as f:
api_key = f.read() api_key = f.read()
self._auth_headers = {'Authorization': f'Bearer {api_key}'} self._auth_headers = {'Authorization': f'Bearer {api_key}'}
def _do_req(self, url, headers): def _do_req(self, url, headers):
"""Used to parse collection."""
url = url.replace(self.docker_url, self.host_url) url = url.replace(self.docker_url, self.host_url)
resp = requests.get(url, headers=headers) resp = requests.get(url, headers=headers)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def _parse_collection(self, payload=None, url=None): def _parse_collection(self, payload=None, url=None):
"""Parses a collection (go through all the pages)."""
return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req)
def ping(self): def ping(self):
"""Ensures the homepage is reachable.""" """Ensures the homepage is reachable."""
resp = self.session.get(f'{self.host_url}/') resp = requests.get(f'{self.host_url}/')
resp.raise_for_status() resp.raise_for_status()
assert resp.status_code == 200 assert resp.status_code == 200
def debug(self): def debug(self):
resp = self.session.get(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) """Returns the debug infos (number of items in the inbox/outbox."""
resp = requests.get(
f'{self.host_url}/api/debug',
headers={**self._auth_headers, 'Accept': 'application/json'},
)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def drop_db(self): def drop_db(self):
resp = self.session.delete(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) """Drops the MongoDB DB."""
resp = requests.delete(
f'{self.host_url}/api/debug',
headers={**self._auth_headers, 'Accept': 'application/json'},
)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def login(self):
resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'})
resp.raise_for_status()
assert resp.status_code == 200
def block(self, actor_url) -> None: def block(self, actor_url) -> None:
"""Blocks an actor."""
# Instance1 follows instance2 # Instance1 follows instance2
resp = self.session.get(f'{self.host_url}/api/block', params={'actor': actor_url}) resp = requests.post(
f'{self.host_url}/api/block',
params={'actor': actor_url},
headers=self._auth_headers,
)
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay/2) time.sleep(self._create_delay/2)
return resp.headers.get('microblogpub-created-activity') return resp.json().get('activity')
def follow(self, instance: 'Instance') -> None: def follow(self, instance: 'Instance') -> str:
"""Follows another instance."""
# Instance1 follows instance2 # Instance1 follows instance2
resp = self.session.get(f'{self.host_url}/api/follow', params={'actor': instance.docker_url}) resp = requests.post(
f'{self.host_url}/api/follow',
json={'actor': instance.docker_url},
headers=self._auth_headers,
)
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.headers.get('microblogpub-created-activity') return resp.json().get('activity')
def new_note(self, content, reply=None): def new_note(self, content, reply=None) -> str:
"""Creates a new note."""
params = {'content': content} params = {'content': content}
if reply: if reply:
params['reply'] = reply params['reply'] = reply
resp = self.session.get(f'{self.host_url}/api/new_note', params=params)
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.headers.get('microblogpub-created-activity')
def boost(self, activity_id):
resp = self.session.get(f'{self.host_url}/api/boost', params={'id': activity_id})
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.headers.get('microblogpub-created-activity')
def like(self, activity_id):
resp = self.session.get(f'{self.host_url}/api/like', params={'id': activity_id})
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.headers.get('microblogpub-created-activity')
def delete(self, oid: str) -> None:
resp = requests.post( resp = requests.post(
f'{self.host_url}/api/note/delete', f'{self.host_url}/api/new_note',
json={'id': oid}, json=params,
headers=self._auth_headers, headers=self._auth_headers,
) )
assert resp.status_code == 201 assert resp.status_code == 201
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.json().get('activity') return resp.json().get('activity')
def undo(self, oid: str) -> None: def boost(self, oid: str) -> str:
resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) """Creates an Announce activity."""
resp = requests.post(
f'{self.host_url}/api/boost',
json={'id': oid},
headers=self._auth_headers,
)
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.json().get('activity')
def like(self, oid: str) -> str:
"""Creates a Like activity."""
resp = requests.post(
f'{self.host_url}/api/like',
json={'id': oid},
headers=self._auth_headers,
)
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.json().get('activity')
def delete(self, oid: str) -> str:
"""Creates a Delete activity."""
resp = requests.post(
f'{self.host_url}/api/note/delete',
json={'id': oid},
headers=self._auth_headers,
)
assert resp.status_code == 201
time.sleep(self._create_delay)
return resp.json().get('activity')
def undo(self, oid: str) -> str:
"""Creates a Undo activity."""
resp = requests.post(
f'{self.host_url}/api/undo',
json={'id': oid},
headers=self._auth_headers,
)
assert resp.status_code == 201 assert resp.status_code == 201
# We need to wait for the Follow/Accept dance # We need to wait for the Follow/Accept dance
time.sleep(self._create_delay) time.sleep(self._create_delay)
return resp.headers.get('microblogpub-created-activity') return resp.json().get('activity')
def followers(self): def followers(self) -> List[str]:
resp = self.session.get(f'{self.host_url}/followers', headers={'Accept': 'application/activity+json'}) """Parses the followers collection."""
resp = requests.get(
f'{self.host_url}/followers',
headers={'Accept': 'application/activity+json'},
)
resp.raise_for_status() resp.raise_for_status()
data = resp.json() data = resp.json()
@ -125,7 +170,11 @@ class Instance(object):
return self._parse_collection(payload=data) return self._parse_collection(payload=data)
def following(self): def following(self):
resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) """Parses the following collection."""
resp = requests.get(
f'{self.host_url}/following',
headers={'Accept': 'application/activity+json'},
)
resp.raise_for_status() resp.raise_for_status()
data = resp.json() data = resp.json()
@ -133,38 +182,50 @@ class Instance(object):
return self._parse_collection(payload=data) return self._parse_collection(payload=data)
def outbox(self): def outbox(self):
resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) """Returns the instance outbox."""
resp = requests.get(
f'{self.host_url}/following',
headers={'Accept': 'application/activity+json'},
)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def outbox_get(self, aid): def outbox_get(self, aid):
resp = self.session.get(aid.replace(self.docker_url, self.host_url), headers={'Accept': 'application/activity+json'}) """Fetches a specific item from the instance outbox."""
resp = requests.get(
aid.replace(self.docker_url, self.host_url),
headers={'Accept': 'application/activity+json'},
)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def stream_jsonfeed(self): def stream_jsonfeed(self):
resp = self.session.get(f'{self.host_url}/api/stream', headers={'Accept': 'application/json'}) """Returns the "stream"'s JSON feed."""
resp = requests.get(
f'{self.host_url}/api/stream',
headers={**self._auth_headers, 'Accept': 'application/json'},
)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
def _instances(): def _instances() -> Tuple[Instance, Instance]:
"""Initializes the client for the two test instances."""
instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005')
instance1.ping() instance1.ping()
instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005')
instance2.ping() instance2.ping()
# Login # Return the DB
instance1.login()
instance1.drop_db() instance1.drop_db()
instance2.login()
instance2.drop_db() instance2.drop_db()
return instance1, instance2 return instance1, instance2
def test_follow(): def test_follow() -> None:
"""instance1 follows instance2."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -181,6 +242,7 @@ def test_follow():
def test_follow_unfollow(): def test_follow_unfollow():
"""instance1 follows instance2, then unfollows it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
follow_id = instance1.follow(instance2) follow_id = instance1.follow(instance2)
@ -210,6 +272,7 @@ def test_follow_unfollow():
def test_post_content(): def test_post_content():
"""Instances follow each other, and instance1 creates a note."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -230,6 +293,7 @@ def test_post_content():
def test_block_and_post_content(): def test_block_and_post_content():
"""Instances follow each other, instance2 blocks instance1, instance1 creates a new note."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -251,6 +315,7 @@ def test_block_and_post_content():
def test_post_content_and_delete(): def test_post_content_and_delete():
"""Instances follow each other, instance1 creates a new note, then deletes it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -280,6 +345,7 @@ def test_post_content_and_delete():
def test_post_content_and_like(): def test_post_content_and_like():
"""Instances follow each other, instance1 creates a new note, instance2 likes it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -302,10 +368,13 @@ def test_post_content_and_like():
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f'{create_id}/activity')
assert 'likes' in note assert 'likes' in note
assert note['likes']['totalItems'] == 1 assert note['likes']['totalItems'] == 1
# assert note['likes']['items'][0]['id'] == like_id likes = instance1._parse_collection(url=note['likes']['first'])
assert len(likes) == 1
assert likes[0]['id'] == like_id
def test_post_content_and_like_unlike(): def test_post_content_and_like_unlike() -> None:
"""Instances follow each other, instance1 creates a new note, instance2 likes it, then unlikes it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -328,8 +397,9 @@ def test_post_content_and_like_unlike():
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f'{create_id}/activity')
assert 'likes' in note assert 'likes' in note
assert note['likes']['totalItems'] == 1 assert note['likes']['totalItems'] == 1
# FIXME(tsileo): parse the collection likes = instance1._parse_collection(url=note['likes']['first'])
# assert note['likes']['items'][0]['id'] == like_id assert len(likes) == 1
assert likes[0]['id'] == like_id
instance2.undo(like_id) instance2.undo(like_id)
@ -342,7 +412,8 @@ def test_post_content_and_like_unlike():
assert note['likes']['totalItems'] == 0 assert note['likes']['totalItems'] == 0
def test_post_content_and_boost(): def test_post_content_and_boost() -> None:
"""Instances follow each other, instance1 creates a new note, instance2 "boost" it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -365,11 +436,13 @@ def test_post_content_and_boost():
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f'{create_id}/activity')
assert 'shares' in note assert 'shares' in note
assert note['shares']['totalItems'] == 1 assert note['shares']['totalItems'] == 1
# FIXME(tsileo): parse the collection shares = instance1._parse_collection(url=note['shares']['first'])
# assert note['shares']['items'][0]['id'] == boost_id assert len(shares) == 1
assert shares[0]['id'] == boost_id
def test_post_content_and_boost_unboost(): def test_post_content_and_boost_unboost() -> None:
"""Instances follow each other, instance1 creates a new note, instance2 "boost" it, then "unboost" it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -392,8 +465,9 @@ def test_post_content_and_boost_unboost():
note = instance1.outbox_get(f'{create_id}/activity') note = instance1.outbox_get(f'{create_id}/activity')
assert 'shares' in note assert 'shares' in note
assert note['shares']['totalItems'] == 1 assert note['shares']['totalItems'] == 1
# FIXME(tsileo): parse the collection shares = instance1._parse_collection(url=note['shares']['first'])
# assert note['shares']['items'][0]['id'] == boost_id assert len(shares) == 1
assert shares[0]['id'] == boost_id
instance2.undo(boost_id) instance2.undo(boost_id)
@ -406,7 +480,8 @@ def test_post_content_and_boost_unboost():
assert note['shares']['totalItems'] == 0 assert note['shares']['totalItems'] == 0
def test_post_content_and_post_reply(): def test_post_content_and_post_reply() -> None:
"""Instances follow each other, instance1 creates a new note, instance2 replies to it."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -425,7 +500,10 @@ def test_post_content_and_post_reply():
assert len(instance2_inbox_stream['items']) == 1 assert len(instance2_inbox_stream['items']) == 1
assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id
instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') instance2_create_id = instance2.new_note(
f'hey @instance1@{instance1.docker_url}',
reply=f'{instance1_create_id}/activity',
)
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there
assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity
@ -441,10 +519,13 @@ def test_post_content_and_post_reply():
instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity')
assert 'replies' in instance1_note assert 'replies' in instance1_note
assert instance1_note['replies']['totalItems'] == 1 assert instance1_note['replies']['totalItems'] == 1
# TODO(tsileo): inspect the `replies` collection replies = instance1._parse_collection(url=instance1_note['replies']['first'])
assert len(replies) == 1
assert replies[0]['id'] == f'{instance2_create_id}/activity'
def test_post_content_and_post_reply_and_delete(): def test_post_content_and_post_reply_and_delete() -> None:
"""Instances follow each other, instance1 creates a new note, instance2 replies to it, then deletes its reply."""
instance1, instance2 = _instances() instance1, instance2 = _instances()
# Instance1 follows instance2 # Instance1 follows instance2
instance1.follow(instance2) instance1.follow(instance2)
@ -463,7 +544,10 @@ def test_post_content_and_post_reply_and_delete():
assert len(instance2_inbox_stream['items']) == 1 assert len(instance2_inbox_stream['items']) == 1
assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id
instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') instance2_create_id = instance2.new_note(
f'hey @instance1@{instance1.docker_url}',
reply=f'{instance1_create_id}/activity',
)
instance2_debug = instance2.debug() instance2_debug = instance2.debug()
assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there
assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity

View file

@ -18,6 +18,9 @@ class Error(Exception):
return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})'
class NotFromOutboxError(Error):
pass
class ActivityNotFoundError(Error): class ActivityNotFoundError(Error):
status_code = 404 status_code = 404

View file

@ -25,6 +25,7 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str:
class Key(object): class Key(object):
DEFAULT_KEY_SIZE = 2048
def __init__(self, user: str, domain: str, create: bool = True) -> None: def __init__(self, user: str, domain: str, create: bool = True) -> None:
user = user.replace('.', '_') user = user.replace('.', '_')
domain = domain.replace('.', '_') domain = domain.replace('.', '_')
@ -37,7 +38,7 @@ class Key(object):
else: else:
if not create: if not create:
raise Exception('must init private key first') raise Exception('must init private key first')
k = RSA.generate(4096) k = RSA.generate(self.DEFAULT_KEY_SIZE)
self.privkey_pem = k.exportKey('PEM').decode('utf-8') self.privkey_pem = k.exportKey('PEM').decode('utf-8')
self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8')
with open(key_path, 'w') as f: with open(key_path, 'w') as f: