From cd2ae3d7b0c5986abef2dffb8324ff8c67cee8b1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 1 Aug 2019 21:44:35 +0200 Subject: [PATCH] Move stuff arround --- app.py | 20 +++---- blueprints/admin.py | 18 +++--- blueprints/api.py | 10 ++-- blueprints/indieauth.py | 4 +- blueprints/tasks.py | 14 ++--- config.py | 85 ----------------------------- activity_gc.py => core/gc.py | 0 core/indexes.py | 76 ++++++++++++++++++++++++++ migrations.py => core/migrations.py | 0 app_utils.py => core/shared.py | 0 docker-compose-dev.yml | 2 +- run.sh | 4 +- 12 files changed, 112 insertions(+), 121 deletions(-) rename activity_gc.py => core/gc.py (100%) create mode 100644 core/indexes.py rename migrations.py => core/migrations.py (100%) rename app_utils.py => core/shared.py (100%) diff --git a/app.py b/app.py index 914e202..2f57dac 100644 --- a/app.py +++ b/app.py @@ -38,16 +38,16 @@ import blueprints.well_known import config from activitypub import Box from activitypub import embed_collection -from app_utils import MY_PERSON -from app_utils import _add_answers_to_question -from app_utils import _build_thread -from app_utils import _get_ip -from app_utils import back -from app_utils import csrf -from app_utils import login_required -from app_utils import noindex -from app_utils import paginated_query -from app_utils import post_to_outbox +from core.shared import MY_PERSON +from core.shared import _add_answers_to_question +from core.shared import _build_thread +from core.shared import _get_ip +from core.shared import back +from core.shared import csrf +from core.shared import login_required +from core.shared import noindex +from core.shared import paginated_query +from core.shared import post_to_outbox from blueprints.api import _api_required from config import ADMIN_API_KEY from config import BLACKLIST diff --git a/blueprints/admin.py b/blueprints/admin.py index 2ecc948..3ed7967 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -20,15 +20,15 @@ from u2flib_server import u2f import config from activitypub import Box -from app_utils import MY_PERSON -from app_utils import _build_thread -from app_utils import _Response -from app_utils import csrf -from app_utils import login_required -from app_utils import noindex -from app_utils import p -from app_utils import paginated_query -from app_utils import post_to_outbox +from core.shared import MY_PERSON +from core.shared import _build_thread +from core.shared import _Response +from core.shared import csrf +from core.shared import login_required +from core.shared import noindex +from core.shared import p +from core.shared import paginated_query +from core.shared import post_to_outbox from config import DB from config import ID from config import PASS diff --git a/blueprints/api.py b/blueprints/api.py index 52aec64..2b33d3c 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -26,11 +26,11 @@ from werkzeug.utils import secure_filename import activitypub import config from activitypub import Box -from app_utils import MY_PERSON -from app_utils import _Response -from app_utils import back -from app_utils import csrf -from app_utils import post_to_outbox +from core.shared import MY_PERSON +from core.shared import _Response +from core.shared import back +from core.shared import csrf +from core.shared import post_to_outbox from config import BASE_URL from config import DB from config import DEBUG_MODE diff --git a/blueprints/indieauth.py b/blueprints/indieauth.py index 79696d5..a063387 100644 --- a/blueprints/indieauth.py +++ b/blueprints/indieauth.py @@ -16,8 +16,8 @@ from flask import session from flask import url_for from itsdangerous import BadSignature -from app_utils import _get_ip -from app_utils import login_required +from core.shared import _get_ip +from core.shared import login_required from config import DB from config import JWT diff --git a/blueprints/tasks.py b/blueprints/tasks.py index ec2a9a0..40a7bd6 100644 --- a/blueprints/tasks.py +++ b/blueprints/tasks.py @@ -13,15 +13,15 @@ from little_boxes.errors import NotAnActivityError from little_boxes.httpsig import HTTPSigAuth from requests.exceptions import HTTPError -import activity_gc +from core import gc import activitypub import config from activitypub import Box -from app_utils import MY_PERSON -from app_utils import _add_answers_to_question -from app_utils import back -from app_utils import p -from app_utils import post_to_outbox +from core.shared import MY_PERSON +from core.shared import _add_answers_to_question +from core.shared import back +from core.shared import p +from core.shared import post_to_outbox from config import DB from core.notifications import set_inbox_flags from tasks import Tasks @@ -457,7 +457,7 @@ def task_fetch_remote_question(): def task_cleanup(): task = p.parse(flask.request) app.logger.info(f"task={task!r}") - activity_gc.perform() + gc.perform() return "" diff --git a/config.py b/config.py index 040d7a7..bf15438 100644 --- a/config.py +++ b/config.py @@ -4,7 +4,6 @@ import subprocess from datetime import datetime from enum import Enum -import pymongo import yaml from itsdangerous import JSONWebSignatureSerializer from little_boxes import strtobool @@ -16,8 +15,6 @@ from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key from utils.media import MediaCache -from utils.meta import MetaKey -from utils.meta import _meta class ThemeStyle(Enum): @@ -33,16 +30,6 @@ DEFAULT_THEME_PRIMARY_COLOR = { } -def noop(): - pass - - -CUSTOM_CACHE_HOOKS = False -try: - from cache_hooks import purge as custom_cache_purge_hook -except ModuleNotFoundError: - custom_cache_purge_hook = noop - VERSION = ( subprocess.check_output(["git", "describe", "--always"]).split()[0].decode("utf-8") ) @@ -108,78 +95,6 @@ GRIDFS = mongo_client[f"{DB_NAME}_gridfs"] MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) -def create_indexes(): - if "trash" not in DB.collection_names(): - DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB - - DB.command("compact", "activities") - DB.activities.create_index([(_meta(MetaKey.NOTIFICATION), pymongo.ASCENDING)]) - DB.activities.create_index( - [(_meta(MetaKey.NOTIFICATION_UNREAD), pymongo.ASCENDING)] - ) - DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) - DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) - DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) - DB.activities.create_index( - [ - ("meta.thread_root_parent", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ] - ) - DB.activities.create_index( - [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] - ) - DB.cache2.create_index( - [ - ("path", pymongo.ASCENDING), - ("type", pymongo.ASCENDING), - ("arg", pymongo.ASCENDING), - ] - ) - DB.cache2.create_index("date", expireAfterSeconds=3600 * 12) - - # Index for the block query - DB.activities.create_index( - [ - ("box", pymongo.ASCENDING), - ("type", pymongo.ASCENDING), - ("meta.undo", pymongo.ASCENDING), - ] - ) - - # Index for count queries - DB.activities.create_index( - [ - ("box", pymongo.ASCENDING), - ("type", pymongo.ASCENDING), - ("meta.undo", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ] - ) - - DB.activities.create_index([("box", pymongo.ASCENDING)]) - - # Outbox query - DB.activities.create_index( - [ - ("box", pymongo.ASCENDING), - ("type", pymongo.ASCENDING), - ("meta.undo", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ("meta.public", pymongo.ASCENDING), - ] - ) - - DB.activities.create_index( - [ - ("type", pymongo.ASCENDING), - ("activity.object.type", pymongo.ASCENDING), - ("activity.object.inReplyTo", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ] - ) - - def _drop_db(): if not DEBUG_MODE: return diff --git a/activity_gc.py b/core/gc.py similarity index 100% rename from activity_gc.py rename to core/gc.py diff --git a/core/indexes.py b/core/indexes.py new file mode 100644 index 0000000..e756edd --- /dev/null +++ b/core/indexes.py @@ -0,0 +1,76 @@ +import pymongo +from config import DB +from utils.meta import _meta +from utils.meta import MetaKey + + +def create_indexes(): + if "trash" not in DB.collection_names(): + DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB + + DB.command("compact", "activities") + DB.activities.create_index([(_meta(MetaKey.NOTIFICATION), pymongo.ASCENDING)]) + DB.activities.create_index( + [(_meta(MetaKey.NOTIFICATION_UNREAD), pymongo.ASCENDING)] + ) + DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) + DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) + DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) + DB.activities.create_index( + [ + ("meta.thread_root_parent", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ] + ) + DB.activities.create_index( + [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] + ) + DB.cache2.create_index( + [ + ("path", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("arg", pymongo.ASCENDING), + ] + ) + DB.cache2.create_index("date", expireAfterSeconds=3600 * 12) + + # Index for the block query + DB.activities.create_index( + [ + ("box", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("meta.undo", pymongo.ASCENDING), + ] + ) + + # Index for count queries + DB.activities.create_index( + [ + ("box", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("meta.undo", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ] + ) + + DB.activities.create_index([("box", pymongo.ASCENDING)]) + + # Outbox query + DB.activities.create_index( + [ + ("box", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("meta.undo", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ("meta.public", pymongo.ASCENDING), + ] + ) + + DB.activities.create_index( + [ + ("type", pymongo.ASCENDING), + ("activity.object.type", pymongo.ASCENDING), + ("activity.object.inReplyTo", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ] + ) diff --git a/migrations.py b/core/migrations.py similarity index 100% rename from migrations.py rename to core/migrations.py diff --git a/app_utils.py b/core/shared.py similarity index 100% rename from app_utils.py rename to core/shared.py diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4e653e5..730f287 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,7 +7,7 @@ services: ports: - "27017:27017" poussetaches: - image: "poussetaches:latest" + image: "poussetaches/poussetaches:latest" volumes: - "${DATA_DIR}/poussetaches:/app/poussetaches_data" environment: diff --git a/run.sh b/run.sh index 313d405..b3fa579 100755 --- a/run.sh +++ b/run.sh @@ -1,4 +1,4 @@ #!/bin/bash -python -c "import logging; logging.basicConfig(level=logging.DEBUG); import migrations; migrations.perform()" -python -c "import config; config.create_indexes()" +python -c "import logging; logging.basicConfig(level=logging.DEBUG); from core import migrations; migrations.perform()" +python -c "from core import indexes; indexes.create_indexes()" gunicorn -t 600 -w 5 -b 0.0.0.0:5005 --log-level debug app:app