askiiart/microblog.pub
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Cleanup

Browse source
This commit is contained in:
Thomas Sileo 2019-07-21 21:47:52 +02:00
parent 2180a79cf2
commit 48618c9694
2 changed files with 49 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config.py
View file

@ -226,6 +226,7 @@ ME = {
"publicKey": KEY.to_dict(), "publicKey": KEY.to_dict(),
} }
# Default emojis, space-separated, update `me.yml` to customize emojis
EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾" EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾"
if conf.get("emojis"): if conf.get("emojis"):
EMOJIS = conf["emojis"] EMOJIS = conf["emojis"]
@ -235,5 +236,8 @@ EMOJI_TPL = '<img src="https://cdn.jsdelivr.net/npm/twemoji@12.0.0/2/svg/{filena
if conf.get("emoji_tpl"): if conf.get("emoji_tpl"):
EMOJI_TPL = conf["emoji_tpl"] EMOJI_TPL = conf["emoji_tpl"]
# Host blacklist # Hosts blacklist
BLACKLIST = conf.get("blacklist", []) BLACKLIST = conf.get("blacklist", [])
# By default, we keep 14 of inbox data ; outbox is kept forever (along with bookmarked stuff, outbox replies, liked...)
DAYS_TO_KEEP = 14

21
gc.py
View file

@ -1,10 +1,7 @@
import logging import logging
from datetime import datetime from datetime import datetime
from datetime import timedelta from datetime import timedelta
from typing import Any
from typing import Dict
from typing import List from typing import List
from urllib.parse import urlparse
from little_boxes import activitypub as ap from little_boxes import activitypub as ap
@ -13,9 +10,8 @@ from activitypub import Box
from config import ID from config import ID
from config import ME from config import ME
from config import MEDIA_CACHE from config import MEDIA_CACHE
from config import DAYS_TO_KEEP
from utils.migrations import DB from utils.migrations import DB
from utils.migrations import Migration
from utils.migrations import logger
back = activitypub.MicroblogPubBackend() back = activitypub.MicroblogPubBackend()
ap.use_backend(back) ap.use_backend(back)
@ -50,8 +46,9 @@ def threads_of_interest() -> List[str]:
def perform() -> None: def perform() -> None:
d = (datetime.utcnow() - timedelta(days=2)).strftime("%Y-%m-%d") d = (datetime.utcnow() - timedelta(days=DAYS_TO_KEEP)).strftime("%Y-%m-%d")
toi = threads_of_interest() toi = threads_of_interest()
logger.info(f"thread_of_interest={toi!r}")
# Go over the old Create activities # Go over the old Create activities
for data in DB.activities.find( for data in DB.activities.find(
@ -60,11 +57,12 @@ def perform() -> None:
"type": ap.ActivityType.CREATE.value, "type": ap.ActivityType.CREATE.value,
"activity.published": {"$lt": d}, "activity.published": {"$lt": d},
} }
).limit(1000): ):
try:
remote_id = data["remote_id"] remote_id = data["remote_id"]
meta = data["meta"] meta = data["meta"]
activity = ap.parse_activity(data["activity"]) activity = ap.parse_activity(data["activity"])
logger.info(f"{activity}") logger.info(f"activity={activity!r}")
# This activity has been bookmarked, keep it # This activity has been bookmarked, keep it
if meta.get("bookmarked"): if meta.get("bookmarked"):
@ -92,9 +90,16 @@ def perform() -> None:
if meta.get("boosted") or meta.get("liked"): if meta.get("boosted") or meta.get("liked"):
continue continue
# TODO(tsileo): remove after tests
if meta.get("keep"):
logger.warning(f"{activity!r} would not have been deleted, skipping for now")
continue
# Delete the cached attachment # Delete the cached attachment
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id) MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity # Delete the activity
DB.activities.delete_one({"_id": data["_id"]}) DB.activities.delete_one({"_id": data["_id"]})
except Exception:
logger.exception(f"failed to process {data!r}")